The Complete Production SOP: How to Deploy and Run Firecrawl Clean Markdown Extraction for LLMs in Autonomous AI Agents

---
name: "firecrawl-web-to-markdown"
description: "Turn entire websites into clean, LLM-ready Markdown and structured JSON with Firecrawl."
metadata:
  author: "Mendable"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "scrape website to markdown firecrawl"
  - "crawl entire domain firecrawl"
  - "extract structured json firecrawl"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import os
from firecrawl import FirecrawlApp
from dotenv import load_dotenv

load_dotenv('.env.local')
api_key = os.getenv('FIRECRAWL_API_KEY')
if not api_key:
    raise ValueError('Missing FIRECRAWL_API_KEY in environment')

app = FirecrawlApp(api_key=api_key)

# Deterministic execution script following Firecrawl Clean Markdown SOP
python3 -c "
import os, json
from firecrawl import FirecrawlApp
app = FirecrawlApp(api_key=os.getenv('FIRECRAWL_API_KEY'))
res = app.scrape_url('https://docs.firecrawl.dev', params={'formats': ['markdown']})
os.makedirs('cache', exist_ok=True)
with open('cache/docs.md', 'w') as f:
    f.write(res.get('markdown', ''))
print('Extraction verified: cache/docs.md written successfully.')
"

import sys, json
from pathlib import Path

def verify(filepath: str) -> None:
    path = Path(filepath)
    if not path.exists():
        sys.exit(f'Error: File {filepath} was not generated.')
    content = path.read_text()
    if len(content.strip()) < 50:
        sys.exit('Error: Output content contains insufficient data.')
    print(f'Verification passed: {filepath} ({len(content)} bytes)')

if __name__ == '__main__':
    verify('cache/docs.md')