The Complete Production SOP: How to Deploy and Run Firecrawl LLM-Ready Web Harvester in Autonomous AI Agents

---
name: "firecrawl-crawler"
description: "Convert entire web domains into clean, token-efficient Markdown suitable for AI training and RAG."
dependencies:
  npm:
    - "@mendable/firecrawl-js"
runtimes:
  - "codex"
  - "claude-code"
  - "cursor"
  - "windsurf"
---

# Firecrawl LLM-Ready Web Harvester SOP

1. Use `@mendable/firecrawl-js` exclusively for domain traversal and Markdown conversion.
2. Output crawled documents into `./crawls/<sanitized-domain>/` as individual `.md` files.
3. Enforce maximum crawl depth and concurrency boundaries to prevent rate-limit exhaustion.
4. Ensure `.env` contains `FIRECRAWL_API_KEY` before triggering crawl execution.

import FirecrawlApp from '@mendable/firecrawl-js';
import * as fs from 'fs';
import * as path from 'path';
import 'dotenv/config';

const apiKey = process.env.FIRECRAWL_API_KEY;
if (!apiKey) {
  throw new Error('FIRECRAWL_API_KEY is not defined in environment variables.');
}

export const app = new FirecrawlApp({ apiKey });

export async function harvestDomain(targetUrl: string, maxPages: number = 10) {
  const crawlResponse = await app.crawlUrl(targetUrl, {
    limit: maxPages,
    scrapeOptions: {
      formats: ['markdown'],
      onlyMainContent: true
    }
  });

  if (!crawlResponse.success) {
    throw new Error(`Crawl failed: ${crawlResponse.error}`);
  }

  return crawlResponse.data;
}

# Execute crawl automation through TypeScript runtime
npx ts-node scripts/crawl.ts --url "https://docs.example.com" --limit 25

import * as fs from 'fs';
import * as path from 'path';

describe('Crawl Output Integrity', () => {
  const outDir = path.resolve(__dirname, '../crawls');

  test('crawler output directory exists and has markdown files', () => {
    expect(fs.existsSync(outDir)).toBe(true);
    const mdFiles = fs.readdirSync(outDir).filter(f => f.endsWith('.md'));
    expect(mdFiles.length).toBeGreaterThan(0);

    mdFiles.forEach(file => {
      const content = fs.readFileSync(path.join(outDir, file), 'utf8');
      expect(content.trim().length).toBeGreaterThan(100);
      expect(content).not.toContain('<nav');
      expect(content).not.toContain('<script');
    });
  });
});