The Complete Production SOP: How to Deploy and Run Trafilatura Precision Web Article Text Extractor in Autonomous AI Agents

---
name: "trafilatura-web-content-extractor"
description: "Extract core article text, metadata, and comments from web pages with peak precision."
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import trafilatura

def extract_clean_article(raw_html: str) -> dict:
    """Deterministic web article extraction SOP using Trafilatura."""
    # Primary extraction: strip boilerplate, keep markdown formatting and tables
    extracted_text = trafilatura.extract(
        raw_html,
        output_format="markdown",
        include_tables=True,
        include_comments=False,
        favor_recall=False,
        deduplicate=True
    )
    
    # Fallback heuristic if DOM is non-standard
    if not extracted_text:
        extracted_text = trafilatura.extract(
            raw_html,
            output_format="markdown",
            favor_recall=True
        )

    # Extract structural document metadata
    meta = trafilatura.extract_metadata(raw_html)
    
    return {
        "title": meta.title if meta else None,
        "author": meta.author if meta else None,
        "date": meta.date if meta else None,
        "sitename": meta.sitename if meta else None,
        "content": extracted_text or ""
    }

Please extract main article text trafilatura from the local HTML document stored in `fixtures/sample_article.html`. Strip boilerplate from web page and extract article metadata python into a JSON output format.

import unittest
from extractor import extract_clean_article

class TestExtractor(unittest.TestCase):
    def test_boilerplate_removal(self):
        mock_html = """
        <html><head><title>Engineering Insights</title></head>
        <body>
            <header><nav>Home | Pricing | Contact Us</nav></header>
            <article>
                <h1>Building Agentic Workflows</h1>
                <p class='author'>By Jane Doe</p>
                <p>Autonomous agents require deterministic tools to reliably process web documents.</p>
            </article>
            <aside><button>Subscribe to Newsletter</button></aside>
            <footer>Cookie Policy - All Rights Reserved</footer>
        </body></html>
        """
        result = extract_clean_article(mock_html)
        self.assertEqual(result["title"], "Engineering Insights")
        self.assertIn("Autonomous agents require deterministic tools", result["content"])
        self.assertNotIn("Subscribe to Newsletter", result["content"])
        self.assertNotIn("Cookie Policy", result["content"])

if __name__ == '__main__':
    unittest.main()