The Complete Production SOP: How to Deploy and Run Newspaper4k Multi-Language News Article Scraper in Autonomous AI Agents

---
name: "newspaper4k-article-scraper"
description: "Scrape, parse, and analyze multilingual news articles with Newspaper4k."
metadata:
  author: "Newspaper4k"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "scrape news article python"
  - "extract news image and author"
  - "newspaper4k article parsing"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import nltk
from newspaper import Article

def extract_article_data(target_url: str, lang: str = 'en') -> dict:
    article = Article(target_url, language=lang)
    article.download()
    article.parse()
    article.nlp()
    return {
        'title': article.title,
        'authors': article.authors,
        'publish_date': str(article.publish_date),
        'summary': article.summary,
        'keywords': article.keywords,
        'top_image': article.top_image,
        'text': article.text
    }

import json
from scraper_pipeline import extract_article_data

try:
    data = extract_article_data('https://example.com/world/news-2026', lang='en')
    with open('output.json', 'w', encoding='utf-8') as f:
        json.dump(data, f, indent=2, ensure_ascii=False)
except Exception as err:
    print(f'Ingestion failed: {err}')
    exit(1)

#!/usr/bin/env bash
set -euo pipefail

if [ ! -s "output.json" ]; then
  echo "Error: output.json is empty or missing" >&2
  exit 1
fi

python -c "import json; d=json.load(open('output.json')); print(f'Successfully parsed: {d.get(\"title\")}')"