The Complete Production SOP: How to Deploy and Run High-Throughput Streaming CSV/TSV Parser in Autonomous AI Agents

---
name: "csv-tsv-streaming-large-parser"
description: "Process gigabyte-scale delimited CSV and TSV files with streaming Node.js pipelines."
metadata:
  author: "mafintosh"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "stream parse large csv nodejs"
  - "handle rfc4180 quotes csv"
  - "convert tsv to csv stream"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

# High-Throughput Streaming CSV/TSV Parser
Streams massive delimited datasets using csv-parser or fast-csv in Node.js with stream chunking, RFC 4180 quote escaping, and backpressure management.

import fs from 'node:fs';
import { pipeline } from 'node:stream/promises';
import csvParser from 'csv-parser';
import stripBomStream from 'strip-bom-stream';
import { Transform } from 'node:stream';

export async function processDelimitedFile(sourcePath, separator = ',') {
  let processedCount = 0;
  const transformer = new Transform({
    objectMode: true,
    async transform(row, encoding, callback) {
      try {
        // Process row asynchronously with backpressure guarantee
        processedCount++;
        callback(null, row);
      } catch (err) {
        callback(err);
      }
    }
  });

  await pipeline(
    fs.createReadStream(sourcePath),
    stripBomStream(),
    csvParser({
      separator,
      mapHeaders: ({ header }) => header.trim().toLowerCase(),
      strict: false
    }),
    transformer
  );

  return processedCount;
}

import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { processDelimitedFile } from '../src/stream-pipeline.js';

test('processes quoted records with embedded newlines correctly', async () => {
  const fixturePath = 'test/fixtures/rfc4180.csv';
  fs.mkdirSync('test/fixtures', { recursive: true });
  fs.writeFileSync(fixturePath, 'id,notes\n1,"Line 1\nLine 2"\n2,"Regular text"\n');
  
  const count = await processDelimitedFile(fixturePath, ',');
  assert.equal(count, 2);
  fs.unlinkSync(fixturePath);
});