The Complete Production SOP: How to Deploy and Run Vaex Out-of-Core Billion Row Data Exploration in Autonomous AI Agents

---
name: "vaex-outofcore-dataframe"
description: "Visualize and compute statistics on 1 billion rows without loading into RAM using Vaex."
metadata:
  author: "Vaex"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "explore billion row dataset"
  - "use vaex out of core"
  - "calculate histogram vaex"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

#!/usr/bin/env bash
python -c "import vaex; print(f'Vaex version: {vaex.__version__}')"
python -c "import pyarrow; print(f'PyArrow version: {pyarrow.__version__}')"

import vaex

def run_analysis(input_path: str):
    # Open memory-mapped dataset lazily (0-RAM consumption)
    df = vaex.open(input_path)
    print(f"Dataset registered with {len(df):,} rows.")

    # Create virtual column (computed on-the-fly, zero copy)
    df['normalized_val'] = df.raw_val / 100.0

    # Compute lazy aggregations across full dataset
    mean_val = df.mean(df.normalized_val)
    count_filtered = df.count(selection=df.status == 'PROCESSED')

    # Generate 1D binned histogram for distribution without loading rows
    counts = df.count(binby=df.normalized_val, shape=10, limits=[0, 100])

    print(f"Mean: {mean_val}")
    print(f"Filtered count: {count_filtered}")
    print(f"Histogram bins: {counts}")

if __name__ == '__main__':
    run_analysis('massive_data.parquet')

import resource
import sys
import vaex

df = vaex.open('massive_data.parquet')
# Execute computation
_ = df.mean(df.raw_val)

max_mem_mb = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024
if sys.platform == 'darwin':
    max_mem_mb = max_mem_mb / 1024  # Darwin reports bytes

print(f"Resident Memory Usage: {max_mem_mb:.2f} MB")
assert max_mem_mb < 512, f"Memory threshold exceeded: {max_mem_mb} MB"