The Complete Production SOP: How to Deploy and Run Pandas Memory Downcasting & Categorical Dtypes in Autonomous AI Agents

---
name: "pandas-memory-downcasting-dtypes"
description: "Shrink Pandas DataFrame memory consumption by up to 80% using optimal dtypes."
metadata:
  author: "Pandas"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "reduce pandas memory usage"
  - "downcast pandas dtypes"
  - "use categorical dtype pandas"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import pandas as pd

def optimize_memory_usage(df: pd.DataFrame, categorical_threshold: float = 0.5) -> pd.DataFrame:
    """Downcasts numeric columns and converts low-cardinality string columns to categories."""
    start_mem = df.memory_usage(deep=True).sum() / (1024 ** 2)
    for col in df.columns:
        col_type = df[col].dtype
        if col_type == 'object' or isinstance(col_type, pd.StringDtype):
            num_unique = df[col].nunique()
            num_total = len(df[col])
            if num_total > 0 and (num_unique / num_total) < categorical_threshold:
                df[col] = df[col].astype('category')
        elif 'int' in str(col_type):
            df[col] = pd.to_numeric(df[col], downcast='integer')
        elif 'float' in str(col_type):
            df[col] = pd.to_numeric(df[col], downcast='float')
    end_mem = df.memory_usage(deep=True).sum() / (1024 ** 2)
    print(f'Memory reduced from {start_mem:.2f} MB to {end_mem:.2f} MB (-{(1 - end_mem/start_mem)*100:.1f}%)')
    return df

# In Claude Code CLI:
claude "downcast pandas dtypes in src/etl/pipeline.py and reduce pandas memory usage"

# In Codex CLI:
codex exec "use categorical dtype pandas for transaction logs in data_loader.py"

import numpy as np
import pandas as pd
from memory_utils import optimize_memory_usage

def test_optimize_memory_parity():
    raw_data = {
        'id': [1, 2, 3, 4, 5] * 1000,
        'val': [1.123, 2.345, 3.456, 4.567, 5.678] * 1000,
        'status': ['PENDING', 'ACTIVE', 'COMPLETED', 'FAILED', 'PENDING'] * 1000
    }
    df_before = pd.DataFrame(raw_data)
    df_after = optimize_memory_usage(df_before.copy())
    
    assert df_after['status'].dtype.name == 'category'
    assert df_after['id'].dtype in ('int8', 'int16')
    assert np.allclose(df_before['val'], df_after['val'], rtol=1e-5)
    assert df_after.memory_usage(deep=True).sum() < df_before.memory_usage(deep=True).sum()