The Complete Production SOP: How to Deploy and Run Distributed Cron Jobs & Task Queues in Autonomous AI Agents

---
name: "cron-scheduler"
description: "Schedule recurring cron jobs, retry policies, and background worker queues with BullMQ."
category: "devops"
dependencies: {"npm": ["bullmq", "ioredis"]}
---

# Distributed Cron Jobs & Task Queues SOP

1. Always create dedicated `Queue`, `Worker`, and `QueueScheduler` instances with isolated `ioredis` connection handles.
2. Mandate `attempts: 3` with `backoff: { type: 'exponential', delay: 1000 }` for all registered jobs.
3. Implement explicit Dead Letter Queue (DLQ) handlers for jobs transitioning to 'failed'.
4. Never create inline `new Redis()` connections inside hot worker execution loops.

import { Redis, RedisOptions } from 'ioredis';

const redisConfig: RedisOptions = {
  host: process.env.REDIS_HOST || '127.0.0.1',
  port: Number(process.env.REDIS_PORT) || 6379,
  maxRetriesPerRequest: null,
  enableReadyCheck: false,
  lazyConnect: true,
};

export const redisConnection = new Redis(redisConfig);

import { Queue, Worker, Job } from 'bullmq';
import { redisConnection } from './connection';

export const billingQueue = new Queue('billing-sync', { connection: redisConnection });

export async function registerBillingCron() {
  await billingQueue.upsertJobScheduler(
    'daily-billing-sync',
    { pattern: '0 0 * * *' },
    {
      name: 'execute-sync',
      data: { trigger: 'cron', tenantId: 'global' },
      opts: {
        attempts: 3,
        backoff: { type: 'exponential', delay: 2000 },
        removeOnComplete: 100,
        removeOnFail: 500,
      },
    }
  );
}

export const billingWorker = new Worker(
  'billing-sync',
  async (job: Job) => {
    // Task payload execution
    return { success: true, processedAt: new Date().toISOString() };
  },
  { connection: redisConnection, concurrency: 5 }
);

import { billingQueue, registerBillingCron } from './scheduler';
import { redisConnection } from './connection';

async function verify() {
  await redisConnection.connect();
  await registerBillingCron();
  const schedulers = await billingQueue.getJobSchedulers();
  const isRegistered = schedulers.some((s) => s.id === 'daily-billing-sync');
  
  if (!isRegistered) {
    throw new Error('Verification failed: Cron scheduler not registered.');
  }
  console.log('✓ BullMQ cron scheduler validated successfully.');
  await redisConnection.quit();
}

verify().catch((err) => {
  console.error(err);
  process.exit(1);
});