The Complete Production SOP: How to Deploy and Run LanceDB Serverless Vector Database on Apache Arrow in Autonomous AI Agents

---
name: "lancedb-serverless-arrow-vectors"
description: "Execute serverless, disk-based vector search directly from S3 or local disk using LanceDB."
metadata:
  author: "LanceDB"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "query lancedb arrow table"
  - "create lancedb ivf pq index"
  - "serverless vector search lancedb"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import os
import lancedb

def get_vector_db():
    db_uri = os.getenv("LANCEDB_STORAGE_URI", "./data/lancedb")
    storage_options = {}
    if db_uri.startswith("s3://"):
        storage_options = {
            "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
            "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
            "aws_region": os.getenv("AWS_REGION", "us-east-1"),
        }
    return lancedb.connect(db_uri, storage_options=storage_options)

import lancedb
import pyarrow as pa
import numpy as np

def init_and_index_vectors(db: lancedb.DBConnection, table_name: str, schema: pa.Schema, data: list):
    tbl = db.create_table(table_name, data=data, schema=schema, mode="overwrite")
    # Enforce IVF-PQ index creation per SOP
    tbl.create_index(
        metric="cosine",
        vector_column_name="vector",
        num_partitions=64,
        num_sub_vectors=16,
        replace=True
    )
    return tbl

def search_vectors(tbl, query_vector: list, top_k: int = 10):
    return tbl.search(query_vector).metric("cosine").limit(top_k).to_arrow()

import pytest
import numpy as np
import lancedb

def test_lancedb_vector_lifecycle(tmp_path):
    db = lancedb.connect(str(tmp_path))
    dim = 128
    num_rows = 1000
    data = [
        {"id": i, "vector": np.random.randn(dim).astype(np.float32).tolist(), "doc": f"doc_{i}"}
        for i in range(num_rows)
    ]
    tbl = db.create_table("test_tbl", data=data)
    tbl.create_index(metric="cosine", num_partitions=16, num_sub_vectors=8)
    query = np.random.randn(dim).astype(np.float32).tolist()
    results = tbl.search(query).limit(5).to_arrow()
    assert len(results) == 5
    assert "vector" in results.column_names