The Complete Production SOP: How to Deploy and Run Web Browser Agent Accessibility Tree & Actions in Autonomous AI Agents

---
name: "browser-agent-dom-action-tree"
description: "Control web browsers autonomously using accessibility DOM trees and click coordinates."
metadata:
  author: "Browserbase"
  version: "1.0.0"
  runtime_compatibility:
    - "codex"
    - "claude-code"
    - "cursor"
    - "windsurf"
triggers:
  - "control web browser with ai"
  - "extract accessibility tree for agent"
  - "click button in browser agent"
permissions:
  fs_read: true
  fs_write: "workspace"
  shell_exec: false
  net_access: false
---

import { chromium } from 'playwright';
import * as fs from 'fs';

export async function captureActionTree(url: string, outputPath: string = 'dist/a11y-tree.json') {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();
  await page.goto(url, { waitUntil: 'domcontentloaded' });
  
  const snapshot = await page.accessibility.snapshot({ interestingOnly: true });
  let elementCounter = 1;
  
  function assignNumericIds(node: any): any {
    if (!node) return null;
    const isInteractive = ['button', 'link', 'textbox', 'checkbox', 'combobox'].includes(node.role);
    const enrichedNode = {
      id: isInteractive ? elementCounter++ : null,
      role: node.role,
      name: node.name || '',
      value: node.value || undefined,
      children: node.children ? node.children.map(assignNumericIds).filter(Boolean) : []
    };
    return enrichedNode;
  }

  const indexedTree = assignNumericIds(snapshot);
  fs.writeFileSync(outputPath, JSON.stringify(indexedTree, null, 2));
  await browser.close();
}

import { Page } from 'playwright';

// Agent executes actions based on numeric action mappings
export async function executeAgentAction(page: Page, action: { type: 'click' | 'fill', id: number, text?: string }, idMap: Record<number, string>) {
  const selector = idMap[action.id];
  if (!selector) throw new Error(`Action ID ${action.id} not found in DOM map.`);
  
  if (action.type === 'click') {
    await page.click(selector);
  } else if (action.type === 'fill' && action.text !== undefined) {
    await page.fill(selector, action.text);
  }
  await page.waitForLoadState('networkidle');
}

import { test, expect } from '@playwright/test';

test('verify deterministic browser agent action flow', async ({ page }) => {
  await page.goto('http://localhost:3000/checkout');
  await page.getByRole('textbox', { name: 'Full Name' }).fill('Ada Lovelace');
  await page.getByRole('button', { name: 'Complete Order' }).click();
  await expect(page.getByRole('heading', { name: 'Order Confirmed' })).toBeVisible();
});