Running Compliance Tests

# From SDK repo (with compliance submodule)
cd compliance

# Run all tests
python -m compliance.runner

# Run specific test
python -m compliance.runner tests/transport/completions-basic.md

# Run by tag
python -m compliance.runner --tag core

python -m compliance.runner [options] [test_files...]

Options:
  --base-url URL       SDK server URL (default: http://localhost:8765)
  --tag TAG            Run tests with specific tag
  --skip-tag TAG       Skip tests with specific tag
  --strict-only        Only run strict assertions (no LLM)
  --cache-mode MODE    Cache mode: read_write, write_only, disabled
  --temperature TEMP   LLM temperature (default: 0)
  --timeout SECONDS    Test timeout (default: 60)
  --parallel N         Run N tests in parallel
  --output FORMAT      Output format: text, json, junit
  --verbose            Show detailed output

# All compliance tests
python -m compliance.runner

# Specific transport
python -m compliance.runner tests/transport/

# Single test
python -m compliance.runner tests/transport/completions-basic.md

Compliance Test Results
=======================

✓ completions-basic: 3/3 passed
  ✓ 1. Simple Message
  ✓ 2. Empty Message Handling  
  ✓ 3. Streaming Response

✓ completions-tools: 2/2 passed
  ✓ 1. Tool Call Request
  ✓ 2. Tool Result Handling

✗ multiagent-handoff: 1/2 passed
  ✓ 1. Successful Handoff
  ✗ 2. Auth Required Handoff
    Assertion failed: "Authorization header present"
    Response did not include Authorization header

Summary: 6/7 tests passed (85.7%)

# Use cache (default)
python -m compliance.runner --cache-mode read_write

# Refresh cache
python -m compliance.runner --cache-mode write_only

# No cache
python -m compliance.runner --cache-mode disabled

compliance/
├── .cache/
│   ├── openai/
│   │   └── {hash}.json
│   └── manifest.json

# Clear all cache
rm -rf compliance/.cache/

# Clear specific provider
rm -rf compliance/.cache/openai/

name: Compliance Tests

on: [push, pull_request]

jobs:
  compliance:
    runs-on: ubuntu-latest
    
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true  # For compliance submodule
      
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      
      - name: Install dependencies
        run: |
          pip install -e .
          pip install -r compliance/requirements.txt
      
      - name: Start server
        run: |
          webagents serve --port 8765 &
          sleep 5
      
      - name: Run compliance tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          python -m compliance.runner \
            --base-url http://localhost:8765 \
            --cache-mode read_write \
            --output junit \
            > test-results.xml
      
      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: compliance-results
          path: test-results.xml

      - name: Cache compliance responses
        uses: actions/cache@v4
        with:
          path: compliance/.cache
          key: compliance-cache-${{ hashFiles('compliance/tests/**/*.md') }}
          restore-keys: |
            compliance-cache-

python -m compliance.runner --strict-only

python -m compliance.runner --output json

{
  "summary": {
    "total": 7,
    "passed": 6,
    "failed": 1,
    "skipped": 0
  },
  "tests": [
    {
      "name": "completions-basic",
      "cases": [
        {"name": "Simple Message", "status": "passed"},
        {"name": "Empty Handling", "status": "passed"}
      ]
    }
  ]
}

python -m compliance.runner --output junit > results.xml

python -m compliance.runner --verbose tests/transport/completions-basic.md

python -m compliance.runner \
  tests/transport/completions-basic.md \
  --test-case "Simple Message"

DEBUG=compliance python -m compliance.runner