pip install arize-ax-cli

git clone https://github.com/Arize-ai/arize-ax-cli.git
cd arize-ax-cli
pip install -e .

ax --version

ax profiles create

ax profiles show

ax datasets list

ax projects list

ax spans export <project-id> --stdout

ax traces list <project-id>

# Create with flags (no prompts)
ax profiles create staging --api-key ak_abc123 --region US --output-format json

# Create from a TOML file
ax profiles create production --from-file ./prod.toml

# Create from file and override the API key
ax profiles create production --from-file ./prod.toml --api-key ak_override

# Update the API key in the active profile
ax profiles update --api-key ak_new_key

# Update the region in a named profile
ax profiles update production --region EU

# Replace an entire profile from a TOML file
ax profiles update production --from-file ./prod.toml

# Load from file and override the API key
ax profiles update staging --from-file ./staging.toml --api-key ak_override

ax profiles create

ax profiles show --expand

# Create a production profile (name as argument skips the name prompt)
ax profiles create production

# Create a staging profile interactively
ax profiles create staging

# List all profiles
ax profiles list

# Switch profiles
ax profiles use production
ax profiles use staging

# Update a field in a specific profile
ax profiles update staging --region EU

# Run commands against a different profile by switching first
ax profiles use production
ax datasets list

# Delete a profile (prompts for confirmation)
ax profiles delete staging

# Delete a profile without confirmation
ax profiles delete staging --force

ax --install-completion

ax <TAB>         # Shows available commands (cache, datasets, experiments, profiles, projects, spans, traces)
ax datasets <TAB> # Shows dataset subcommands (list, get, export, create, append, update-examples, delete-examples, update, delete)
ax datasets list --<TAB>  # Shows available options

# View the completion script for your shell
ax --show-completion

# Save it to a file and source it manually
ax --show-completion >> ~/.bashrc  # For bash
ax --show-completion >> ~/.zshrc   # For zsh

# List AI integrations
ax ai-integrations list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get a specific integration
ax ai-integrations get <integration>

# Create an integration (OpenAI example)
ax ai-integrations create --name "OpenAI Prod" --provider OPEN_AI \
  --api-key <key> --model-name gpt-4o --model-name gpt-4o-mini

# Create an integration with custom headers
ax ai-integrations create --name "Custom LLM" --provider CUSTOM \
  --base-url https://my-llm.example.com \
  --headers-json '{"X-API-Key": "secret"}'

# Create an AWS Bedrock integration
ax ai-integrations create --name "Bedrock" --provider AWS_BEDROCK \
  --provider-metadata-json '{"role_arn": "arn:aws:iam::123456789:role/MyRole"}'

# Update an integration
ax ai-integrations update <integration> --name "Renamed" --api-key <new-key>

# Delete an integration
ax ai-integrations delete <integration> [--force]

# List annotation configs
ax annotation-configs list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get a specific annotation config
ax annotation-configs get <annotation-config>

# Create a freeform annotation config (free-text feedback)
ax annotation-configs create --name "Quality" --space <space> --type FREEFORM

# Create a continuous annotation config (numeric score range)
ax annotation-configs create --name "Score" --space <space> --type CONTINUOUS \
  --min-score 0 --max-score 1 --optimization-direction MAXIMIZE

# Create a categorical annotation config (discrete labels)
ax annotation-configs create --name "Verdict" --space <space> --type CATEGORICAL \
  --value good --value neutral --value bad --optimization-direction MAXIMIZE

# Update an annotation config (pick the subcommand for its type; only the fields you pass change)
ax annotation-configs update freeform <annotation-config> --new-name "New name"
ax annotation-configs update continuous <annotation-config> --min-score 0 --max-score 10
ax annotation-configs update categorical <annotation-config> --value good --value bad

# Delete an annotation config
ax annotation-configs delete <annotation-config> [--force]

# List annotation queues
ax annotation-queues list [--space <space>] [--name <substring>] [--limit 15] [--cursor <cursor>]

# Get a specific annotation queue
ax annotation-queues get <queue>

# Create an annotation queue (at least one --annotation-config-id required)
ax annotation-queues create --name "My Queue" --space <space> \
  --annotation-config-id <config-id>

# Create a queue with annotators and assignment method
ax annotation-queues create --name "My Queue" --space <space> \
  --annotation-config-id <config-id> \
  --annotator-email alice@example.com --annotator-email bob@example.com \
  --instructions "Please evaluate carefully" \
  --assignment-method RANDOM

# Create a queue and seed it with initial records from a file
ax annotation-queues create --name "My Queue" --space <space> \
  --annotation-config-id <config-id> \
  --record-sources sources.json

# Update a queue (list fields fully replace existing values)
ax annotation-queues update <queue> [--name <name>] [--instructions <text>] \
  [--annotation-config-id <id>] [--annotator-email <email>]

# Delete a queue
ax annotation-queues delete <queue> [--force]

# List records in a queue
ax annotation-queues list-records <queue> [--space <space>] [--limit 15] [--cursor <cursor>]

# Delete records from a queue
ax annotation-queues delete-records <queue> --record-id <id> [--record-id <id> ...] [--force]

# Annotate a record (upserted by annotation config name; call again for additional configs)
ax annotation-queues annotate-record <queue> <record-id> \
  --annotation-name "quality" [--score 0.9] [--label good] [--text "Looks great"]

# Assign users to a record (replaces all existing assignments)
ax annotation-queues assign-record <queue> <record-id> \
  --email alice@example.com [--email bob@example.com]

# Remove all assignments from a record
ax annotation-queues assign-record <queue> <record-id>

# Add records to a queue from a file
ax annotation-queues add-records <queue> \
  --space <space> \
  --record-sources sources.json

# Add records inline (span source)
ax annotation-queues add-records <queue> \
  --record-sources '[{"record_type": "SPAN", "project_id": "<proj>",
    "start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-02T00:00:00Z"}]'

# Add records inline (dataset example source)
ax annotation-queues add-records <queue> \
  --record-sources '[{"record_type": "EXAMPLE", "dataset_id": "<ds>",
    "example_ids": ["ex-1", "ex-2"]}]'

# List API keys
ax api-keys list [--key-type USER|SERVICE] [--status ACTIVE|REVOKED] \
  [--limit 15] [--cursor <cursor>]

# Create a user key (authenticates as you)
ax api-keys create --name "My Key" [--description "..."] [--expires-at 2025-12-31T23:59:59]

# Create a service key (scoped to one or more orgs and spaces)
ax api-keys create-service-key --name "CI bot" \
  --assignments '[{"org_id":"<org-id>","role":"READ_ONLY",
                   "spaces":[{"space":"prod","role":"MEMBER"},
                              {"space":"staging"}]}]' \
  [--account-role MEMBER] [--description "..."] [--expires-at 2025-12-31T23:59:59]

# Load assignments from a file
ax api-keys create-service-key --name "CI bot" --assignments bindings.json

# Refresh a key (revokes old key, issues replacement)
ax api-keys refresh <key-id> [--expires-at 2025-12-31T23:59:59] \
  [--grace-period-seconds 300]

# Revoke a key
ax api-keys revoke <key-id> [--force]

# Clear the cache
ax cache clear

# List resource restrictions you can manage
ax resource-restrictions list [--resource-type PROJECT] [--limit 50] [--cursor <cursor>]

# Restrict a resource (prevents higher-level roles from granting access)
ax resource-restrictions restrict --resource-id <global-id>

# Remove a restriction from a resource
ax resource-restrictions unrestrict --resource-id <global-id> [--force]

# List datasets
ax datasets list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get dataset metadata
ax datasets get <dataset>

# Export all examples to a file
ax datasets export <dataset> [--version-id <version-id>] [--output-dir .] [--stdout]

# Create a new dataset
ax datasets create --name "My Dataset" --space <space> --file data.csv

# Create a dataset from stdin (pipe or heredoc)
ax datasets create --name "My Dataset" --space <space> --file - < data.json

# Append examples (inline JSON)
ax datasets append <dataset> --json '[{"question": "...", "answer": "..."}]'

# Append examples (from file)
ax datasets append <dataset> --file new_examples.csv [--version-id <version-id>]

# Append examples from stdin
ax datasets append <dataset> --file -

# Update existing examples in place, matched by ID (patch — omitted fields are kept)
ax datasets update-examples <dataset> --json '[{"id": "<example-id>", "answer": "..."}]'

# Update examples and capture the change as a new dataset version
ax datasets update-examples <dataset> --file updates.csv --new-version "v2"

# Delete specific examples from a dataset version (inline JSON array of IDs)
ax datasets delete-examples <dataset> --version-id <version-id> \
  --example-ids '["<example-id>", "<example-id>"]' [--force]

# Update a dataset (e.g. rename)
ax datasets update <dataset> --new-name "New Name" [--space <space>]

# Delete a dataset
ax datasets delete <dataset> [--force]

# List evaluators (optionally filtered by space)
ax evaluators list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get an evaluator with its latest version
ax evaluators get <evaluator>

# Get an evaluator at a specific version
ax evaluators get <evaluator> --version-id <version-id>

# Create a new template evaluator
ax evaluators create-template-evaluator \
  --name "Response Relevance" \
  --space <space> \
  --commit-message "Initial version" \
  --template-name relevance \
  --template "Is this response relevant to the query? {{input}} {{output}}" \
  --ai-integration-id <integration-id> \
  --model-name gpt-4o

# Create a classification evaluator (label → numeric score; omit flag for freeform)
ax evaluators create-template-evaluator \
  --name "Relevance classifier" \
  --space <space> \
  --commit-message "Initial version" \
  --template-name relevance \
  --template "Classify: {{output}}" \
  --ai-integration-id <integration-id> \
  --model-name gpt-4o \
  --classification-choices '{"relevant":1,"irrelevant":0}' \
  --direction maximize \
  --data-granularity SPAN

# Update evaluator metadata
ax evaluators update <evaluator> --name "New Name"
ax evaluators update <evaluator> --description "Updated description"

# Delete an evaluator (and all its versions)
ax evaluators delete <evaluator> [--force]

# List all versions of an evaluator
ax evaluators list-versions <evaluator-id> [--limit 15] [--cursor <cursor>]

# Get a specific version by ID
ax evaluators get-version <version-id>

# Create a new template version of an existing evaluator
ax evaluators create-template-evaluator-version <evaluator-id> \
  --commit-message "Improved prompt" \
  --template-name relevance \
  --template "Rate the relevance of the response: {{input}} {{output}}" \
  --ai-integration-id <integration-id> \
  --model-name gpt-4o

# Same optional template fields apply (e.g. classification choices)
ax evaluators create-template-evaluator-version <evaluator-id> \
  --commit-message "Add rails" \
  --template-name relevance \
  --template "Classify: {{output}}" \
  --ai-integration-id <integration-id> \
  --model-name gpt-4o \
  --classification-choices '{"relevant":1,"irrelevant":0}'

# Managed built-in: regex check
ax evaluators create-code-evaluator \
  --name "Regex Check" \
  --space <space> \
  --commit-message "Initial version" \
  --code-type managed \
  --code-name regex_match \
  --managed-evaluator MATCHES_REGEX \
  --variables '["output"]' \
  --static-params '[{"name":"pattern","type":"REGEX","default_value":"^yes"}]'

# Custom Python, loading source from a file (use @ prefix)
ax evaluators create-code-evaluator \
  --name "Custom Eval" \
  --space <space> \
  --commit-message "Initial version" \
  --code-type custom \
  --code-name my_eval \
  --code @./evaluator.py \
  --imports @./imports.py \
  --variables '["input","output"]'

# Inline custom code is also supported in create-code-evaluator-version
ax evaluators create-code-evaluator-version <evaluator-id> \
  --commit-message "v2" \
  --code-type custom \
  --code-name my_eval \
  --code 'class MyEval:
      def evaluate(self, output):
          return 1 if "yes" in output else 0' \
  --variables '["output"]'

# List experiments (optionally filtered by dataset)
ax experiments list [--dataset <dataset>] [--limit 15] [--cursor <cursor>]

# Get a specific experiment
ax experiments get <experiment>

# Export all runs from an experiment
ax experiments export <experiment> [--output-dir .] [--stdout]

# Run an experiment — execute a task function against a dataset and upload results
ax experiments run --dataset <dataset> --name "My Experiment" --task task.py

# Dry-run to validate locally without uploading (processes first 10 examples)
ax experiments run --dataset <dataset> --name "My Experiment" --task task.py --dry-run

# Create a new experiment from a pre-computed data file
ax experiments create --name "My Experiment" --dataset <dataset> --file runs.csv

# Create an experiment from stdin
ax experiments create --name "My Experiment" --dataset <dataset> --file -

# List runs for an experiment (paginated)
ax experiments list-runs <experiment> [--limit 30] [--cursor <cursor>]

# Delete an experiment
ax experiments delete <experiment> [--force]

# List organizations
ax organizations list [--name <substring>] [--limit 50] [--cursor <cursor>]

# Get an organization by name or ID
ax organizations get <organization>

# Create a new organization
ax organizations create --name "My Org" [--description "..."]

# Update an organization's name or description (at least one field required)
ax organizations update <organization> --name "New Name"
ax organizations update <organization> --description "Updated description"

# Add a user to an organization (or update their role if already a member)
ax organizations add-user <organization> --user-id <user-id> --role <role>

# Remove a user from an organization (also removes them from all child spaces)
ax organizations remove-user <organization> --user-id <user-id> [--force]

# Delete an organization (irreversible)
ax organizations delete <organization> [--force]

# List projects
ax projects list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get project metadata
ax projects get <project>

# Create a new project
ax projects create --name "My Project" --space <space>

# Update a project (by ID or name)
ax projects update <project> --name "Updated Name" [--space <space>]

# Delete a project
ax projects delete <project> [--force]

# List prompts
ax prompts list [--name <substring>] [--space <space>] [--limit 15] [--cursor <cursor>]

# Get a prompt (latest version by default)
ax prompts get <prompt>

# Get a specific version by ID or label
ax prompts get <prompt> --version-id <version-id>
ax prompts get <prompt> --label production

# Create a prompt with an initial version
ax prompts create \
  --name "My Prompt" \
  --space <space> \
  --provider OPEN_AI \
  --input-variable-format F_STRING \
  --messages messages.json \
  --commit-message "Initial version"

# Create a prompt with invocation and provider parameters
ax prompts create \
  --name "My Prompt" \
  --space <space> \
  --provider OPEN_AI \
  --model gpt-4o \
  --input-variable-format F_STRING \
  --messages messages.json \
  --commit-message "Initial version" \
  --invocation-params '{"temperature": 0.7, "max_tokens": 512}' \
  --provider-params '{"deployment": "my-deployment"}'

# Update a prompt's description
ax prompts update <prompt> --description "Updated description"

# Delete a prompt (removes all versions)
ax prompts delete <prompt> [--force]

# List versions for a prompt
ax prompts list-versions <prompt> [--limit 15] [--cursor <cursor>]

# Create a new version
ax prompts create-version <prompt> \
  --provider OPEN_AI \
  --input-variable-format F_STRING \
  --messages messages_v2.json \
  --commit-message "Improved system prompt"

# Create a new version (inline messages JSON)
ax prompts create-version <prompt> \
  --provider OPEN_AI \
  --input-variable-format F_STRING \
  --messages '[{"role": "USER", "content": "Your prompt here"}]' \
  --commit-message "Minimal inline JSON example"

# Create a new version with invocation and provider parameters
ax prompts create-version <prompt> \
  --provider OPEN_AI \
  --model gpt-4o \
  --input-variable-format F_STRING \
  --messages messages_v2.json \
  --commit-message "Tuned parameters" \
  --invocation-params '{"temperature": 0.5}' \
  --provider-params '{"deployment": "my-deployment"}'


# Resolve a label to its version
ax prompts get-version-by-label <prompt> --label production

# Set labels on a version (replaces all existing labels)
ax prompts set-version-labels <version-id> --label production --label staging

# Remove a label from a version
ax prompts remove-version-label <version-id> --label staging

# List all roles (predefined and custom)
ax roles list [--limit 15] [--cursor <cursor>]

# List only custom roles
ax roles list --is-custom

# List only system-defined predefined roles
ax roles list --is-predefined

# Get a role by name or ID
ax roles get <role-name-or-id>

# Create a custom role (at least one permission required)
ax roles create --name "Data Analyst" --permissions DATASET_READ,PROJECT_READ

# Create with description
ax roles create --name "Data Analyst" --permissions DATASET_READ --description "Read-only data access"

# Update a role (at least one field required; --permissions fully replaces existing permissions)
ax roles update <role-name-or-id> --name "Senior Analyst"
ax roles update <role-name-or-id> --permissions DATASET_READ,DATASET_CREATE

# Delete a role by name or ID (pass --force to skip confirmation)
ax roles delete <role-name-or-id> [--force]

# List spaces
ax spaces list [--organization-id <org-id>] [--limit 15] [--cursor <cursor>]

# Get a space by name or ID
ax spaces get <space>

# Create a new space
ax spaces create --name "My Space" --organization-id <org-id> [--description "..."]

# Update a space's name or description (at least one field required)
ax spaces update <space> --name "New Name"
ax spaces update <space> --description "Updated description"

# Delete a space and all resources within it
ax spaces delete <space> [--force]

# Add a user to a space (or update their role if already a member)
# The user must already be a member of the space's parent organization.
ax spaces add-user <space> --user-id <user-id> --role <role>

# Remove a user from a space
ax spaces remove-user <space> --user-id <user-id> [--force]

# Interactive install (detects installed agents, prompts for selection)
ax skills install

# Install for a specific agent, non-interactively
ax skills install --agent claude-code --yes

# Install for multiple agents
ax skills install --agent claude-code --agent cursor --yes

# Install globally (~/.claude/skills/, ~/.cursor/skills/, ~/.codex/skills/, ~/.windsurf/skills/)
ax skills install --global

# Overwrite existing skills
ax skills install --agent claude-code --force --yes

# Remove installed skills (checks all known agents)
ax skills clear
ax skills clear --yes

# Remove for a specific agent only
ax skills clear --agent claude-code

# Export all spans (writes to file by default)
ax spans export <project-id>

# Export with filter
ax spans export <project-id> --filter "status_code = 'ERROR'"

# Export by trace, span, or session ID
ax spans export <project-id> --trace-id <trace-id>
ax spans export <project-id> --span-id <span-id>
ax spans export <project-id> --session-id <session-id>

# Export to stdout
ax spans export <project-id> --stdout

ax spans export <project-id> --filter "status_code = 'ERROR'"
ax spans export <project-id> --filter "latency_ms > 1000"
ax spans export <project-id> --trace-id abc123 --filter "latency_ms > 1000"
ax spans export <project-id> --start-time 2024-01-01T00:00:00Z --end-time 2024-01-02T00:00:00Z

# Delete one or more spans by ID (prompts for confirmation)
ax spans delete <project-id> --span-id <span-id> [--span-id <span-id> ...] [--space <space>]

# Delete without confirmation
ax spans delete <project-id> --span-id <span-id> --force

# Comma-separated IDs are also accepted
ax spans delete <project-id> --span-id id1,id2,id3 --force

# List tasks (filtered by space, project, dataset, or type)
ax tasks list [--name <substring>] [--space <space>] [--project-id <project-name-or-id>] \
  [--dataset-id <dataset-id>] \
  [--task-type TEMPLATE_EVALUATION|CODE_EVALUATION|RUN_EXPERIMENT] \
  [--limit 15] [--cursor <cursor>]

# Get a specific task
ax tasks get <task-id>

# ── Evaluation tasks (TEMPLATE_EVALUATION / CODE_EVALUATION) ─────────────────

# Create a project-based evaluation task (use ax evaluators list to find IDs)
ax tasks create \
  --name "Relevance Check" \
  --task-type TEMPLATE_EVALUATION \
  --evaluators '[{"evaluator_id": "<id from ax evaluators list>", "query_filter": null, "column_mappings": null}]' \
  --project <project> [--space <space>] \
  --is-continuous

# Create a dataset-based evaluation task
ax tasks create \
  --name "Dataset Eval" \
  --task-type TEMPLATE_EVALUATION \
  --evaluators '[{"evaluator_id": "<evaluator-id>"}]' \
  --dataset <dataset> \
  --experiment-ids <exp-id-1>,<exp-id-2>

# Narrower subcommand (same options, cleaner help text)
ax tasks create-evaluation \
  --name "Relevance Check" \
  --task-type TEMPLATE_EVALUATION \
  --evaluators '[{"evaluator_id": "<id>"}]' \
  --project <project>

# ── Run-experiment tasks ──────────────────────────────────────────────────────

# Create a run-experiment task (dispatched via ax tasks create)
ax tasks create \
  --name "LLM Eval Run" \
  --task-type RUN_EXPERIMENT \
  --dataset <dataset> \
  --run-configuration '{
    "experiment_type": "LLM_GENERATION",
    "ai_integration_id": "<integration-id>",
    "model_name": "gpt-4o",
    "input_variable_format": "MUSTACHE",
    "messages": [{"role": "USER", "content": "{{input}}"}]
  }'

# Or use the dedicated subcommand
ax tasks create-run-experiment \
  --name "LLM Eval Run" \
  --dataset <dataset> \
  --run-configuration @run_config.json [--space <space>]

# ── Common task management ────────────────────────────────────────────────────

# Update mutable fields on an evaluation task (provide at least one field)
ax tasks update <task> --name "New Name"
ax tasks update <task> --sampling-rate 0.25
ax tasks update <task> --is-continuous
ax tasks update <task> --query-filter "attributes.env = 'prod'"
ax tasks update <task> --query-filter ""    # clear the task-level query filter
ax tasks update <task> --evaluators '[{"evaluator_id": "<id>"}]'

# Update the run configuration on a run_experiment task
ax tasks update <task> --run-configuration @new_config.json

# Delete a task (irreversible; prompts unless --force is passed)
ax tasks delete <task> [--space <space>] [--force]

# ── Triggering runs ───────────────────────────────────────────────────────────

# Trigger an on-demand evaluation run
ax tasks trigger-run <task-id>

# Trigger a run and wait for it to complete
ax tasks trigger-run <task-id> --wait

# Trigger an evaluation run over a specific data window
ax tasks trigger-run <task-id> \
  --data-start-time 2024-01-01T00:00:00Z \
  --data-end-time 2024-01-02T00:00:00Z \
  --max-spans 5000

# Trigger a run-experiment run (SDK dispatches based on task type)
ax tasks trigger-run <task-id> \
  --experiment-name "My Experiment" \
  --dataset-version-id <version-id> \
  --max-examples 200 \
  --tracing-metadata '{"env": "prod"}'

# Run against specific example IDs instead of all/N examples
ax tasks trigger-run <task-id> \
  --experiment-name "My Experiment" \
  --example-ids <example-id-1>,<example-id-2>

# Chain evaluation tasks to run after the experiment completes
ax tasks trigger-run <task-id> \
  --experiment-name "My Experiment" \
  --evaluation-task-ids <eval-task-id-1>,<eval-task-id-2>

# ── Run management ────────────────────────────────────────────────────────────

# List runs for a task (optionally filtered by status)
ax tasks list-runs <task-id> [--status PENDING|RUNNING|COMPLETED|FAILED|CANCELLED] \
  [--limit 15] [--cursor <cursor>]

# Get a specific run
ax tasks get-run <run-id>

# Cancel a run (only valid when pending or running)
ax tasks cancel-run <run-id> [--force]

# Wait for a run to reach a terminal state
ax tasks wait-for-run <run-id> [--poll-interval 5] [--timeout 600]

# List traces
ax traces list <project-id> [--start-time <iso8601>] [--end-time <iso8601>] \
  [--filter "<expr>"] [--limit 15] [--cursor <cursor>] [--output <format>]

ax traces list <project-id> --filter "status_code = 'ERROR'"
ax traces list <project-id> --start-time 2024-01-01T00:00:00Z
ax traces list <project-id> --filter "latency_ms > 5000" --limit 50

# List users (optionally filtered by email or status)
ax users list [--email <substring>] [--status ACTIVE] [--status INVITED] \
  [--limit 50] [--cursor <cursor>]

# Get a specific user by ID
ax users get <user-id>

# Create a new user and invite them
ax users create --full-name "Jane Doe" --email jane@example.com \
  --role MEMBER --invite-mode EMAIL_LINK

# Update a user's full name or developer permission flag
ax users update <user-id> --full-name "Jane Smith"
ax users update <user-id> --is-developer
ax users update <user-id> --no-is-developer

# Delete one or more users by ID (comma-separated or repeated flag)
ax users delete --id <user-id> [--force]
ax users delete --id id1,id2,id3 [--force]

# Delete by email address (resolves to user ID before deletion)
ax users delete --email user@example.com [--force]

# Mix IDs and emails; each deletion is attempted independently
ax users delete --id id1 --email user@example.com [--force]

# Resend an invitation email to a pending user
ax users resend-invitation <user-id>

# Send a password-reset email to a user
ax users reset-password <user-id>

ax datasets create \
  --name "Customer Churn Dataset" \
  --space sp_abc123 \
  --file ./data/churn.csv

# Pipe from a file
cat data.json | ax datasets create \
  --name "customer-support-evals" \
  --space "U3BhY2U6OTA1MDoxSmtS" \
  --file -

# Inline heredoc — useful for scripting or quick one-offs
ax datasets create \
  --name "customer-support-evals" \
  --space "U3BhY2U6OTA1MDoxSmtS" \
  --file - <<'EOF'
[
  {"question": "How do I reset my password?", "ideal_answer": "Go to the login page and click 'Forgot Password'. Enter your email address and we'll send you a reset link within a few minutes.", "category": "Account Management"},
  ...
]
EOF

ax datasets list --space sp_abc123 --output json > datasets.json

# Export to a timestamped directory
ax datasets export ds_xyz789

# Export a specific version
ax datasets export ds_xyz789 --version-id ver_abc123

# Pipe to jq for processing
ax datasets export ds_xyz789 --stdout | jq '.[].input'

# Validate locally first (first 10 examples, no upload)
ax experiments run \
  --dataset ds_abc123 \
  --name "GPT-4o-mini eval" \
  --task task.py \
  --dry-run

# Run against the full dataset and upload results
ax experiments run \
  --dataset ds_abc123 \
  --name "GPT-4o-mini eval" \
  --task task.py \
  --concurrency 5

# Export all runs to a timestamped directory
ax experiments export exp_abc123

# Pipe to stdout for processing
ax experiments export exp_abc123 --stdout | jq '.[] | select(.output != null)'

# Export all spans in a trace
ax spans export proj_abc123 --trace-id tr_xyz789

# Export a session's spans to stdout
ax spans export proj_abc123 --session-id sess_456 --stdout

# Export with a custom lookback window
ax spans export proj_abc123 --trace-id tr_xyz789 --days 7

# Export all spans from a project
ax spans export proj_abc123

# Export error spans
ax spans export proj_abc123 --filter "status_code = 'ERROR'" --limit 100

# Export spans in a time window to stdout
ax spans export proj_abc123 --start-time 2024-01-01T00:00:00Z --end-time 2024-01-02T00:00:00Z --stdout

# List root traces in a project
ax traces list proj_abc123

# Export slow traces to Parquet for analysis
ax traces list proj_abc123 --filter "latency_ms > 2000" --limit 500 --output traces_slow.parquet

# List traces in JSON format
ax traces list proj_abc123 --output json

# First page
ax datasets list --space sp_abc123 --limit 20

# Next page (use cursor from previous response)
ax datasets list --space sp_abc123 --limit 20 --cursor <cursor-value>

# Setup profiles for different environments
ax profiles create  # Create "production" profile
ax profiles create  # Create "staging" profile

# Switch contexts
ax profiles use production
ax datasets list --space sp_prod123

ax profiles use staging
ax datasets list --space sp_stage456

ax spans export <project-id> --filter "status_code = 'ERROR'" --stdout

ax traces list <project-id> \
  --start-time 2024-01-01T00:00:00Z \
  --end-time 2024-01-02T00:00:00Z

ax profiles create  # Select output format during setup

ax datasets list --output json
ax datasets list --output datasets.csv
ax datasets list --output datasets.parquet

#!/bin/bash

# Export datasets to JSON
DATASETS=$(ax datasets list --space sp_abc123 --output json)

# Process with jq
echo "$DATASETS" | jq '.data[] | select(.name | contains("test"))'

# Export to file
ax datasets export ds_xyz789

ax datasets list --space sp_abc123 --verbose

ax --help
ax datasets --help
ax datasets create --help
ax profiles --help