CLI Reference

pip install osmosis-ai

osmosis
osmosis-ai
osmosis_ai

osmosis [command] [options]

osmosis login

osmosis logout

osmosis whoami

Logged in as: [email protected]
Workspace: My Workspace (ws_abc123)

osmosis workspace [subcommand]

# List all workspaces
osmosis workspace list

# Show current workspace
osmosis workspace current

# Switch workspace
osmosis workspace switch ws_abc123

osmosis preview --path <file_path>

osmosis preview --path rubric_configs.yaml

osmosis preview --path sample_data.jsonl

osmosis eval --rubric <rubric_id> --data <data_path> [options]

osmosis eval --rubric helpfulness --data responses.jsonl

osmosis eval --rubric helpfulness --data responses.jsonl --number 5

osmosis eval --rubric helpfulness --data responses.jsonl --output ./results/eval_001.json

osmosis eval --rubric helpfulness --data new_responses.jsonl --baseline ./results/baseline.json

osmosis eval --rubric helpfulness --data responses.jsonl --config ./configs/custom_rubrics.yaml

version: 1
default_score_min: 0.0
default_score_max: 1.0

rubrics:
  - id: rubric_identifier
    title: Human-Readable Title
    rubric: |
      Your evaluation criteria here.
      Can be multiple lines.
    model_info:
      provider: openai
      model: gpt-5
      api_key_env: OPENAI_API_KEY
      timeout: 30
    score_min: 0.0  # Optional override
    score_max: 1.0  # Optional override

{"solution_str": "The AI's response text to evaluate"}

{
  "conversation_id": "ticket-12345",
  "rubric_id": "helpfulness",
  "original_input": "How do I reset my password?",
  "solution_str": "Click 'Forgot Password' on the login page and follow the email instructions.",
  "ground_truth": "Users should use the password reset link sent to their registered email.",
  "metadata": {
    "customer_tier": "premium",
    "category": "account_management"
  },
  "score_min": 0.0,
  "score_max": 10.0
}

Evaluating records: 100%|████████████████| 50/50 [00:45<00:00, 1.1record/s]

Overall Statistics:
  Average Score: 0.847
  Min Score: 0.200
  Max Score: 1.000
  Variance: 0.034
  Std Deviation: 0.185
  Success Rate: 100.0% (50/50)

Evaluation Results:
  ...

Results saved to:
  ~/.cache/osmosis/eval_result/helpfulness/rubric_eval_result_20250114_143022.json

{
  "rubric_id": "helpfulness",
  "timestamp": "2025-01-14T14:30:22.123456",
  "duration_seconds": 45.2,
  "total_records": 50,
  "successful_evaluations": 50,
  "failed_evaluations": 0,
  "statistics": {
    "average": 0.847,
    "min": 0.200,
    "max": 1.000,
    "variance": 0.034,
    "std_dev": 0.185
  },
  "results": [
    {
      "conversation_id": "ticket-12345",
      "runs": [
        {
          "score": 0.85,
          "explanation": "The response directly addresses the question...",
          "raw": {...},
          "duration_ms": 890
        }
      ],
      "aggregate_stats": {
        "average": 0.85,
        "variance": 0.0
      }
    }
  ],
  "model_info": {
    "provider": "openai",
    "model": "gpt-5"
  }
}

model_info:
  provider: openai           # Or: anthropic, gemini, xai, openrouter, cerebras
  model: gpt-5        # Provider-specific model identifier
  api_key_env: OPENAI_API_KEY  # Environment variable name
  timeout: 30               # Optional timeout in seconds

# Create baseline
osmosis eval --rubric helpfulness --data baseline.jsonl --output baseline.json

# Compare new data against baseline
osmosis eval --rubric helpfulness --data new_data.jsonl --baseline baseline.json

osmosis eval --rubric helpfulness --data responses.jsonl --number 10

for file in data/*.jsonl; do
  osmosis eval --rubric helpfulness --data "$file" --output "results/$(basename $file .jsonl).json"
done

export OSMOSIS_CACHE_DIR=/path/to/custom/cache
osmosis eval --rubric helpfulness --data responses.jsonl

Error: API key not found for provider 'openai'

export OPENAI_API_KEY="your-key-here"

Error: Rubric 'helpfulness' not found in configuration

Error: Invalid JSON on line 5

Error: Model 'gpt-5' not available for provider 'openai'

Error: Request timed out after 30 seconds

model_info:
  timeout: 60

export OSMOSIS_DEBUG=1
osmosis eval --rubric helpfulness --data responses.jsonl

pip show osmosis-ai

osmosis preview --path rubric_configs.yaml

ls -lh ~/.cache/osmosis/eval_result/

osmosis serve -m <module:attribute> [options]

# Start server with validation (default port 9000)
osmosis serve -m server:agent_loop

# Specify port
osmosis serve -m server:agent_loop -p 8080

# Enable debug logging
osmosis serve -m server:agent_loop --log ./rollout_logs

# Enable auto-reload for development
osmosis serve -m server:agent_loop --reload

# Local debug mode (no auth)
osmosis serve -m server:agent_loop --local

osmosis validate -m <module:attribute> [options]

# Validate agent loop
osmosis validate -m server:agent_loop

# Verbose output
osmosis validate -m server:agent_loop -v

osmosis test -m <module:attribute> -d <dataset> [options]

# Batch test with GPT-4o (default)
osmosis test -m server:agent_loop -d multiply.parquet

# Use Claude
osmosis test -m server:agent_loop -d data.jsonl --model anthropic/claude-sonnet-4-20250514

# Test subset of data
osmosis test -m server:agent_loop -d data.jsonl --limit 10

# Save results
osmosis test -m server:agent_loop -d data.jsonl -o results.json

# Interactive debugging
osmosis test -m server:agent_loop -d data.jsonl --interactive