litlm

import os
from litlm import complete, cost_breakdown

os.environ["OPENROUTER_API_KEY"] = "sk-or-..." 
os.environ["NVIDIA_NIM_API_KEY"] = "nvapi-..."  # Optional, used for the first NIM fallback
os.environ["NVIDIA_NIM_API_BASE"] = "https://integrate.api.nvidia.com/v1/"  # Optional

# 1. Simple String (Synchronous feel, but Async under the hood)
res = complete("What is 2+2?")
print(res) 
# > "4"

# 2. Batch Processing (Parallel execution + Progress Bar)
questions = ["Meaning of life?", "Capital of France?", "Who is TURING?"]
answers = complete(questions)
# > [Completing] 100%|██████████| 3/3 [00:01<00:00, cost=$0.000123, ⚠ 1/3 (33.3%)]

res = complete("Write a haiku")

print(res)              # It prints the content directly
print(res.usage)        # Access token usage
print(res.model_used)   # See which fallback route actually answered
print(res.cost)         # LiteLLM/OpenRouter cost when available
print(res.call_id)      # Unique ID for the call
print(res.reasoning)    # Access reasoning content (e.g. for DeepSeek-R1 / o1)

# Automatically finds 'openrouter/openai/gpt-4o-mini'
complete("Hello", model="gpt-4o-mini") 

# Automatically finds 'openrouter/anthropic/claude-3.5-sonnet'
complete("Hello", model="sonnet")

# Also works when you already know the provider/model slug
complete("Hello", model="meta-llama/llama-3.3-70b-instruct")

# Prefer "latest" variants when fuzzy matching, e.g. "haiku" -> haiku-latest
complete("Hello", model="haiku")

# System prompt shortcut
complete("Summarize this", system="Be concise.")

# JSON mode returns parsed JSON
data = complete("Return {'topic': string} as JSON", json=True)

# Lightweight in-memory spend summaries
session_costs = cost_breakdown("session")  # all calls since importing litlm
print(f"Session cost: ${sum(session_costs.values()):.6f}")
cost_breakdown("day")          # cost by model over the last day
cost_breakdown("week", by="day")

import pandas as pd

df = pd.DataFrame({"prompts": ["Joke about cats", "Joke about dogs"]})

# Returns a list of Text objects, keeping order
df["results"] = complete(df["prompts"]) 

# OpenRouter prompt caching: top-level cache_control passthrough
res = complete("Question over a long stable context...", prompt_cache=True)

# 1-hour TTL for providers that support it
res = complete("Question over a long stable context...", prompt_cache="1h")

# Full passthrough when you want exact control
res = complete("Question...", cache_control={"type": "ephemeral", "ttl": "1h"})

# Old LiteLLM response cache remains available, but is off by default.
# If you run this again, it returns instantly without API cost.
res = complete("Complex query...", caching=True)

# LiteLLM retries failed requests; timeout applies to each attempt.
res = complete("Flaky API...", num_retries=5, timeout=60)

from litlm import get_history

# Get the last result
last_res = get_history()

# Get a specific result by index
first_res = get_history(0)

# A failed batch item is an empty string-compatible result with metadata;
# other items continue and the progress bar displays the failure rate.
from litlm import get_failures
failures = get_failures()       # all failures since import
print(failures[-1].error)       # original LiteLLM/provider exception
print(failures[-1].prompt)      # input that failed

# Restrict inspection to the batch containing a particular result.
batch_failures = get_failures(failures[-1].call_id)

complete(
    "Hello", 
    model="deepseek/deepseek-chat", 
    temperature=0.7, 
    max_tokens=500,
    api_key="sk-..." # Optional if env var is set
)