[build-system]

requires = ["setuptools >= 77.0.3"]

build-backend = "setuptools.build_meta"

[project]

name = "trl"

description = "Train transformer language models with reinforcement learning."

authors = [

{ name = "Leandro von Werra", email = "leandro.vonwerra@gmail.com" }

]

readme = { file = "README.md", content-type = "text/markdown" }

license = "Apache-2.0"

license-files = ["LICENSE"]

keywords = [

"transformers", "huggingface", "language modeling", "post-training", "rlhf", "sft", "dpo", "grpo"

]

classifiers = [

"Development Status :: 2 - Pre-Alpha",

"Intended Audience :: Developers",

"Intended Audience :: Science/Research",

"Natural Language :: English",

"Operating System :: OS Independent",

"Programming Language :: Python :: 3",

"Programming Language :: Python :: 3.11",

"Programming Language :: Python :: 3.12",

"Programming Language :: Python :: 3.13",

"Programming Language :: Python :: 3.14"

]

requires-python = ">=3.11"

dependencies = [

"accelerate>=1.4.0",

"datasets>=4.7.0", # Support Json type and on_mixed_types="use_json"

"jinja2",

"packaging>20.0",

"transformers>=4.56.2",

]

dynamic = ["version"]

[project.urls]

Homepage = "https://github.com/huggingface/trl"

[project.scripts]

trl = "trl.cli:main"

[project.optional-dependencies]

deepspeed = [

"deepspeed>=0.18.6",

"transformers!=5.1.0", # see transformers#43780

]

harbor = [

"harbor>=0.13.0; python_version >= '3.12'", # harbor requires Python 3.12+ (pulls its sandbox backends)

]

kernels = [ # transformers renamed the "hub-kernels" extra to "kernels" in 5.1.0

"transformers[kernels]", # transformers >= 5.1.0

"transformers[hub-kernels]", # transformers < 5.1.0

]

liger = [

"liger-kernel>=0.8.2"

]

math_verify = [

"math-verify>=0.5.2",

]

openreward = [

"openreward>=0.1.109",

]

peft = [

"peft>=0.13.0"

]

quality = [

"pre-commit",

"hf-doc-builder"

]

quantization = [

"bitsandbytes"

]

test = [

"pytest-cov",

"pytest-datadir>=1.7.0", # lazy datadirs

"pytest-rerunfailures>=16.7", # only_rerun matches wrapped exceptions

"pytest-xdist",

"pytest",

]

vllm = [

"vllm>=0.21.0,<=0.31.0",

"aiohttp>=3.13.3",

"requests",

]

vlm = [

"Pillow",

"torchvision",

"num2words==0.5.14"

]

dev = [

# deepspeed

"deepspeed>=0.18.6",

# kernels: transformers renamed the "hub-kernels" extra to "kernels" in 5.1.0

"transformers[kernels]", # transformers >= 5.1.0

"transformers[hub-kernels]", # transformers < 5.1.0

# liger

"liger-kernel>=0.8.2",

# openreward

"openreward>=0.1.109",

# peft

"peft>=0.13.0",

# quality

"pre-commit",

"hf-doc-builder",

# quantization

"bitsandbytes",

# test

"pytest-cov",

"pytest-datadir>=1.7.0", # lazy datadirs

"pytest-rerunfailures>=16.7", # only_rerun matches wrapped exceptions

"pytest-xdist",

"pytest",

# vllm: not included in dev by default due to CUDA error; see GH-4228

# vlm

"Pillow",

"torchvision",

"num2words==0.5.14",

# for response parsing (required for training with tools)

"jmespath",

]

[tool.setuptools]

package-dir = {"trl" = "trl"}

[tool.setuptools.dynamic]

version = { file = "VERSION" }

[tool.coverage.run]

branch = true

[tool.ruff]

target-version = "py311"

line-length = 119

src = ["trl"]

[tool.ruff.lint]

ignore = [

"B028", # warning without explicit stacklevel

"C408", # dict() calls (stylistic)

"C901", # function complexity

"E501",

]

# `select`, not `extend-select`: the rule set is explicit, so a ruff release that expands its default

# selection (0.16 raised the defaults from 59 to 413 rules) does not silently enable new rules.

select = ["E", "F", "I", "W", "UP", "B", "T", "C"]

[tool.ruff.lint.per-file-ignores]

# Allow prints in auxiliary scripts

"examples/**.py" = ["T201"]

"scripts/**.py" = ["T201"]

"trl/cli/**.py" = ["T201"]

# Ignore import violations in all `__init__.py` files.

"__init__.py" = ["F401"]

[tool.ruff.lint.isort]

lines-after-imports = 2

known-first-party = ["trl"]

[tool.pytest.ini_options]

markers = [

"invariant: training-invariant checks (run with '-m invariant')",

]

norecursedirs = [

"tests/experimental",

"tests/invariant",

]

filterwarnings = [

# SWIG deprecations from SWIG-generated C/C++ extensions: sentencepiece

# Upstream issue: https://github.com/google/sentencepiece/issues/1150

# Remove once: no supported sentencepiece version emits it

"ignore:builtin type SwigPyPacked has no __module__ attribute:DeprecationWarning",

"ignore:builtin type SwigPyObject has no __module__ attribute:DeprecationWarning",

"ignore:builtin type swigvarlink has no __module__ attribute:DeprecationWarning",

# PyTorch DataLoader pin_memory device argument deprecations

# Triggered internally by torch.utils.data, not by our code

# Upstream issue: https://github.com/pytorch/pytorch/issues/174546

# Remove once: no supported torch version emits it

"ignore:The argument 'device' of Tensor.pin_memory:DeprecationWarning",

"ignore:The argument 'device' of Tensor.is_pinned:DeprecationWarning",

# bitsandbytes calls the deprecated torch._check_is_size in its CUDA quant ops (upstream, not actionable in TRL)

# Triggered indirectly by loading bitsandbytes-quantized models in the PEFT + quantization tests

# Tracking issue: https://github.com/huggingface/trl/issues/6447

# Upstream fix released in bitsandbytes 0.50.0: https://github.com/bitsandbytes-foundation/bitsandbytes/pull/1940

# Remove once: bitsandbytes >= 0.50.0 is required

"ignore:_check_is_size will be removed in a future PyTorch release:FutureWarning",

# triton builds an ast.AnnAssign node without the required `simple` field during kernel compilation (upstream,

# not actionable in TRL); a DeprecationWarning on Python 3.14 that becomes an error on 3.15

# Tracking issue: https://github.com/huggingface/trl/issues/6466

# Upstream issue: https://github.com/triton-lang/triton/issues/10981

# Upstream fix (merged, unreleased as of triton 3.7.1): https://github.com/triton-lang/triton/pull/10986

# Remove once: no supported torch resolves a triton without the fix

"ignore:AnnAssign.__init__ missing 1 required positional argument:DeprecationWarning",

# kernels-community/mamba-ssm (a Hub-hosted build of state-spaces/mamba) calls the deprecated

# torch.get_autocast_gpu_dtype() in its Triton ssd_combined.py kernel (upstream, not actionable in TRL)

# Triggered by the NemotronH (Nemotron 3) Mamba2 mixer fast path during training under CUDA autocast, only with

# transformers 5.3 to 5.5, which load the Hub kernel ignoring `use_mamba_kernels=False` (fixed upstream in 5.6)

# Tracking issue: https://github.com/huggingface/trl/issues/6555

# Upstream fix: https://github.com/huggingface/transformers/pull/44853

# Remove once: transformers >= 5.6 is required (kernels-community#1187 won't reach the v1 build 5.3-5.5 pin)

"ignore:torch.get_autocast_gpu_dtype\\(\\) is deprecated:DeprecationWarning",

# bitsandbytes warns, on every forward of a 4-bit layer whose inner dimension is not a multiple of the quantization

# blocksize, that it falls back to the slower dequantize + linear path (upstream, not actionable in TRL)

# Triggered by the tiny models in the PEFT + quantization tests: their hidden_size (8) and intermediate_size (32)

# are not multiples of the 64 blocksize, and transformers exposes no option to change it

# Tracking issue: https://github.com/huggingface/trl/issues/6580

# Upstream issue: https://github.com/bitsandbytes-foundation/bitsandbytes/issues/2027

# Remove once: no supported bitsandbytes version emits it

"ignore:inner dimension \\(\\d+\\) is not aligned for fast kernel:UserWarning",

# transformers passes the `num_patches` torch tensor to np.cumsum in the InternVL processor, so numpy falls back to

# the legacy Tensor.__array_wrap__ path, whose signature predates NumPy 2.0 (upstream, not actionable in TRL)

# Triggered by the InternVL cases of the GRPO, RLOO and Distillation VLM tests

# Tracking issue: https://github.com/huggingface/trl/issues/6848

# Upstream issue: https://github.com/huggingface/transformers/issues/48177

# Upstream issue (torch, for the outdated signature): https://github.com/pytorch/pytorch/issues/180657

# Remove once: no supported transformers version passes a tensor to np.cumsum there

"ignore:__array_wrap__ must accept context and return_scalar arguments:DeprecationWarning",

# torch inductor advises enabling TF32 on every compile when the float32 matmul precision is left at its default

# (upstream, not actionable in TRL: the precision is a process-wide numerics setting only the user should change)

# Triggered by the DPO and KTO Liger tests, the only ones that compile, since the GPU CI moved from T4 to L40S:

# the advisory is guarded on compute capability >= (8, 0), unreachable on T4

# Tracking issue: https://github.com/huggingface/trl/issues/6849

# Upstream issue: https://github.com/pytorch/pytorch/issues/175484

# Remove once: no supported torch version emits it

"ignore:TensorFloat32 tensor cores for float32 matrix multiplication available but not enabled:UserWarning",

# peft warns on every generate step of prompt learning models (prompt tuning, P-tuning) that it ignores

# position_ids, which transformers' generate builds itself (upstream, not actionable in TRL)

# Triggered by the GRPO prompt tuning test, the only prompt tuning test that generates

# Tracking issue: https://github.com/huggingface/trl/issues/7585

# Upstream issue: https://github.com/huggingface/peft/issues/3911

# Remove once: no supported peft version emits it

"ignore:Position ids are not supported for parameter efficient tuning:UserWarning",

]