export type NativeEmbeddingParams = {

embd_normalize?: number

}

export type NativeSpeculativeType =

| 'none'

| 'draft-mtp'

/**

* Alias for draft-mtp.

*/

| 'mtp'

export type NativeSpeculativeParams = {

enabled?: boolean

type?: NativeSpeculativeType

types?: Array<NativeSpeculativeType>

n_max?: number

n_min?: number

p_min?: number

p_split?: number

draft?: {

/**

* Optional separate draft model path for MTP/speculative decoding.

* When omitted, MTP uses the loaded target model's embedded draft layers.

*/

model?: string

path?: string

model_draft?: string

draft_model?: string

n_max?: number

n_min?: number

p_min?: number

p_split?: number

n_gpu_layers?: number

cache_type_k?: string

cache_type_v?: string

}

}

export type NativeSpeculativeConfig =

| NativeSpeculativeParams

| NativeSpeculativeType

| boolean

export type NativeContextParams = {

model: string

/**

* Optional separate draft model path for MTP/speculative decoding.

* Leave unset for hybrid/embedded MTP models such as Qwen MTP.

*/

model_draft?: string

/**

* Alias for model_draft.

*/

draft_model?: string

is_model_draft_asset?: boolean

/**

* Chat template to override the default one from the model.

*/

chat_template?: string

is_model_asset?: boolean

use_progress_callback?: boolean

n_ctx?: number

n_batch?: number

n_ubatch?: number

/**

* Number of parallel sequences to support (sets n_seq_max).

* This determines the maximum number of parallel slots that can be used.

* Default: 8

*/

n_parallel?: number

n_threads?: number

/**

* CPU affinity mask string (e.g., "0-3" or "0,2,4,6").

* Specifies which CPU cores to use for inference.

*/

cpu_mask?: string

/**

* Use strict CPU placement.

* When true, enforces strict CPU core affinity.

* Default: false

*/

cpu_strict?: boolean

/**

* Number of layers to store in VRAM (Currently only for iOS)

*/

n_gpu_layers?: number

/**

* Backend devices to use. If omitted, llama.rn uses the platform default selection.

* On Android, Hexagon is opt-in: use `['HTP0']` for one session, select multiple

* HTP devices explicitly, or use `['HTP*']` for every available HTP session.

*/

devices?: Array<string>

/**

* Skip GPU devices (iOS only) (Deprecated: Please set devices params instead)

*/

no_gpu_devices?: boolean

/**

* Enable flash attention, only recommended in GPU device.

*/

flash_attn_type?: string

/**

* Enable flash attention, only recommended in GPU device

* Deprecated: use flash_attn_type instead

*/

flash_attn?: boolean

/**

* KV cache data type for the K (Experimental in llama.cpp)

*/

cache_type_k?: string

/**

* KV cache data type for the V (Experimental in llama.cpp)

*/

cache_type_v?: string

use_mlock?: boolean

use_mmap?: boolean

vocab_only?: boolean

/**

* Disable extra buffer types for weight repacking.

* Reduces memory usage at the cost of slower prompt processing.

* Default: false

*/

no_extra_bufts?: boolean

/**

* Single LoRA adapter path

*/

lora?: string

/**

* Single LoRA adapter scale

*/

lora_scaled?: number

/**

* LoRA adapter list

*/

lora_list?: Array<{ path: string; scaled?: number }>

rope_freq_base?: number

rope_freq_scale?: number

/**

* Enable speculative decoding support at context creation time.

* MTP on recurrent/hybrid models must be enabled here so llama.cpp can

* allocate recurrent-state rollback slots.

*/

speculative?: NativeSpeculativeConfig

spec_type?: NativeSpeculativeType | Array<NativeSpeculativeType>

spec_draft_n_max?: number

spec_draft_n_min?: number

spec_draft_p_min?: number

spec_draft_p_split?: number

spec_draft_n_gpu_layers?: number

spec_draft_cache_type_k?: string

spec_draft_cache_type_v?: string

pooling_type?: number

/**

* Enable context shifting to handle prompts larger than context size

*/

ctx_shift?: boolean

/**

* Use a unified buffer across the input sequences when computing the attention.

* Try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix.

*/

kv_unified?: boolean

/**

* Use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)

*/

swa_full?: boolean

/**

* Number of layers to keep MoE weights on CPU

*/

n_cpu_moe?: number

/**

* Memory budget (MiB) for the cross-turn KV prefix cache on recurrent/hybrid

* models. 0 disables it; no-op on pure-attention models. Default 160.

*/

state_cache_budget_mb?: number

/**

* Max snapshots to keep (secondary cap; the byte budget is primary).

* 0 = no count cap. Default 8.

*/

state_cache_max_checkpoints?: number

// Embedding params

embedding?: boolean

embd_normalize?: number

}

export type NativeCompletionParams = {

prompt: string

n_threads?: number

/**

* Enable Jinja. Default: true if supported by the model

*/

jinja?: boolean

/**

* JSON schema for convert to grammar for structured JSON output.

* It will be override by grammar if both are set.

*/

json_schema?: string

/**

* Set grammar for grammar-based sampling. Default: no grammar

*/

grammar?: string

/**

* Lazy grammar sampling, trigger by grammar_triggers. Default: false

*/

grammar_lazy?: boolean

/**

* Enable thinking if jinja is enabled. Default: true

*/

enable_thinking?: boolean

/**

* Force thinking to be open. Default: false

*/

thinking_forced_open?: boolean

/**

* Maximum number of tokens allowed inside a thinking block before forcing it to close.

* Only applies when chat formatting exposes thinking tags.

*/

thinking_budget_tokens?: number

/**

* Message injected before the thinking end tag when the thinking budget is exhausted.

*/

thinking_budget_message?: string

/**

* Assistant generation prompt returned by jinja chat formatting.

* Used for PEG chat parsing and grammar prefill.

*/

generation_prompt?: string

/**

* Serialized PEG parser for chat output parsing.

* Required for COMMON_CHAT_FORMAT_PEG_* formats.

* This is typically obtained from getFormattedChat with jinja enabled.

*/

chat_parser?: string

/**

* Lazy grammar triggers. Default: []

*/

grammar_triggers?: Array<{

type: number

value: string

token: number

}>

preserved_tokens?: Array<string>

chat_format?: number

reasoning_format?: 'none' | 'auto' | 'deepseek'

/**

* Path to an image file to process before generating text.

* When provided, the image will be processed and added to the context.

* Requires multimodal support to be enabled via initMultimodal.

*/

media_paths?: Array<string>

/**

* Specify a JSON array of stopping strings.

* These words will not be included in the completion, so make sure to add them to the prompt for the next iteration. Default: `[]`

*/

stop?: Array<string>

/**

* Set the maximum number of tokens to predict when generating text.

* **Note:** May exceed the set limit slightly if the last token is a partial multibyte character.

* When 0,no tokens will be generated but the prompt is evaluated into the cache. Default: `-1`, where `-1` is infinity.

*/

n_predict?: number

/**

* If greater than 0, the response also contains the probabilities of top N tokens for each generated token.

* By default these are the sampler chain's candidate probabilities, i.e. after `top_k` / `top_p` / `min_p` / `temperature` / `grammar` / ... have been applied and renormalised.

* Set `post_sampling_probs: false` to get a plain softmax of the raw logits instead.

* Default: `0`

*/

n_probs?: number

/**

* Controls what `n_probs` reports.

* `true`: probabilities from the sampler chain's candidates (post-sampling).

* `false`: softmax of the raw logits, ignoring every sampler setting. Use this for calibrated readouts / thresholds.

* Default: `true`

*/

post_sampling_probs?: boolean

/**

* Per-completion speculative decoding override. For MTP on recurrent/hybrid

* models, load the model with matching MTP options first.

*/

speculative?: NativeSpeculativeConfig

spec_type?: NativeSpeculativeType | Array<NativeSpeculativeType>

spec_draft_n_max?: number

spec_draft_n_min?: number

spec_draft_p_min?: number

spec_draft_p_split?: number

/**

* Limit the next token selection to the K most probable tokens. Default: `40`

*/

top_k?: number

/**

* Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P. Default: `0.95`

*/

top_p?: number

/**

* The minimum probability for a token to be considered, relative to the probability of the most likely token. Default: `0.05`

*/

min_p?: number

/**

* Set the chance for token removal via XTC sampler. Default: `0.0`, which is disabled.

*/

xtc_probability?: number

/**

* Set a minimum probability threshold for tokens to be removed via XTC sampler. Default: `0.1` (> `0.5` disables XTC)

*/

xtc_threshold?: number

/**

* Enable locally typical sampling with parameter p. Default: `1.0`, which is disabled.

*/

typical_p?: number

/**

* Adjust the randomness of the generated text. Default: `0.8`

*/

temperature?: number

/**

* Last n tokens to consider for penalizing repetition. Default: `64`, where `0` is disabled and `-1` is ctx-size.

*/

penalty_last_n?: number

/**

* Control the repetition of token sequences in the generated text. Default: `1.0`

*/

penalty_repeat?: number

/**

* Repeat alpha frequency penalty. Default: `0.0`, which is disabled.

*/

penalty_freq?: number

/**

* Repeat alpha presence penalty. Default: `0.0`, which is disabled.

*/

penalty_present?: number

/**

* Enable Mirostat sampling, controlling perplexity during text generation. Default: `0`, where `0` is disabled, `1` is Mirostat, and `2` is Mirostat 2.0.

*/

mirostat?: number

/**

* Set the Mirostat target entropy, parameter tau. Default: `5.0`

*/

mirostat_tau?: number

/**

* Set the Mirostat learning rate, parameter eta. Default: `0.1`

*/

mirostat_eta?: number

/**

* Set the DRY (Don't Repeat Yourself) repetition penalty multiplier. Default: `0.0`, which is disabled.

*/

dry_multiplier?: number

/**

* Set the DRY repetition penalty base value. Default: `1.75`

*/

dry_base?: number

/**

* Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2`

*/

dry_allowed_length?: number

/**

* How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size.

*/

dry_penalty_last_n?: number

/**

* Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']`

*/

dry_sequence_breakers?: Array<string>

/**

* Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641. Default: `-1.0` (Disabled)

*/

top_n_sigma?: number

/**

* Ignore end of stream token and continue generating. Default: `false`

*/

ignore_eos?: boolean

/**

* Modify the likelihood of a token appearing in the generated text completion.

* For example, use `"logit_bias": [[15043,1.0]]` to increase the likelihood of the token 'Hello', or `"logit_bias": [[15043,-1.0]]` to decrease its likelihood.

* Setting the value to false, `"logit_bias": [[15043,false]]` ensures that the token `Hello` is never produced. The tokens can also be represented as strings,

* e.g.`[["Hello, World!",-0.5]]` will reduce the likelihood of all the individual tokens that represent the string `Hello, World!`, just like the `presence_penalty` does.

* Default: `[]`

*/

logit_bias?: Array<Array<number>>

/**

* Set the random number generator (RNG) seed. Default: `-1`, which is a random seed.

*/

seed?: number

/**

* Output token embeddings during generation.

* When enabled, completion results include generated token embeddings and their dimension.

* Default: `false`

*/

embedding?: boolean

emit_partial_completion: boolean

}

/**

* Parameters for parallel completion requests (queueCompletion).

* Extends NativeCompletionParams with parallel-mode specific options.

*/

export type NativeParallelCompletionParams = NativeCompletionParams & {

/**

* File path to load state from before processing.

* This allows you to resume from a previously saved completion state.

* Use with `save_state_path` to enable conversation continuity across requests.

* Example: `'/path/to/state.bin'` or `'file:///path/to/state.bin'`

*/

load_state_path?: string

/**

* File path to save state to after completion.

* The state will be saved to this file path when the completion finishes.

* You can then pass this path to `load_state_path` in a subsequent request to resume.

* For multimodal conversations a `<path>.meta` sidecar file is written next

* to the state file (media identity); keep the two files together.

* Example: `'/path/to/state.bin'` or `'file:///path/to/state.bin'`

*/

save_state_path?: string

/**

* File path to save prompt-only state to after prompt processing.

* Useful for fast prompt reuse (especially for recurrent/hybrid models).

* Example: `'/path/to/prompt_state.bin'` or `'file:///path/to/prompt_state.bin'`

*/

save_prompt_state_path?: string

/**

* Number of tokens to load when loading state.

* If not specified or <= 0, all tokens from the state file will be loaded.

* Use this to limit how much of a saved state is restored.

* Example: `512` to load only the first 512 tokens from the state file

*/

load_state_size?: number

/**

* Number of tokens to save when saving state.

* If not specified or <= 0, all tokens will be saved.

* Use this to limit the size of saved state files.

* Example: `512` to save only the last 512 tokens

*/

save_state_size?: number

}

export type NativeCompletionTokenProbItem = {

tok_id: number

tok_str: string

prob: number

}

export type NativeCompletionTokenProb = {

content: string

probs: Array<NativeCompletionTokenProbItem>

}

export type NativeCompletionResultTimings = {

cache_n: number

prompt_n: number

prompt_ms: number

prompt_per_token_ms: number

prompt_per_second: number

predicted_n: number

predicted_ms: number

predicted_per_token_ms: number

predicted_per_second: number

}

export type NativeCompletionResult = {

/**

* Original text (Ignored reasoning_content / tool_calls)

*/

text: string

/**

* Reasoning content (parsed for reasoning model)

*/

reasoning_content: string

/**

* Tool calls

*/

tool_calls: Array<{

type: 'function'

function: {

name: string

arguments: string

}

id?: string

}>

/**

* Content text (Filtered text by reasoning_content / tool_calls)

*/

content: string

chat_format: number

tokens_predicted: number

tokens_evaluated: number

draft_tokens: number

draft_tokens_accepted: number

truncated: boolean

stopped_eos: boolean

stopped_word: string

stopped_limit: number

stopping_word: string

context_full: boolean

interrupted: boolean

tokens_cached: number

timings: NativeCompletionResultTimings

completion_probabilities?: Array<NativeCompletionTokenProb>

embeddings?: Array<number>

embedding_dim?: number

audio_tokens?: Array<number>

}

export type NativeTokenizeResult = {

tokens: Array<number>

/**

* Whether the tokenization contains media

*/

has_media: boolean

/**

* Bitmap hashes of the media

*/

bitmap_hashes: Array<number>

/**

* Chunk positions of the text and media

*/

chunk_pos: Array<number>

/**

* Chunk positions of the media

*/

chunk_pos_media: Array<number>

}

export type NativeEmbeddingResult = {

embedding: Array<number>

}

/*

* Typed decision models (System One).

*

* The request and the response follow the TypeSafe `/v1/systemone` API, the

* same contract llama-server serves: a model answers typed questions about a

* state in one forward pass, no token is generated.

*/

/**

* The readout a decision model declares in `<arch>.decision.type`.

* `system_one`: an older model that declares no type but carries a `system_one`

* template; its readout is derived from the model, see `readout`.

* `unknown`: the model is a decision model this build cannot serve, see `error`.

*/

export type DecisionModelType =

| 'openjev'

| 'lev'

| 'kev'

| 'nimble'

| 'laya'

| 'clef'

| 'pplx-decider'

| 'lfm2-d1'

| 'lfm2-d1-omni'

| 'system_one'

| 'unknown'

/**

* How a `system_one` model is read:

* `letter_slot`: a causal model, every question answered from one prompt.

* `rank_head`: a classification head scoring one prompt per option. The context

* must be initialized with `pooling_type: 'rank'` and `embedding: true`.

*/

export type DecisionReadout = 'letter_slot' | 'rank_head'

/** Any JSON value */

export type DecisionValue =

| string

| number

| boolean

| null

| readonly DecisionValue[]

| { [key: string]: DecisionValue }

/** A string, an object or an array. A value that is not a string is given to the model as JSON text. */

export type DecisionContent =

| string

| readonly DecisionValue[]

| { [key: string]: DecisionValue }

export type DecisionChoiceQuestion<K extends string = string> = {

type: 'choice'

instructions: DecisionContent

/**

* Maps each option to its description, in the order the options are shown.

* That is the order JavaScript enumerates the keys in: keys that are whole

* numbers (`'1'`, `'10'`) come first, in ascending order, wherever they were

* written. Give such options a prefix (`'n1'`) to keep them where they are.

*/

criteria: { [key in K]: string | null }

}

export type DecisionScoreQuestion = {

type: 'score'

instructions: DecisionContent

/** 2 to 10 level descriptions, lowest level first. */

criteria: readonly string[]

}

export type DecisionNoulQuestion = {

type: 'noul'

instructions: DecisionContent

/** `lfm2-d1-omni` also takes the descriptions under `yes` and `no` */

criteria?: {

true?: string | null

false?: string | null

yes?: string | null

no?: string | null

}

}

export type DecisionQuestion =

| DecisionChoiceQuestion

| DecisionScoreQuestion

| DecisionNoulQuestion

export type DecisionQuestions = Record<string, DecisionQuestion>

export type DecisionRequest<Q extends DecisionQuestions = DecisionQuestions> = {

/**

* The content to evaluate. A state made of chat messages (an array of

* messages, or an object with a `messages` array) may carry `image_url`

* and `input_audio` parts, they are taken as media and removed from the state.

* For a `system_one` model the state may also be a list of content parts

* (`{ type: 'text', text }`, `{ type: 'image_url', image_url: { url } }`):

* the text is joined and each image stays where it is.

* `lfm2-d1` and `lfm2-d1-omni` take `null`, to ask about the media only.

*/

state: DecisionContent | null

questions: Q

/**

* Images, or audio clips for a model with `audioInput`, as file paths or

* data URLs. Needs `initMultimodal()`. All media go before the state, these

* first. `lfm2-d1-omni` takes images or one audio clip, not both.

*/

files?: string[]

/** Alias of `files` */

images?: string[]

}

export type DecisionChoiceAnswer<K extends string = string> = {

type: 'choice'

/** The option with the highest probability. */

choice: K

/**

* Option to its probability, they sum to 1.

* Look an option up by its key: the key order is not guaranteed.

*/

probabilities: { [key in K]: number }

/** 0 when all options are equally likely, 1 when one option has all the mass. */

confidence: number

}

export type DecisionScoreAnswer = {

type: 'score'

/** The expected level index, weighted by probability. Can be between two levels. */

score: number

/** Level index to its description. */

legend: Record<string, string>

/** Level index to its probability, they sum to 1. */

probabilities: Record<string, number>

confidence: number

}

export type DecisionNoulAnswer = {

type: 'noul'

/** The probability that the answer is true. */

noul: number

}

export type DecisionAnswer =

| DecisionChoiceAnswer

| DecisionScoreAnswer

| DecisionNoulAnswer

/** The answer type of a question type, choice keys included. */

export type DecisionAnswerOf<Q extends DecisionQuestion> =

Q extends DecisionChoiceQuestion<infer K>

? DecisionChoiceAnswer<K>

: Q extends DecisionScoreQuestion

? DecisionScoreAnswer

: DecisionNoulAnswer

export type DecisionResult<Q extends DecisionQuestions = DecisionQuestions> = {

/** `general.name` of the model, or its file name if it has none */

model: string

answers: { [id in keyof Q]: DecisionAnswerOf<Q[id]> }

usage: {

/** Prompt tokens of all the questions */

input_tokens: number

/** Always 0 */

output_tokens: number

}

}

export type NativeDecisionResult = DecisionResult

/** What the native side hands to the `llamaQueueDecide` callback */

export type NativeQueuedDecisionResult = NativeDecisionResult | { error: string }

export type NativeLlamaContext = {

contextId: number

model: {

desc: string

size: number

nEmbd: number

nParams: number

is_recurrent: boolean

is_hybrid: boolean

chatTemplates: {

llamaChat: boolean // Chat template in llama-chat.cpp

jinja: {

// Chat template supported by jinja engine

default: boolean

defaultCaps: {

tools: boolean

toolCalls: boolean

systemRole: boolean

parallelToolCalls: boolean

}

toolUse: boolean

toolUseCaps?: {

tools: boolean

toolCalls: boolean

systemRole: boolean

parallelToolCalls: boolean

}

}

}

metadata: Object

isChatTemplateSupported: boolean // Deprecated

/**

* Set if the model is a typed decision model, see `LlamaContext.decide()`

*/

decision?: {

type: DecisionModelType

/** Most options a `choice` question can have */

nOptionsMax: number

/** The prompt has a place for images (multimodal still has to be initialized) */

imageInput: boolean

/** The prompt has a place for an audio clip (multimodal still has to be initialized) */

audioInput: boolean

/** If false, `completion()` rejects: the model only answers decisions (clef, for one) */

textGeneration: boolean

/** Only for `system_one` models */

readout?: DecisionReadout

/** Why the model cannot be used, when `type` is `unknown` */

error?: string

}

}

/**

* Loaded library name for Android

*/

androidLib?: string

/**

* Name of the GPU device used on Android/iOS (if available)

*/

devices?: Array<string>

gpu: boolean

reasonNoGPU: string

systemInfo: string

}

export type NativeSessionLoadResult = {

tokens_loaded: number

prompt: string

}

export type NativeLlamaMessagePart = {

type: 'text'

text: string

}

export type NativeLlamaChatMessage = {

role: string

content: string | Array<NativeLlamaMessagePart>

}

export type FormattedChatResult = {

type: 'jinja' | 'llama-chat'

prompt: string

has_media: boolean

media_paths?: Array<string>

}

export type JinjaFormattedChatResult = FormattedChatResult & {

chat_format?: number

grammar?: string

grammar_lazy?: boolean

grammar_triggers?: Array<{

type: number

value: string

token: number

}>

generation_prompt?: string

thinking_forced_open?: boolean

thinking_start_tag?: string

thinking_end_tag?: string

preserved_tokens?: Array<string>

additional_stops?: Array<string>

/**

* Serialized PEG parser for chat output parsing.

* Required for COMMON_CHAT_FORMAT_PEG_* formats.

*/

chat_parser?: string

}

export type NativeImageProcessingResult = {

success: boolean

prompt: string

error?: string

}

export type NativeRerankParams = {

normalize?: number

}

export type NativeRerankResult = {

score: number

index: number

}

// Raw bench result as produced by the native side (snake_case); the

// LlamaContext.bench wrapper maps it to BenchResult.

export type NativeBenchResult = {

n_kv_max: number

n_batch: number

n_ubatch: number

flash_attn: number

is_pp_shared: number

n_gpu_layers: number

n_threads: number

n_threads_batch: number

pp: number

tg: number

pl: number

n_kv: number

t_pp: number

speed_pp: number

t_tg: number

speed_tg: number

t: number

speed: number

}

export type NativeBackendDeviceInfo = {

backend: string

type: string

deviceName: string

maxMemorySize: number

metadata?: Record<string, any>

}

export type ParallelRequestStatus = {

request_id: number

type: 'completion' | 'embedding' | 'rerank' | 'decision'

state: 'queued' | 'processing_prompt' | 'generating' | 'done'

prompt_length: number

tokens_generated: number

prompt_ms: number

generation_ms: number

tokens_per_second: number

}

export type ParallelStatus = {

n_parallel: number

active_slots: number

queued_requests: number

requests: ParallelRequestStatus[]

}