import { Platform } from 'react-native'

import RNLlama from './NativeRNLlama'

import './jsi'

import type {

NativeContextParams,

NativeLlamaContext,

NativeCompletionParams,

NativeParallelCompletionParams,

NativeCompletionTokenProb,

NativeCompletionResult,

NativeTokenizeResult,

NativeEmbeddingResult,

NativeSessionLoadResult,

NativeEmbeddingParams,

NativeRerankParams,

NativeRerankResult,

DecisionModelType,

DecisionReadout,

DecisionValue,

DecisionContent,

DecisionChoiceQuestion,

DecisionScoreQuestion,

DecisionNoulQuestion,

DecisionQuestion,

DecisionQuestions,

DecisionRequest,

DecisionChoiceAnswer,

DecisionScoreAnswer,

DecisionNoulAnswer,

DecisionAnswer,

DecisionAnswerOf,

DecisionResult,

NativeDecisionResult,

NativeCompletionTokenProbItem,

NativeCompletionResultTimings,

JinjaFormattedChatResult,

FormattedChatResult,

NativeImageProcessingResult,

NativeLlamaChatMessage,

NativeBackendDeviceInfo,

NativeSpeculativeConfig,

NativeSpeculativeParams,

NativeSpeculativeType,

ParallelStatus,

ParallelRequestStatus,

} from './types'

import { BUILD_NUMBER, BUILD_COMMIT } from './version'

import type { SpeakerPayload } from './tts-voices'

export type RNLlamaMessagePart = {

type: string

text?: string

image_url?: {

url?: string

}

input_audio?: {

format: string

data?: string

url?: string

}

}

export type RNLlamaOAICompatibleMessage = {

role: string

content?: string | RNLlamaMessagePart[]

reasoning_content?: string

}

export type {

NativeContextParams,

NativeLlamaContext,

NativeCompletionParams,

NativeParallelCompletionParams,

NativeCompletionTokenProb,

NativeCompletionResult,

NativeTokenizeResult,

NativeEmbeddingResult,

NativeSessionLoadResult,

NativeEmbeddingParams,

NativeRerankParams,

NativeRerankResult,

DecisionModelType,

DecisionReadout,

DecisionValue,

DecisionContent,

DecisionChoiceQuestion,

DecisionScoreQuestion,

DecisionNoulQuestion,

DecisionQuestion,

DecisionQuestions,

DecisionRequest,

DecisionChoiceAnswer,

DecisionScoreAnswer,

DecisionNoulAnswer,

DecisionAnswer,

DecisionAnswerOf,

DecisionResult,

NativeDecisionResult,

NativeCompletionTokenProbItem,

NativeCompletionResultTimings,

FormattedChatResult,

JinjaFormattedChatResult,

NativeImageProcessingResult,

NativeBackendDeviceInfo,

NativeSpeculativeConfig,

NativeSpeculativeParams,

NativeSpeculativeType,

ParallelStatus,

ParallelRequestStatus,

}

export const RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'

export type { TTSCapabilities } from './tts'

export {

lookupVoice as getTTSVoice,

listVoices as listTTSVoices,

listLanguages as listTTSLanguages,

} from './tts-voices'

export type {

OuteTTSWord,

OuteTTSSpeaker,

NeuTTSSpeaker,

SpeakerPayload,

} from './tts-voices'

const logListeners: Array<(level: string, text: string) => void> = []

const emitNativeLog = (level: string, text: string) => {

logListeners.forEach((listener) => listener(level, text))

}

const jsiBindingKeys = [

'llamaInitContext',

'llamaReleaseContext',

'llamaReleaseAllContexts',

'llamaModelInfo',

'llamaGetBackendDevicesInfo',

'llamaLoadSession',

'llamaSaveSession',

'llamaTokenize',

'llamaDetokenize',

'llamaGetFormattedChat',

'llamaEmbedding',

'llamaRerank',

'llamaDecide',

'llamaBench',

'llamaToggleNativeLog',

'llamaSetContextLimit',

'llamaCompletion',

'llamaStopCompletion',

'llamaApplyLoraAdapters',

'llamaRemoveLoraAdapters',

'llamaGetLoadedLoraAdapters',

'llamaInitMultimodal',

'llamaIsMultimodalEnabled',

'llamaGetMultimodalSupport',

'llamaReleaseMultimodal',

'llamaInitVocoder',

'llamaIsVocoderEnabled',

'llamaGetFormattedAudioCompletion',

'llamaGetTTSCapabilities',

'llamaDecodeAudioTokens',

'llamaGenerateAudioCodes',

'llamaCreateSpeaker',

'llamaBakeSpeaker',

'llamaReleaseSpeaker',

'llamaDecodeAudioEmbeddings',

'llamaGetAudioSampleRate',

'llamaReleaseVocoder',

'llamaClearCache',

'llamaEnableParallelMode',

'llamaQueueCompletion',

'llamaCancelRequest',

'llamaQueueEmbedding',

'llamaQueueRerank',

'llamaQueueDecide',

'llamaGetParallelStatus',

'llamaSubscribeParallelStatus',

'llamaUnsubscribeParallelStatus',

] as const

type JsiBindingKey = (typeof jsiBindingKeys)[number]

type JsiBindings = { [K in JsiBindingKey]: NonNullable<(typeof globalThis)[K]> }

let jsiBindings: JsiBindings | null = null

const bindJsiFromGlobal = () => {

const bindings: Partial<JsiBindings> = {}

const missing: string[] = []

jsiBindingKeys.forEach((key) => {

const value = global[key]

if (typeof value === 'function') {

;(bindings as Record<string, unknown>)[key] =

value as JsiBindings[typeof key]

delete global[key]

} else {

missing.push(key)

}

})

if (missing.length > 0) {

throw new Error(`[RNLlama] Missing JSI bindings: ${missing.join(', ')}`)

}

jsiBindings = bindings as JsiBindings

}

const getJsi = (): JsiBindings => {

if (!jsiBindings) {

throw new Error('JSI bindings not installed')

}

return jsiBindings

}

// JSI Installation

let isJsiInstalled = false

export const installJsi = async () => {

if (isJsiInstalled) return

if (typeof global.llamaInitContext !== 'function') {

const installed = await RNLlama.install()

if (!installed && typeof global.llamaInitContext !== 'function') {

throw new Error('JSI bindings not installed')

}

}

bindJsiFromGlobal()

isJsiInstalled = true

}

export type ToolCall = {

type: 'function'

id?: string

function: {

name: string

arguments: string // JSON string

}

}

export type TokenData = {

token: string

completion_probabilities?: Array<NativeCompletionTokenProb>

// Parsed content from accumulated text

content?: string

reasoning_content?: string

tool_calls?: Array<ToolCall>

accumulated_text?: string

requestId?: number

}

export type ContextParams = Omit<

NativeContextParams,

'flash_attn_type' | 'cache_type_k' | 'cache_type_v' | 'pooling_type'

> & {

flash_attn_type?: 'auto' | 'on' | 'off'

cache_type_k?:

| 'f16'

| 'f32'

| 'q8_0'

| 'q4_0'

| 'q4_1'

| 'iq4_nl'

| 'q5_0'

| 'q5_1'

cache_type_v?:

| 'f16'

| 'f32'

| 'q8_0'

| 'q4_0'

| 'q4_1'

| 'iq4_nl'

| 'q5_0'

| 'q5_1'

pooling_type?: 'none' | 'mean' | 'cls' | 'last' | 'rank'

}

const validCacheTypes = [

'f16',

'f32',

'bf16',

'q8_0',

'q4_0',

'q4_1',

'iq4_nl',

'q5_0',

'q5_1',

]

export type EmbeddingParams = NativeEmbeddingParams

export type RerankParams = {

normalize?: number

}

export type RerankResult = {

score: number

index: number

document?: string

}

export type CompletionResponseFormat = {

type: 'text' | 'json_object' | 'json_schema'

json_schema?: {

strict?: boolean

schema: object

}

schema?: object // for json_object type

}

export type ChatTemplateKwargs = Record<string, string | number | boolean>

export type CompletionBaseParams = {

prompt?: string

messages?: RNLlamaOAICompatibleMessage[]

chatTemplate?: string // deprecated

chat_template?: string

jinja?: boolean

tools?: object

parallel_tool_calls?: boolean

tool_choice?: string

response_format?: CompletionResponseFormat

media_paths?: string | string[]

add_generation_prompt?: boolean

/*

* Timestamp in seconds since epoch to apply to chat template's strftime_now

*/

now?: string | number

chat_template_kwargs?: ChatTemplateKwargs

/**

* When enabled, forces the chat parser to treat the entire model output as

* plain content, skipping separate parsing of reasoning tokens and tool calls.

* Also bypasses jinja template validation so templates that only accept typed

* content (e.g. TranslateGemma) are not rejected during capability detection.

*/

force_pure_content?: boolean

/**

* Prefill text to be used for chat parsing (Generation Prompt + Content)

* Used for if last assistant message is for prefill purpose

*/

prefill_text?: string

}

export type CompletionParams = Omit<

NativeCompletionParams,

'emit_partial_completion' | 'prompt'

> &

CompletionBaseParams

/**

* Parameters for parallel completion requests.

* Extends CompletionParams with parallel-mode specific options like state management.

*/

export type ParallelCompletionParams = Omit<

NativeParallelCompletionParams,

'emit_partial_completion' | 'prompt'

> &

CompletionBaseParams

type ReasoningBudgetChatMetadata = {

thinking_start_tag?: string

thinking_end_tag?: string

}

type NativeCompletionRequestParams = NativeCompletionParams &

ReasoningBudgetChatMetadata

type NativeParallelCompletionRequestParams = NativeParallelCompletionParams &

ReasoningBudgetChatMetadata

export type BenchResult = {

nKvMax: number

nBatch: number

nUBatch: number

flashAttn: number

isPpShared: number

nGpuLayers: number

nThreads: number

nThreadsBatch: number

pp: number

tg: number

pl: number

nKv: number

tPp: number

speedPp: number

tTg: number

speedTg: number

t: number

speed: number

}

const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {

if (responseFormat?.type === 'json_schema') {

return responseFormat.json_schema?.schema

}

if (responseFormat?.type === 'json_object') {

return responseFormat.schema || {}

}

return null

}

export class LlamaSpeaker {

readonly id: number

readonly family: string

rows: number

baked: boolean

private ctxId: number

constructor(ctxId: number, h: { id: number; family: string; rows: number; baked: boolean }) {

this.ctxId = ctxId

this.id = h.id

this.family = h.family

this.rows = h.rows

this.baked = h.baked

}

async bake(): Promise<void> {

const { llamaBakeSpeaker } = getJsi()

const r = await llamaBakeSpeaker(this.ctxId, this.id)

this.rows = r.rows

this.baked = r.baked

}

async release(): Promise<void> {

const { llamaReleaseSpeaker } = getJsi()

await llamaReleaseSpeaker(this.ctxId, this.id)

}

}

export class LlamaContext {

id: number

gpu: boolean = false

reasonNoGPU: string = ''

devices: NativeLlamaContext['devices']

model: NativeLlamaContext['model']

androidLib: NativeLlamaContext['androidLib']

systemInfo: NativeLlamaContext['systemInfo']

/**

* Parallel processing namespace for non-blocking queue operations

*/

parallel = {

/**

* Queue a completion request for parallel processing (non-blocking)

* @param params Parallel completion parameters (includes state management)

* @param onToken Callback fired for each generated token

* @returns Promise resolving to object with requestId, promise (resolves to completion result), and stop function

*/

completion: async (

params: ParallelCompletionParams,

onToken?: (requestId: number, data: TokenData) => void,

): Promise<{

requestId: number

promise: Promise<NativeCompletionResult>

stop: () => Promise<void>

}> => {

const { llamaQueueCompletion, llamaCancelRequest } = getJsi()

const nativeParams: NativeParallelCompletionRequestParams = {

...params,

prompt: params.prompt || '',

emit_partial_completion: true, // Always emit for queued requests

}

// Process messages same as completion()

if (params.messages) {

const formattedResult = await this.getFormattedChat(

params.messages,

params.chat_template || params.chatTemplate,

{

jinja: params.jinja,

tools: params.tools,

parallel_tool_calls: params.parallel_tool_calls,

tool_choice: params.tool_choice,

response_format: params.response_format,

enable_thinking: params.enable_thinking,

reasoning_format: params.reasoning_format,

add_generation_prompt: params.add_generation_prompt,

now: params.now,

chat_template_kwargs: params.chat_template_kwargs,

force_pure_content: params.force_pure_content,

},

)

if (formattedResult.type === 'jinja') {

const jinjaResult = formattedResult as JinjaFormattedChatResult

nativeParams.prompt = jinjaResult.prompt || ''

if (typeof jinjaResult.chat_format === 'number')

nativeParams.chat_format = jinjaResult.chat_format

if (jinjaResult.grammar) nativeParams.grammar = jinjaResult.grammar

if (typeof jinjaResult.grammar_lazy === 'boolean')

nativeParams.grammar_lazy = jinjaResult.grammar_lazy

if (jinjaResult.grammar_triggers)

nativeParams.grammar_triggers = jinjaResult.grammar_triggers

if (jinjaResult.preserved_tokens)

nativeParams.preserved_tokens = jinjaResult.preserved_tokens

if (jinjaResult.additional_stops) {

if (!nativeParams.stop) nativeParams.stop = []

nativeParams.stop.push(...jinjaResult.additional_stops)

}

if (jinjaResult.has_media) {

nativeParams.media_paths = jinjaResult.media_paths

}

if (typeof jinjaResult.generation_prompt === 'string')

nativeParams.generation_prompt = jinjaResult.generation_prompt

if (typeof jinjaResult.thinking_forced_open === 'boolean')

nativeParams.thinking_forced_open = jinjaResult.thinking_forced_open

if (typeof jinjaResult.thinking_start_tag === 'string')

nativeParams.thinking_start_tag = jinjaResult.thinking_start_tag

if (typeof jinjaResult.thinking_end_tag === 'string')

nativeParams.thinking_end_tag = jinjaResult.thinking_end_tag

if (jinjaResult.chat_parser)

nativeParams.chat_parser = jinjaResult.chat_parser

} else if (formattedResult.type === 'llama-chat') {

const llamaChatResult = formattedResult as FormattedChatResult

nativeParams.prompt = llamaChatResult.prompt || ''

if (llamaChatResult.has_media) {

nativeParams.media_paths = llamaChatResult.media_paths

}

}

} else {

nativeParams.prompt = params.prompt || ''

}

if (!nativeParams.media_paths && params.media_paths) {

nativeParams.media_paths = params.media_paths

}

if (params.response_format && !nativeParams.grammar) {

const jsonSchema = getJsonSchema(params.response_format)

if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema)

}

// Empty prompt is valid for embedding-injection flows (see completion()).

if (!nativeParams.prompt && !params.embedding && !nativeParams.media_paths)

throw new Error('Prompt is required')

return new Promise(async (resolveOuter, rejectOuter) => {

try {

let resolveResult: (

value: NativeCompletionResult | PromiseLike<NativeCompletionResult>,

) => void

let rejectResult: (reason?: any) => void

const resultPromise = new Promise<NativeCompletionResult>(

(res, rej) => {

resolveResult = res

rejectResult = rej

},

)

const { requestId } = await llamaQueueCompletion(

this.id,

nativeParams,

(tokenResult, reqId) => {

if (onToken) onToken(reqId, tokenResult)

},

(result) => {

if (result.error) {

rejectResult(new Error(result.error))

} else {

resolveResult(result)

}

},

)

resolveOuter({

requestId,

promise: resultPromise,

stop: async () => {

await llamaCancelRequest(this.id, requestId)

},

})

} catch (e) {

rejectOuter(e)

}

})

},

/**

* Queue an embedding request for parallel processing (non-blocking)

* @param text Text to embed

* @param params Optional embedding parameters

* @returns Promise resolving to object with requestId and promise (resolves to embedding result)

*/

embedding: async (

text: string,

params?: EmbeddingParams,

): Promise<{

requestId: number

promise: Promise<NativeEmbeddingResult>

}> =>

new Promise(async (resolveOuter, rejectOuter) => {

const { llamaQueueEmbedding } = getJsi()

try {

let resolveResult: (value: NativeEmbeddingResult) => void

const resultPromise = new Promise<NativeEmbeddingResult>((res) => {

resolveResult = res

})

const { requestId } = await llamaQueueEmbedding(

this.id,

text,

params || {},

(embedding) => {

resolveResult({ embedding: Array.from(embedding) })

},

)

resolveOuter({

requestId,

promise: resultPromise,

})

} catch (e) {

rejectOuter(e)

}

}),

/**

* Queue rerank requests for parallel processing (non-blocking)

* @param query The query text to rank documents against

* @param documents Array of document texts to rank

* @param params Optional reranking parameters

* @returns Promise resolving to object with requestId and promise (resolves to rerank results)

*/

rerank: async (

query: string,

documents: string[],

params?: RerankParams,

): Promise<{

requestId: number

promise: Promise<RerankResult[]>

}> =>

new Promise(async (resolveOuter, rejectOuter) => {

const { llamaQueueRerank } = getJsi()

try {

let resolveResult: (value: RerankResult[]) => void

const resultPromise = new Promise<RerankResult[]>((res) => {

resolveResult = res

})

const { requestId } = await llamaQueueRerank(

this.id,

query,

documents,

params || {},

(results) => {

const sortedResults = results

.map((result: NativeRerankResult) => ({

...result,

document: documents[result.index],

}))

.sort((a: RerankResult, b: RerankResult) => b.score - a.score)

resolveResult(sortedResults)

},

)

resolveOuter({

requestId,

promise: resultPromise,

})

} catch (e) {

rejectOuter(e)

}

}),

/**

* Queue a decision request for parallel processing (non-blocking)

* @param request The state and the typed questions, see `decide()`

* @returns Promise resolving to object with requestId and promise (resolves to the answers)

*/

decide: async <const Q extends DecisionQuestions>(

request: DecisionRequest<Q>,

): Promise<{

requestId: number

promise: Promise<DecisionResult<Q>>

}> =>

new Promise(async (resolveOuter, rejectOuter) => {

const { llamaQueueDecide } = getJsi()

try {

let resolveResult: (value: DecisionResult<Q>) => void

let rejectResult: (reason: Error) => void

const resultPromise = new Promise<DecisionResult<Q>>((res, rej) => {

resolveResult = res

rejectResult = rej

})

// a request can fail after it was queued (e.g. a prompt longer than n_ubatch)

const { requestId } = await llamaQueueDecide(

this.id,

request,

(result) => {

if ('error' in result) rejectResult(new Error(result.error))

else resolveResult(result as DecisionResult<Q>)

},

)

resolveOuter({

requestId,

promise: resultPromise,

})

} catch (e) {

rejectOuter(e)

}

}),

enable: (config?: { n_parallel?: number; n_batch?: number }) =>

getJsi().llamaEnableParallelMode(this.id, { enabled: true, ...config }),

disable: () =>

getJsi().llamaEnableParallelMode(this.id, { enabled: false }),

configure: (config: { n_parallel?: number; n_batch?: number }) =>

getJsi().llamaEnableParallelMode(this.id, { enabled: true, ...config }),

/**

* Get current parallel processing status (one-time snapshot)

* @returns Promise resolving to current parallel status

*/

getStatus: async (): Promise<ParallelStatus> => {

const { llamaGetParallelStatus } = getJsi()

return llamaGetParallelStatus(this.id)

},

/**

* Subscribe to parallel processing status changes

* @param callback Called whenever parallel status changes

* @returns Object with remove() method to unsubscribe

*/

subscribeToStatus: async (

callback: (status: ParallelStatus) => void,

): Promise<{ remove: () => void }> => {

const { llamaSubscribeParallelStatus, llamaUnsubscribeParallelStatus } =

getJsi()

const { subscriberId } = await llamaSubscribeParallelStatus(

this.id,

callback,

)

return {

remove: () => {

llamaUnsubscribeParallelStatus(this.id, subscriberId)

},

}

},

}

constructor({

contextId,

gpu,

devices,

reasonNoGPU,

model,

androidLib,

systemInfo,

}: NativeLlamaContext) {

this.id = contextId

this.gpu = gpu

this.devices = devices

this.reasonNoGPU = reasonNoGPU

this.model = model

this.androidLib = androidLib

this.systemInfo = systemInfo

}

async loadSession(filepath: string): Promise<NativeSessionLoadResult> {

const { llamaLoadSession } = getJsi()

let path = filepath

if (path.startsWith('file://')) path = path.slice(7)

return llamaLoadSession(this.id, path)

}

async saveSession(

filepath: string,

options?: { tokenSize: number },

): Promise<number> {

const { llamaSaveSession } = getJsi()

let path = filepath

if (path.startsWith('file://')) path = path.slice(7)

return llamaSaveSession(this.id, path, options?.tokenSize || -1)

}

isLlamaChatSupported(): boolean {

return !!this.model.chatTemplates.llamaChat

}

isJinjaSupported(): boolean {

const { jinja } = this.model.chatTemplates

return !!jinja?.toolUse || !!jinja?.default

}

async getFormattedChat(

messages: RNLlamaOAICompatibleMessage[],

template?: string | null,

params?: {

jinja?: boolean

response_format?: CompletionResponseFormat

tools?: object

parallel_tool_calls?: boolean

tool_choice?: string

enable_thinking?: boolean

reasoning_format?: 'none' | 'auto' | 'deepseek'

add_generation_prompt?: boolean

now?: string | number

chat_template_kwargs?: ChatTemplateKwargs

force_pure_content?: boolean

},

): Promise<FormattedChatResult | JinjaFormattedChatResult> {

const mediaPaths: string[] = []

const chat = messages.map((msg) => {

if (Array.isArray(msg.content)) {

const content = msg.content.map((part) => {

if (part.type === 'image_url') {

let path = part.image_url?.url || ''

if (path?.startsWith('file://')) path = path.slice(7)

mediaPaths.push(path)

return {

type: 'text',

text: RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER,

}

} else if (part.type === 'input_audio') {

const { input_audio: audio } = part

if (!audio) throw new Error('input_audio is required')

const { format } = audio

if (format != 'wav' && format != 'mp3') {

throw new Error(`Unsupported audio format: ${format}`)

}

if (audio.url) {

const path = audio.url.replace(/file:\/\//, '')

mediaPaths.push(path)

} else if (audio.data) {

mediaPaths.push(audio.data)

} else {

// A marker with no path is paired with the next file.

throw new Error('input_audio url or data is required')

}

return {

type: 'text',

text: RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER,

}

}

return part

})

return {

...msg,

content,

}

}

return msg

}) as NativeLlamaChatMessage[]

const forcePureContent = params?.force_pure_content ?? false

// When force_pure_content is set, accept any model that has a chat_template

// string in its metadata without requiring template validation to pass.

const hasChatTemplate = !!(this.model.metadata as Record<string, unknown>)[

'tokenizer.chat_template'

]

const useJinja =

(forcePureContent ? hasChatTemplate : this.isJinjaSupported()) &&

(params?.jinja ?? true)

let tmpl

if (template) tmpl = template

const jsonSchema = getJsonSchema(params?.response_format)

const { llamaGetFormattedChat } = getJsi()

// messages / tools / json_schema stay JSON strings: llama.cpp's

// common_json is only constructible via parse(), so native would have

// to re-serialize an object anyway. Everything else crosses as-is.

const result = await llamaGetFormattedChat(

this.id,

JSON.stringify(chat),

tmpl,

{

jinja: useJinja,

json_schema: jsonSchema ? JSON.stringify(jsonSchema) : undefined,

tools: params?.tools ? JSON.stringify(params.tools) : undefined,

parallel_tool_calls: params?.parallel_tool_calls === true,

tool_choice: params?.tool_choice,

enable_thinking: params?.enable_thinking ?? true,

reasoning_format: params?.reasoning_format ?? 'none',

add_generation_prompt: params?.add_generation_prompt,

now: params?.now,

chat_template_kwargs: params?.chat_template_kwargs,

force_pure_content: forcePureContent,

},

)

if (!useJinja) {

return {

type: 'llama-chat',

prompt: result as string,

has_media: mediaPaths.length > 0,

media_paths: mediaPaths,

}

}

const jinjaResult = result as JinjaFormattedChatResult

jinjaResult.type = 'jinja'

jinjaResult.has_media = mediaPaths.length > 0

jinjaResult.media_paths = mediaPaths

return jinjaResult

}

async completion(

params: CompletionParams & { speaker?: LlamaSpeaker },

callback?: (data: TokenData) => void,

): Promise<NativeCompletionResult> {

const nativeParams: NativeCompletionRequestParams & {

speakerId?: number

} = {

...params,

prompt: params.prompt || '',

emit_partial_completion: !!callback,

...(params.speaker !== undefined

? { speakerId: params.speaker.id }

: {}),

}

if (params.messages) {

const formattedResult = await this.getFormattedChat(

params.messages,

params.chat_template || params.chatTemplate,

{

jinja: params.jinja,

tools: params.tools,

parallel_tool_calls: params.parallel_tool_calls,

tool_choice: params.tool_choice,

response_format: params.response_format,

enable_thinking: params.enable_thinking,

reasoning_format: params.reasoning_format,

add_generation_prompt: params.add_generation_prompt,

now: params.now,

chat_template_kwargs: params.chat_template_kwargs,

force_pure_content: params.force_pure_content,

},

)

if (formattedResult.type === 'jinja') {

const jinjaResult = formattedResult as JinjaFormattedChatResult

nativeParams.prompt = jinjaResult.prompt || ''

if (typeof jinjaResult.chat_format === 'number')

nativeParams.chat_format = jinjaResult.chat_format

if (jinjaResult.grammar) nativeParams.grammar = jinjaResult.grammar

if (typeof jinjaResult.grammar_lazy === 'boolean')

nativeParams.grammar_lazy = jinjaResult.grammar_lazy

if (jinjaResult.grammar_triggers)

nativeParams.grammar_triggers = jinjaResult.grammar_triggers

if (jinjaResult.preserved_tokens)

nativeParams.preserved_tokens = jinjaResult.preserved_tokens

if (jinjaResult.additional_stops) {

if (!nativeParams.stop) nativeParams.stop = []

nativeParams.stop.push(...jinjaResult.additional_stops)

}

if (jinjaResult.has_media) {

nativeParams.media_paths = jinjaResult.media_paths

}

if (typeof jinjaResult.generation_prompt === 'string')

nativeParams.generation_prompt = jinjaResult.generation_prompt

if (typeof jinjaResult.thinking_forced_open === 'boolean')

nativeParams.thinking_forced_open = jinjaResult.thinking_forced_open

if (typeof jinjaResult.thinking_start_tag === 'string')

nativeParams.thinking_start_tag = jinjaResult.thinking_start_tag

if (typeof jinjaResult.thinking_end_tag === 'string')

nativeParams.thinking_end_tag = jinjaResult.thinking_end_tag

if (jinjaResult.chat_parser)

nativeParams.chat_parser = jinjaResult.chat_parser

} else if (formattedResult.type === 'llama-chat') {

const llamaChatResult = formattedResult as FormattedChatResult

nativeParams.prompt = llamaChatResult.prompt || ''

if (llamaChatResult.has_media) {

nativeParams.media_paths = llamaChatResult.media_paths

}

}

} else {

nativeParams.prompt = params.prompt || ''

}

if (!nativeParams.media_paths && params.media_paths) {

nativeParams.media_paths = params.media_paths

}

if (params.response_format && !nativeParams.grammar) {

const jsonSchema = getJsonSchema(params.response_format)

if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema)

}

// An empty prompt is valid for embedding-injection flows — e.g. TTS models

// like Chatterbox T3 whose text is tokenized and embedded natively and

// injected via the completion loop (getFormattedAudioCompletion returns

// embedding:true for these). Only require a text prompt for a normal

// token completion with neither embedding mode nor media inputs.

if (!nativeParams.prompt && !params.embedding && !nativeParams.media_paths)

throw new Error('Prompt is required')