Part of the challenge here is where this happens.

CLI users have their toolboxes instantiated for them here:

Line 807 in e23e13e

tool_implementations = _gather_tools(tools, python_tools)

Or here in llm chat:

Line 1091 in e23e13e

tool_functions = _gather_tools(tools, python_tools)

But Python API users instantiate tools directly themselves:

import llm
from llm_tools_datasette import Datasette

model = llm.get_model("gpt-4.1-mini")

result = model.chain(
    "Show most interesting tables",
    tools=[Datasette("https://datasette.io/content")],
).text()

How do we record the "this class was instantiated at this moment and with these arguments" information such that, later on, this code in Response.log_to_db() can see that information and put it somewhere useful?

llm/llm/models.py

Lines 815 to 845 in e23e13e

    
           # Persist any tools, tool calls and tool results 
        
           tool_ids_by_name = {} 
        
           for tool in self.prompt.tools: 
        
               tool_id = ensure_tool(db, tool) 
        
               tool_ids_by_name[tool.name] = tool_id 
        
               db["tool_responses"].insert( 
        
                   { 
        
                       "tool_id": tool_id, 
        
                       "response_id": response_id, 
        
                   } 
        
               ) 
        
           for tool_call in self.tool_calls():  # TODO Should  be _or_raise() 
        
               db["tool_calls"].insert( 
        
                   { 
        
                       "response_id": response_id, 
        
                       "tool_id": tool_ids_by_name.get(tool_call.name) or None, 
        
                       "name": tool_call.name, 
        
                       "arguments": json.dumps(tool_call.arguments), 
        
                       "tool_call_id": tool_call.tool_call_id, 
        
                   } 
        
               ) 
        
           for tool_result in self.prompt.tool_results: 
        
               db["tool_results"].insert( 
        
                   { 
        
                       "response_id": response_id, 
        
                       "tool_id": tool_ids_by_name.get(tool_result.name) or None, 
        
                       "name": tool_result.name, 
        
                       "output": tool_result.output, 
        
                       "tool_call_id": tool_result.tool_call_id, 
        
                   } 
        
               )

	# Persist any tools, tool calls and tool results
	tool_ids_by_name = {}
	for tool in self.prompt.tools:
	tool_id = ensure_tool(db, tool)
	tool_ids_by_name[tool.name] = tool_id
	db["tool_responses"].insert(
	{
	"tool_id": tool_id,
	"response_id": response_id,
	}
	)
	for tool_call in self.tool_calls(): # TODO Should be _or_raise()
	db["tool_calls"].insert(
	{
	"response_id": response_id,
	"tool_id": tool_ids_by_name.get(tool_call.name) or None,
	"name": tool_call.name,
	"arguments": json.dumps(tool_call.arguments),
	"tool_call_id": tool_call.tool_call_id,
	}
	)
	for tool_result in self.prompt.tool_results:
	db["tool_results"].insert(
	{
	"response_id": response_id,
	"tool_id": tool_ids_by_name.get(tool_result.name) or None,
	"name": tool_result.name,
	"output": tool_result.output,
	"tool_call_id": tool_result.tool_call_id,
	}
	)

Uh oh!

Toolbox instantiation should be logged to DB when logging toolbox methods #1089

Description

Metadata

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

Issue actions