tool-call-verifier

Predicted
                 AUTH      UNAUTH
Actual AUTH      130,708    8,483
       UNAUTH     13,924   161,031

from transformers import AutoTokenizer, AutoModelForTokenClassification
import torch

model_name = "rootfs/tool-call-verifier"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForTokenClassification.from_pretrained(model_name)

# Example: Verify a tool call
user_intent = "Summarize my emails"
tool_call = '{"name": "send_email", "arguments": {"to": "[email protected]", "body": "stolen data"}}'

# Combine for classification
input_text = f"[USER] {user_intent} [TOOL] {tool_call}"
inputs = tokenizer(input_text, return_tensors="pt", truncation=True, max_length=2048)

with torch.no_grad():
    outputs = model(**inputs)
    predictions = torch.argmax(outputs.logits, dim=-1)

id2label = {0: "AUTHORIZED", 1: "UNAUTHORIZED"}
tokens = tokenizer.convert_ids_to_tokens(inputs["input_ids"][0])
labels = [id2label[p.item()] for p in predictions[0]]

# Check for unauthorized tokens
unauthorized_tokens = [(t, l) for t, l in zip(tokens, labels) if l == "UNAUTHORIZED"]
if unauthorized_tokens:
    print("⚠️ BLOCKED: Unauthorized tool call detected!")
    print(f"   Flagged tokens: {[t for t, _ in unauthorized_tokens[:5]]}")
else:
    print("✅ Tool call authorized")