jlov7/Dynamic-Function-Calling-Agent
0
1"""2generate_json_syntax_training.py - Ultra-Focused JSON Syntax Training3 4This script creates training data specifically targeting the "Expecting ',' delimiter" 5errors that are the root cause of our 93% failure rate.6 7Analysis of failures shows the model has issues with:81. String parameters containing quotes and special characters 92. Proper JSON object structure and comma placement103. Consistent quote escaping in nested parameters11"""12 13import json14import random15from typing import List, Dict, Any16 17def create_training_pair(schema: Dict, question: str, good_response: str, bad_response: str) -> Dict:18 """Create a single training pair focused on JSON syntax."""19 prompt = f"""<|im_start|>system20You are a helpful assistant that calls functions by responding with valid JSON when given a schema. Always respond with JSON function calls only, never prose.<|im_end|>21 22<schema>23{json.dumps(schema, indent=2)}24</schema>25 26<|im_start|>user27{question}<|im_end|>28<|im_start|>assistant29"""30 31 return {32 "prompt": prompt,33 "chosen": good_response,34 "rejected": bad_response35 }36 37def generate_simple_json_patterns():38 """Generate basic JSON structure patterns to establish fundamentals."""39 examples = []40 41 # Simple single parameter42 examples.append(create_training_pair(43 {44 "name": "simple_function",45 "description": "Simple function with one parameter",46 "parameters": {47 "type": "object",48 "properties": {49 "text": {"type": "string"}50 },51 "required": ["text"]52 }53 },54 "Call with hello world",55 '{"name": "simple_function", "arguments": {"text": "hello world"}}',56 "I'll call the function with hello world"57 ))58 59 # Two parameters with proper comma60 examples.append(create_training_pair(61 {62 "name": "two_param_function",63 "description": "Function with two parameters",64 "parameters": {65 "type": "object",66 "properties": {67 "name": {"type": "string"},68 "age": {"type": "integer"}69 },70 "required": ["name", "age"]71 }72 },73 "Call with name John and age 25",74 '{"name": "two_param_function", "arguments": {"name": "John", "age": 25}}',75 '{"name": "two_param_function", "arguments": {"name": "John" "age": 25}}' # Missing comma76 ))77 78 return examples79 80def generate_string_escaping_patterns():81 """Generate patterns specifically for string parameter handling."""82 examples = []83 84 # String with internal quotes85 examples.append(create_training_pair(86 {87 "name": "analyze_text",88 "description": "Analyze text content",89 "parameters": {90 "type": "object",91 "properties": {92 "content": {"type": "string"},93 "type": {"type": "string"}94 },95 "required": ["content", "type"]96 }97 },98 "Analyze this text: The CEO said we have made tremendous progress this quarter",99 '{"name": "analyze_text", "arguments": {"content": "The CEO said we have made tremendous progress this quarter", "type": "analysis"}}',100 'I will analyze that text for you'101 ))102 103 # Multiple string parameters104 examples.append(create_training_pair(105 {106 "name": "send_message",107 "description": "Send a message",108 "parameters": {109 "type": "object",110 "properties": {111 "to": {"type": "string"},112 "subject": {"type": "string"},113 "body": {"type": "string"}114 },115 "required": ["to", "subject", "body"]116 }117 },118 "Send email to john@company.com with subject Meeting Update and body The meeting has been rescheduled to tomorrow at 2 PM",119 '{"name": "send_message", "arguments": {"to": "john@company.com", "subject": "Meeting Update", "body": "The meeting has been rescheduled to tomorrow at 2 PM"}}',120 'I will send that email for you'121 ))122 123 # Complex string with special characters124 examples.append(create_training_pair(125 {126 "name": "process_query",127 "description": "Process database query",128 "parameters": {129 "type": "object",130 "properties": {131 "query": {"type": "string"},132 "database": {"type": "string"}133 },134 "required": ["query", "database"]135 }136 },137 "Run query SELECT name FROM users WHERE created_at > 2023-01-01 on the main database",138 '{"name": "process_query", "arguments": {"query": "SELECT name FROM users WHERE created_at > 2023-01-01", "database": "main"}}',139 'I will run that database query for you'140 ))141 142 return examples143 144def generate_complex_parameter_patterns():145 """Generate patterns for complex parameter combinations."""146 examples = []147 148 # Boolean and integer mix149 examples.append(create_training_pair(150 {151 "name": "configure_system",152 "description": "Configure system settings", 153 "parameters": {154 "type": "object",155 "properties": {156 "timeout": {"type": "integer"},157 "enabled": {"type": "boolean"},158 "level": {"type": "string"}159 },160 "required": ["timeout", "enabled"]161 }162 },163 "Set timeout to 30 seconds, enable the system, and set level to debug",164 '{"name": "configure_system", "arguments": {"timeout": 30, "enabled": true, "level": "debug"}}',165 'I will configure the system with those settings'166 ))167 168 # Array parameter169 examples.append(create_training_pair(170 {171 "name": "process_files",172 "description": "Process multiple files",173 "parameters": {174 "type": "object",175 "properties": {176 "files": {"type": "array", "items": {"type": "string"}},177 "operation": {"type": "string"}178 },179 "required": ["files", "operation"]180 }181 },182 "Process files data.csv, results.json, and report.pdf with merge operation",183 '{"name": "process_files", "arguments": {"files": ["data.csv", "results.json", "report.pdf"], "operation": "merge"}}',184 'I will process those files for you'185 ))186 187 return examples188 189def generate_exact_failure_patterns():190 """Generate training examples that exactly match our failing schemas."""191 examples = []192 193 # Document summarizer pattern (our only passing schema)194 examples.append(create_training_pair(195 {196 "name": "summarize_document",197 "description": "Summarize document content",198 "parameters": {199 "type": "object",200 "properties": {201 "document_url": {"type": "string"},202 "summary_length": {"type": "string"},203 "target_audience": {"type": "string"}204 },205 "required": ["document_url"]206 }207 },208 "Summarize the document at https://example.com/report.pdf for executives with brief length",209 '{"name": "summarize_document", "arguments": {"document_url": "https://example.com/report.pdf", "summary_length": "brief", "target_audience": "executive"}}',210 'I will summarize that document for executives'211 ))212 213 # Sentiment analysis pattern (0% success)214 examples.append(create_training_pair(215 {216 "name": "analyze_sentiment",217 "description": "Analyze text sentiment",218 "parameters": {219 "type": "object",220 "properties": {221 "text": {"type": "string"},222 "language": {"type": "string"},223 "include_emotions": {"type": "boolean"}224 },225 "required": ["text"]226 }227 },228 "Analyze sentiment of this text: The product was excellent and delivery was fast with emotion details in English",229 '{"name": "analyze_sentiment", "arguments": {"text": "The product was excellent and delivery was fast", "language": "en", "include_emotions": true}}',230 'I will analyze the sentiment of that text'231 ))232 233 # Weather forecast pattern (0% success)234 examples.append(create_training_pair(235 {236 "name": "get_weather_forecast",237 "description": "Get weather forecast",238 "parameters": {239 "type": "object",240 "properties": {241 "location": {"type": "string"},242 "days": {"type": "integer"},243 "units": {"type": "string"},244 "include_hourly": {"type": "boolean"}245 },246 "required": ["location", "days"]247 }248 },249 "Get 3-day weather forecast for New York in metric units with hourly details",250 '{"name": "get_weather_forecast", "arguments": {"location": "New York", "days": 3, "units": "metric", "include_hourly": true}}',251 'I will get the weather forecast for New York'252 ))253 254 # Currency converter pattern (0% success)255 examples.append(create_training_pair(256 {257 "name": "convert_currency",258 "description": "Convert currency amounts",259 "parameters": {260 "type": "object",261 "properties": {262 "amount": {"type": "number"},263 "from_currency": {"type": "string"},264 "to_currency": {"type": "string"},265 "include_fees": {"type": "boolean"}266 },267 "required": ["amount", "from_currency", "to_currency"]268 }269 },270 "Convert 100 US dollars to Euros with fees included",271 '{"name": "convert_currency", "arguments": {"amount": 100, "from_currency": "USD", "to_currency": "EUR", "include_fees": true}}',272 'I will convert that currency amount for you'273 ))274 275 # Database optimizer pattern (0% success)276 examples.append(create_training_pair(277 {278 "name": "optimize_database_query",279 "description": "Optimize database query",280 "parameters": {281 "type": "object",282 "properties": {283 "sql_query": {"type": "string"},284 "database_type": {"type": "string"},285 "performance_target": {"type": "string"}286 },287 "required": ["sql_query", "database_type"]288 }289 },290 "Optimize this MySQL query for speed: SELECT id, name FROM users WHERE active = 1",291 '{"name": "optimize_database_query", "arguments": {"sql_query": "SELECT id, name FROM users WHERE active = 1", "database_type": "mysql", "performance_target": "speed"}}',292 'I will optimize that database query for you'293 ))294 295 return examples296 297def main():298 """Generate ultra-focused JSON syntax training dataset."""299 print("🎯 Generating Ultra-Focused JSON Syntax Training...")300 301 all_examples = []302 303 # Build progressively from simple to complex304 print("📝 Adding simple JSON patterns...")305 base_examples = generate_simple_json_patterns()306 all_examples.extend(base_examples)307 308 print("📝 Adding string escaping patterns...")309 string_examples = generate_string_escaping_patterns()310 all_examples.extend(string_examples)311 312 print("📝 Adding complex parameter patterns...")313 complex_examples = generate_complex_parameter_patterns()314 all_examples.extend(complex_examples)315 316 print("📝 Adding exact failure patterns...")317 failure_examples = generate_exact_failure_patterns()318 all_examples.extend(failure_examples)319 320 # Massively repeat the exact patterns that are failing321 print("📝 Adding 10x repetitions of exact failure patterns...")322 for _ in range(10):323 all_examples.extend(failure_examples)324 all_examples.extend(string_examples)325 all_examples.extend(complex_examples)326 327 # Save ultra-focused training data328 output_file = "tool_pairs_json_syntax.jsonl"329 with open(output_file, 'w') as f:330 for example in all_examples:331 f.write(json.dumps(example) + '\n')332 333 print(f"✅ Generated {len(all_examples)} ultra-focused training examples")334 print(f"💾 Saved to {output_file}")335 336 # Print breakdown337 categories = {338 "Simple JSON patterns": len(base_examples),339 "String escaping patterns": len(string_examples) * 11, # 10 extra repetitions340 "Complex parameters": len(complex_examples) * 11,341 "Exact failure patterns": len(failure_examples) * 11342 }343 344 print(f"\n📊 Ultra-Focused Training Composition:")345 for category, count in categories.items():346 print(f" {category}: {count} examples")347 348 print(f"\n🎯 Ultra-Focused Approach:")349 print(f" • 11x repetition of exact failing patterns")350 print(f" • Progressive complexity from simple to exact failures")351 print(f" • JSON syntax comma and quote handling emphasis")352 print(f" • Directly targeting 'Expecting , delimiter' errors")353 354 return len(all_examples)355 356if __name__ == "__main__":357 main()