diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index d78d4a3c..80ba3143 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -361,7 +361,13 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte auto & p = ctx.p; const auto & inputs = ctx.inputs; - auto until_suffix = p.rule("until-suffix", p.until(arguments.value_suffix)); + // stop at the trimmed closer only before an arg tag, so one inside a value does not match + std::vector value_end = { arguments.value_suffix }; + auto trimmed_suffix = trim_whitespace(arguments.value_suffix); + if (trimmed_suffix != arguments.value_suffix && !arguments.name_prefix.empty()) { + value_end.push_back(trimmed_suffix + arguments.name_prefix); + } + auto until_suffix = p.rule("until-suffix", p.until_one_of(value_end)); common_peg_parser tool_choice = p.choice(); @@ -394,7 +400,7 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte p.tool_arg_json_value(p.schema( p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) + p.space()) + - p.tool_arg_close(p.literal(arguments.value_suffix))); + p.tool_arg_close(p.optspace(arguments.value_suffix))); auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); if (is_required) { diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cpp index dc4a43e3..5fbc9348 100644 --- a/tests/test-chat-auto-parser.cpp +++ b/tests/test-chat-auto-parser.cpp @@ -86,6 +86,7 @@ static void test_normalize_quotes_with_embedded_quotes(testing & t); // TAG_WITH_TAGGED argument parsing tests static void test_tagged_args_with_embedded_quotes(testing & t); +static void test_tagged_args_closer_newline_optional(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); @@ -111,6 +112,7 @@ int main(int argc, char * argv[]) { t.test("standard_json_tools", test_standard_json_tools_formats); t.test("normalize_quotes_to_json", test_normalize_quotes_to_json); t.test("tagged_args_embedded_quotes", test_tagged_args_with_embedded_quotes); + t.test("tagged_args_closer_newline_optional", test_tagged_args_closer_newline_optional); return t.summary(); } @@ -2314,3 +2316,60 @@ static void test_cohere2moe_parser(testing & t) { t.assert_equal("parallel : tool 1 id", std::string("1"), parallel_msg.tool_calls[1].id); } } + +// the input drops the newline the inferred closer carries +static void test_tagged_args_closer_newline_optional(testing & t) { + struct autoparser a; + a.analysis_complete = true; + a.jinja_caps.supports_tool_calls = true; + a.tools.format.mode = tool_format::TAG_WITH_TAGGED; + a.tools.format.per_call_start = ""; + a.tools.format.per_call_end = ""; + a.tools.function.name_suffix = "\n"; + a.tools.arguments.name_prefix = ""; + a.tools.arguments.name_suffix = "\n"; + a.tools.arguments.value_prefix = ""; + a.tools.arguments.value_suffix = "\n"; + a.content.mode = content_mode::PLAIN; + a.reasoning.mode = reasoning_mode::NONE; + + generation_params inputs; + inputs.tools = json::parse(R"([{"type":"function","function":{"name":"search","parameters":{ + "type":"object","properties":{"file":{"type":"string"},"pattern":{"type":"string"}}, + "required":["file","pattern"]}}}])"); + inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO; + inputs.reasoning_format = COMMON_REASONING_FORMAT_NONE; + inputs.enable_thinking = false; + inputs.parallel_tool_calls = false; + + auto arena = a.build_parser(inputs); + common_chat_parser_params pp; + pp.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + pp.reasoning_format = COMMON_REASONING_FORMAT_NONE; + pp.parse_tool_calls = true; + + auto check = [&](const char * label, const std::string & gen) { + common_chat_msg msg; + try { + msg = common_chat_peg_parse(arena, gen, /* is_partial = */ false, pp); + } catch (const std::exception & ex) { + t.assert_true(std::string(label) + " : parsed (" + ex.what() + ")", false); + return; + } + if (!t.assert_equal(std::string(label) + " : tool calls", 1u, msg.tool_calls.size())) { + return; + } + const std::string & args = msg.tool_calls[0].arguments; + t.assert_true(std::string(label) + " : no delimiter absorbed", args.find("") == std::string::npos); + t.assert_true(std::string(label) + " : file value", args.find("\"file\":\"app.log\"") != std::string::npos); + t.assert_true(std::string(label) + " : pattern value", args.find("\"pattern\":\"error\"") != std::string::npos); + }; + + check("closer with newline", + "search\nfile\napp.log\n" + "pattern\nerror\n"); + + check("closer without newline", + "search\nfile\napp.log" + "pattern\nerror\n"); +}