KBaba7/llama.cpp
0
1#ifdef NDEBUG2#undef NDEBUG3#endif4 5#include "unicode.h"6#include "llama-grammar.h"7#include "json-schema-to-grammar.h"8 9#include <cassert>10#include <string>11#include <vector>12 13using json = nlohmann::ordered_json;14 15static llama_grammar * build_grammar(const std::string & grammar_str) {16 return llama_grammar_init_impl(nullptr, grammar_str.c_str(), "root", false, nullptr, 0, nullptr, 0);17}18 19static bool test_build_grammar_fails(const std::string & grammar_str) {20 fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());21 bool grammar_fails = false;22 llama_grammar * grammar = build_grammar(grammar_str);23 if (grammar != nullptr) {24 fprintf(stderr, " ❌ Expected build failure, but succeeded\n");25 } else {26 grammar_fails = true;27 fprintf(stdout, " ✅︎\n");28 }29 return grammar_fails;30}31 32static bool match_string(const std::string & input, llama_grammar * grammar) {33 const auto cpts = unicode_cpts_from_utf8(input);34 35 auto & stacks_cur = llama_grammar_get_stacks(grammar);36 37 for (const auto & cpt : cpts) {38 llama_grammar_accept(grammar, cpt);39 40 if (stacks_cur.empty()) {41 // no stacks means that the grammar failed to match at this point42 return false;43 }44 }45 46 for (const auto & stack : stacks_cur) {47 if (stack.empty()) {48 // An empty stack means that the grammar has been completed49 return true;50 }51 }52 53 return false;54}55 56static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {57 fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());58 fflush(stderr);59 60 auto * grammar = build_grammar(grammar_str);61 62 // Save the original grammar stacks so that we can reset after every new string we want to test63 const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy64 65 llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar);66 67 fprintf(stderr, " 🔵 Valid strings:\n");68 69 // Passing strings70 for (const auto & test_string : passing_strings) {71 fprintf(stderr, " \"%s\" ", test_string.c_str());72 fflush(stderr);73 74 bool matched = match_string(test_string, grammar);75 76 if (!matched) {77 fprintf(stderr, "❌ (failed to match)\n");78 79 // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.80 // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf81 FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");82 if (grammar_file) {83 fprintf(grammar_file, "%s", grammar_str.c_str());84 fclose(grammar_file);85 }86 87 // DEBUG: Write the test string to test-grammar-integration.string.txt88 FILE* string_file = fopen("test-grammar-integration.string.txt", "w");89 if (string_file) {90 fprintf(string_file, "%s", test_string.c_str());91 fclose(string_file);92 }93 94 fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");95 } else {96 fprintf(stdout, "✅︎\n");97 }98 99 assert(matched);100 101 // Reset the grammar stacks102 stacks_cur = stacks_org;103 }104 105 fprintf(stderr, " 🟠 Invalid strings:\n");106 107 // Failing strings108 for (const auto & test_string : failing_strings) {109 fprintf(stderr, " \"%s\" ", test_string.c_str());110 fflush(stderr);111 112 bool matched = match_string(test_string, grammar);113 114 if (matched) {115 fprintf(stderr, "❌ (incorrectly matched)\n");116 } else {117 fprintf(stdout, "✅︎\n");118 }119 assert(!matched);120 121 // Reset the grammar stacks122 stacks_cur = stacks_org;123 }124 125 // Clean up allocated memory126 llama_grammar_free_impl(grammar);127}128static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {129 test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);130}131static void test_schema(const std::string & test_desc, const std::string & schema_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {132 test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str), true), passing_strings, failing_strings);133}134 135static void test_simple_grammar() {136 test_schema(137 "min 0",138 R"""({139 "type": "integer",140 "minimum": 0141 })""",142 // Passing strings143 {144 "0",145 "10",146 "12",147 "10000",148 },149 // Failing strings150 {151 "-1",152 "-10",153 "-10000",154 "-100000000000000000000000000000000",155 "100000000000000000000000000000000",156 "00",157 "01",158 "-0",159 }160 );161 test_schema(162 "min 2",163 // Schema164 R"""({165 "type": "integer",166 "minimum": 2167 })""",168 // Passing strings169 {170 "2",171 "3",172 "4",173 "10",174 "20",175 "1234567890000000",176 },177 // Failing strings178 {179 "0",180 "1",181 "-1",182 "-100",183 "0",184 "1",185 "01",186 "02",187 "12345678900000000",188 }189 );190 test_schema(191 "min 456",192 R"""({193 "type": "integer",194 "minimum": 456195 })""",196 // Passing strings197 {198 "456",199 "4560",200 "457",201 "460",202 "500",203 },204 // Failing strings205 {206 "455",207 "356",208 "50",209 "050",210 "-1",211 "-456",212 }213 );214 test_schema(215 "min -123",216 R"""({217 "type": "integer",218 "minimum": -123219 })""",220 // Passing strings221 {222 "-123",223 "-122",224 "-11",225 "-1",226 "0",227 "1",228 "123",229 "1234",230 "2345",231 },232 // Failing strings233 {234 "-1234",235 "-124",236 }237 );238 239 test_schema(240 "max 9999",241 // Schema242 R"""({243 "type": "integer",244 "maximum": 9999245 })""",246 // Passing strings247 {248 "-99999",249 "0",250 "9999",251 },252 // Failing strings253 {254 "10000",255 "99991",256 }257 );258 test_schema(259 "max -9999",260 // Schema261 R"""({262 "type": "integer",263 "maximum": -9999264 })""",265 // Passing strings266 {267 "-10000",268 "-9999",269 },270 // Failing strings271 {272 "-9998",273 "0",274 "9999",275 }276 );277 test_schema(278 "min 5 max 30",279 // Schema280 R"""({281 "type": "integer",282 "minimum": 5,283 "maximum": 30284 })""",285 // Passing strings286 {287 "5",288 "10",289 "30",290 },291 // Failing strings292 {293 "05",294 "4",295 "-1",296 "31",297 "123",298 "0123",299 }300 );301 test_schema(302 "min -1 max 1",303 R"""({304 "type": "integer",305 "minimum": -1,306 "maximum": 1307 })""",308 // Passing strings309 {310 "-1",311 "0",312 "1",313 },314 // Failing strings315 {316 "-11",317 "-10",318 "-2",319 "2",320 "10",321 "11",322 }323 );324 test_schema(325 "min -123 max 42",326 R"""({327 "type": "integer",328 "minimum": -123,329 "maximum": 42330 })""",331 // Passing strings332 {333 "-123",334 "-122",335 "-13",336 "-11",337 "-2",338 "-1",339 "0",340 "1",341 "5",342 "10",343 "39",344 "40",345 "42",346 },347 // Failing strings348 {349 "-0123",350 "-124",351 "-1123",352 "-200",353 "43",354 "123",355 "0123",356 }357 );358 test_schema(359 "exclusive min / max",360 // Schema361 R"""({362 "type": "integer",363 "exclusiveMinimum": 0,364 "exclusiveMaximum": 10000365 })""",366 // Passing strings367 {368 "1",369 "9999",370 },371 // Failing strings372 {373 "0",374 "01",375 "10000",376 "99999",377 }378 );379 380 // Test case for a simple grammar381 test_grammar(382 "simple grammar",383 R"""(384 root ::= expr385 expr ::= term ("+" term)*386 term ::= number387 number ::= [0-9]+)""",388 // Passing strings389 {390 "42",391 "1+2+3+4+5",392 "123+456",393 },394 // Failing strings395 {396 "+",397 "/ 3",398 "1+2+3+4+5+",399 "12a45",400 }401 );402}403 404static void test_complex_grammar() {405 // Test case for a more complex grammar, with both failure strings and success strings406 test_grammar(407 "medium complexity grammar",408 // Grammar409 R"""(410 root ::= expression411 expression ::= term ws (("+"|"-") ws term)*412 term ::= factor ws (("*"|"/") ws factor)*413 factor ::= number | variable | "(" expression ")" | function-call414 number ::= [0-9]+415 variable ::= [a-zA-Z_][a-zA-Z0-9_]*416 function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"417 ws ::= [ \t\n\r]?)""",418 // Passing strings419 {420 "42",421 "1*2*3*4*5",422 "x",423 "x+10",424 "x1+y2",425 "(a+b)*(c-d)",426 "func()",427 "func(x,y+2)",428 "a*(b+c)-d/e",429 "f(g(x),h(y,z))",430 "x + 10",431 "x1 + y2",432 "(a + b) * (c - d)",433 "func()",434 "func(x, y + 2)",435 "a * (b + c) - d / e",436 "f(g(x), h(y, z))",437 "123+456",438 "123*456*789-123/456+789*123",439 "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456"440 },441 // Failing strings442 {443 "+",444 "/ 3x",445 "x + + y",446 "a * / b",447 "func(,)",448 "func(x y)",449 "(a + b",450 "x + y)",451 "a + b * (c - d",452 "42 +",453 "x +",454 "x + 10 +",455 "(a + b) * (c - d",456 "func(",457 "func(x, y + 2",458 "a * (b + c) - d /",459 "f(g(x), h(y, z)",460 "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",461 }462 );463}464 465static void test_special_chars() {466 // A collection of tests to exercise special characters such as "."467 test_grammar(468 "special characters",469 // Grammar470 R"""(471 root ::= ... "abc" ...472 )""",473 // Passing strings474 {475 "abcabcabc",476 "aaaabcccc",477 // NOTE: Also ensures that multi-byte characters still count as a single character478 "🔵🟠✅abc❌🟠🔵"479 },480 // Failing strings481 {482 "aaabcccc",483 "aaaaabcccc",484 "aaaabccc",485 "aaaabccccc",486 "🔵🟠✅❌abc❌✅🟠🔵",487 "🔵🟠abc🟠🔵"488 }489 );490}491 492static void test_quantifiers() {493 // A collection of tests to exercise * + and ? quantifiers494 495 test_grammar(496 "* quantifier",497 // Grammar498 R"""(root ::= "a"*)""",499 // Passing strings500 {501 "",502 "a",503 "aaaaa",504 "aaaaaaaaaaaaaaaaaa",505 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"506 },507 // Failing strings508 {509 "b",510 "ab",511 "aab",512 "ba",513 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"514 }515 );516 test_grammar(517 "+ quantifier",518 // Grammar519 R"""(root ::= "a"+)""",520 // Passing strings521 {522 "a",523 "aaaaa",524 "aaaaaaaaaaaaaaaaaa",525 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"526 },527 // Failing strings528 {529 "",530 "b",531 "ab",532 "aab",533 "ba",534 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"535 }536 );537 test_grammar(538 "? quantifier",539 // Grammar540 R"""(root ::= "a"?)""",541 // Passing strings542 {543 "",544 "a"545 },546 // Failing strings547 {548 "b",549 "ab",550 "aa",551 "ba",552 }553 );554 test_grammar(555 "mixed quantifiers",556 // Grammar557 R"""(558 root ::= cons+ vowel* cons? (vowel cons)*559 vowel ::= [aeiouy]560 cons ::= [bcdfghjklmnpqrstvwxyz]561 )""",562 // Passing strings563 {564 "yes",565 "no",566 "noyes",567 "crwth",568 "four",569 "bryyyy",570 },571 // Failing strings572 {573 "yess",574 "yesno",575 "forty",576 "catyyy",577 }578 );579 test_grammar(580 "simple exact repetition",581 // Grammar582 R"""(583 root ::= [ab]{4}584 )""",585 // Passing strings586 {587 "aaaa",588 "bbbb",589 "abab",590 },591 // Failing strings592 {593 "a",594 "b",595 "aaaaa",596 }597 );598 test_grammar(599 "simple min repetition",600 // Grammar601 R"""(602 root ::= [ab]{4,}603 )""",604 // Passing strings605 {606 "aaaa",607 "aaaaab",608 "bbbb",609 "ababab",610 },611 // Failing strings612 {613 "",614 "aba",615 }616 );617 test_grammar(618 "simple max repetition",619 // Grammar620 R"""(621 root ::= [ab]{0,4}622 )""",623 // Passing strings624 {625 "",626 "a",627 "aa",628 "aaa",629 "aaab",630 },631 // Failing strings632 {633 "aaaaa",634 }635 );636 test_grammar(637 "min / max repetition",638 // Grammar639 R"""(640 root ::= ("0x" [A-F0-9]{2} " "?){3,5}641 )""",642 // Passing strings643 {644 "0xFF 0x12 0xAB",645 "0xFF 0x12 0xAB 0x00 0x00",646 },647 // Failing strings648 {649 "",650 "0xFF",651 "0xFF 0x12",652 "0xFF 0x12 0xAB 0x00 0x00 0x00",653 }654 );655}656 657static void test_failure_missing_root() {658 fprintf(stderr, "⚫ Testing missing root node:\n");659 // Test case for a grammar that is missing a root rule660 const std::string grammar_str = R"""(661 rot ::= expr662 expr ::= term ("+" term)*663 term ::= number664 number ::= [0-9]+)""";665 666 llama_grammar_parser parsed_grammar;667 parsed_grammar.parse(grammar_str.c_str());668 669 // Ensure we parsed correctly670 assert(!parsed_grammar.rules.empty());671 672 // Ensure we do NOT have a root node673 assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());674 fprintf(stderr, " ✅︎ Passed\n");675}676 677static void test_failure_missing_reference() {678 fprintf(stderr, "⚫ Testing missing reference node:\n");679 680 // Test case for a grammar that is missing a referenced rule681 const std::string grammar_str =682 R"""(root ::= expr683 expr ::= term ("+" term)*684 term ::= numero685 number ::= [0-9]+)""";686 687 fprintf(stderr, " Expected error: ");688 689 llama_grammar_parser parsed_grammar;690 parsed_grammar.parse(grammar_str.c_str());691 692 // Ensure we did NOT parsed correctly693 assert(parsed_grammar.rules.empty());694 695 fprintf(stderr, " End of expected error.\n");696 fprintf(stderr, " ✅︎ Passed\n");697}698 699static void test_failure_left_recursion() {700 fprintf(stderr, "⚫ Testing left recursion detection:\n");701 702 // Test simple left recursion detection703 const std::string simple_str = R"""(root ::= "a" | root "a")""";704 assert(test_build_grammar_fails(simple_str));705 706 // Test more complicated left recursion detection707 const std::string medium_str = R"""(708 root ::= asdf709 asdf ::= "a" | asdf "a"710 )""";711 assert(test_build_grammar_fails(medium_str));712 713 // Test even more complicated left recursion detection714 const std::string hard_str = R"""(715 root ::= asdf716 asdf ::= "a" | foo "b"717 foo ::= "c" | asdf "d" | "e")""";718 assert(test_build_grammar_fails(hard_str));719 720 // Test yet even more complicated left recursion detection721 const std::string hardest_str = R"""(722 root ::= asdf723 asdf ::= "a" | foo "b"724 foo ::= "c" | empty asdf "d" | "e"725 empty ::= "blah" | )""";726 assert(test_build_grammar_fails(hardest_str));727 728 fprintf(stderr, " ✅︎ Passed\n");729}730 731static void test_json_schema() {732 // Note that this is similar to the regular grammar tests,733 // but we convert each json schema to a grammar before parsing.734 // Otherwise, this test structure is the same.735 736 test_schema(737 "empty schema (object)",738 // Schema739 R"""(740 {}741 )""",742 // Passing strings743 {744 R"""({})""",745 R"""({"foo": "bar"})""",746 },747 // Failing strings748 {749 "",750 "[]",751 "null",752 R"""("")""",753 "true",754 }755 );756 757 test_schema(758 "exotic formats (list)",759 // Schema760 R"""({761 "items": [762 { "format": "date" },763 { "format": "uuid" },764 { "format": "time" },765 { "format": "date-time" }766 ]767 })""",768 // Passing strings769 {770 // "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?771 // "[]", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?772 R"""(["2012-04-23", "12345678-1234-1234-1234-1234567890ab", "18:25:43.511Z", "2012-04-23T18:25:43.511Z"])""",773 //R"""(["2012-04-23","12345678-1234-1234-1234-1234567890ab"])""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?774 //R"""({"foo": "bar"})""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?775 },776 // Failing strings777 {778 R"""(["foo", "bar"])""",779 R"""(["12345678-1234-1234-1234-1234567890ab"])""",780 }781 );782 783 test_schema(784 "string",785 // Schema786 R"""({787 "type": "string"788 })""",789 // Passing strings790 {791 R"""("foo")""",792 R"""("bar")""",793 R"""("")""",794 },795 // Failing strings796 {797 R"""({})""",798 R"""("foo": "bar")""",799 }800 );801 802 test_schema(803 "string w/ min length 1",804 // Schema805 R"""({806 "type": "string",807 "minLength": 1808 })""",809 // Passing strings810 {811 R"""("foo")""",812 R"""("bar")""",813 },814 // Failing strings815 {816 R"""("")""",817 R"""({})""",818 R"""("foo": "bar")""",819 }820 );821 822 test_schema(823 "string w/ min length 3",824 // Schema825 R"""({826 "type": "string",827 "minLength": 3828 })""",829 // Passing strings830 {831 R"""("foo")""",832 R"""("bar")""",833 R"""("foobar")""",834 },835 // Failing strings836 {837 R"""("")""",838 R"""("f")""",839 R"""("fo")""",840 }841 );842 843 test_schema(844 "string w/ max length",845 // Schema846 R"""({847 "type": "string",848 "maxLength": 3849 })""",850 // Passing strings851 {852 R"""("foo")""",853 R"""("bar")""",854 R"""("")""",855 R"""("f")""",856 R"""("fo")""",857 },858 // Failing strings859 {860 R"""("foobar")""",861 }862 );863 864 test_schema(865 "string w/ min & max length",866 // Schema867 R"""({868 "type": "string",869 "minLength": 1,870 "maxLength": 4871 })""",872 // Passing strings873 {874 R"""("foo")""",875 R"""("bar")""",876 R"""("f")""",877 R"""("barf")""",878 },879 // Failing strings880 {881 R"""("")""",882 R"""("barfo")""",883 R"""("foobar")""",884 }885 );886 887 test_schema(888 "boolean",889 // Schema890 R"""({891 "type": "boolean"892 })""",893 // Passing strings894 {895 "true",896 "false",897 },898 // Failing strings899 {900 R"""("")""",901 R"""("true")""",902 R"""(True)""",903 R"""(FALSE)""",904 }905 );906 907 test_schema(908 "integer",909 // Schema910 R"""({911 "type": "integer"912 })""",913 // Passing strings914 {915 R"""(0)""",916 R"""(12345)""",917 R"""(1234567890123456)""",918 },919 // Failing strings920 {921 R"""()""",922 R"""(01)""",923 R"""(007)""",924 R"""(12345678901234567 )""",925 }926 );927 928 test_schema(929 "string const",930 // Schema931 R"""({932 "const": "foo"933 })""",934 // Passing strings935 {936 R"""("foo")""",937 },938 // Failing strings939 {940 R"""(foo)""",941 R"""("bar")""",942 }943 );944 945 test_schema(946 "non-string const",947 // Schema948 R"""({949 "const": true950 })""",951 // Passing strings952 {953 R"""(true)""",954 },955 // Failing strings956 {957 R"""()""",958 R"""(foo)""",959 R"""("true")""",960 }961 );962 963 test_schema(964 "non-string const",965 // Schema966 R"""({967 "enum": ["red", "amber", "green", null, 42, ["foo"]]968 })""",969 // Passing strings970 {971 R"""("red")""",972 R"""(null)""",973 R"""(42)""",974 R"""(["foo"])""",975 },976 // Failing strings977 {978 R"""()""",979 R"""(420)""",980 R"""(true)""",981 R"""(foo)""",982 }983 );984 985 test_schema(986 "simple pattern",987 // Schema988 R"""({989 "pattern": "^[a-zA-Z0-9_-]*$"990 })""",991 // Passing strings992 {993 R"""("")""",994 R"""("He_llo-12")""",995 },996 // Failing strings997 {998 R"""("!")""",999 R"""("Hello World")""",1000 }1001 );1002 1003 test_schema(1004 "pattern with escapes",1005 // Schema1006 R"""({1007 "pattern": "^a\\^\\$\\.\\[\\]\\(\\)\\|\\{\\}\\*\\+\\?b$"1008 })""",1009 // Passing strings1010 {1011 R"""("a^$.[]()|{}*+?b")""",1012 },1013 // Failing strings1014 {1015 R"""("ab")""",1016 }1017 );1018 1019 test_schema(1020 "",1021 // Schema1022 R"""(1023 {1024 "type": ["array", "null"],1025 "items": { "type": "string" }1026 }1027 )""",1028 // Passing strings1029 {1030 "null",1031 "[]",1032 "[\"123\"]",1033 "[\"foo\", \"bar\"]",1034 },1035 // Failing strings1036 {1037 "",1038 "[123]",1039 "\"foo\"",1040 "[\"foo\", 42]",1041 }1042 );1043 1044 test_schema(1045 "min+max items",1046 // Schema1047 R"""({1048 "items": {1049 "type": ["number", "integer"]1050 },1051 "minItems": 3,1052 "maxItems": 51053 })""",1054 // Passing strings1055 {1056 R"""([1, 2, 3])""",1057 R"""([1, 2, 3, 4])""",1058 R"""([1, 2, 3, 4, 5])""",1059 },1060 // Failing strings1061 {1062 R"""([1, 2])""",1063 R"""([1, 2, 3, 4, 5, 6])""",1064 R"""(1)""",1065 }1066 );1067 1068 // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)1069 test_schema(1070 "object properties",1071 // Schema1072 R"""({1073 "type": "object",1074 "properties": {1075 "number": { "type": "number" },1076 "street_name": { "type": "string" },1077 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }1078 }1079 })""",1080 // Passing strings1081 {1082 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",1083 // "By default, leaving out properties is valid"1084 R"""({ "street_name": "Pennsylvania" })""",1085 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",1086 // "By extension, even an empty object is valid"1087 R"""({})""",1088 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",1089 },1090 // Failing strings1091 {1092 // Change datatype from number to string1093 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",1094 // Reorder properties1095 R"""({ "street_name": "Pennsylvania", "number": 1600 })""",1096 // Reorder properties1097 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",1098 // "Additional properties default to false for generation, even though the spec says true.1099 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",1100 1101 }1102 );1103 1104 test_schema(1105 "additional properties can't override other properties",1106 R"""({1107 "properties": {1108 "a": {"type": "integer"},1109 "b": {"type": "integer"}1110 },1111 "additionalProperties": true1112 })""",1113 // Passing strings1114 {1115 R"""({"a": 42})""",1116 R"""({"c": ""})""",1117 R"""({"a": 42, "c": ""})""",1118 R"""({"a_": ""})""",1119 },1120 // Failing strings1121 {1122 R"""()""",1123 R"""({"a": ""})""",1124 R"""({"a": "", "b": ""})""",1125 }1126 );1127 1128 // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)1129 test_schema(1130 "object properties, additionalProperties: true",1131 // Schema1132 R"""({1133 "type": "object",1134 "properties": {1135 "number": { "type": "number" },1136 "street_name": { "type": "string" },1137 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }1138 },1139 "additionalProperties": true1140 })""",1141 // Passing strings1142 {1143 // "By extension, even an empty object is valid"1144 R"""({})""",1145 R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",1146 // "By default, leaving out properties is valid"1147 R"""({ "street_name": "Pennsylvania" })""",1148 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",1149 // "By default, providing additional properties is valid"1150 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",1151 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",1152 },1153 // Failing strings1154 {1155 // Change datatype from number to string1156 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",1157 // Reorder properties1158 R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",1159 }1160 );1161 1162 // Additional properties: false1163 test_schema(1164 "required + optional props each in original order",1165 // Schema1166 R"""({1167 "type": "object",1168 "properties": {1169 "number": { "type": "number" },1170 "street_name": { "type": "string" },1171 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }1172 },1173 "additionalProperties": false1174 })""",1175 // Passing strings1176 {1177 R"""({ "street_name": "Pennsylvania" })""",1178 R"""({ "number": 1600, "street_type":"Avenue"})""",1179 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",1180 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",1181 // Spaces are permitted around enum values1182 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",1183 },1184 // Failing strings1185 {1186 // Reorder properties1187 R"""({ "street_type": "Avenue", "number": 1600 })""",1188 // Add "direction"1189 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",1190 }1191 );1192 1193 test_schema(1194 "required + optional props each in original order",1195 // Schema1196 R"""({1197 "properties": {1198 "b": {"type": "string"},1199 "a": {"type": "string"},1200 "d": {"type": "string"},