Brunobkr/llama.cpp_AlgMor24_github

1#ifdef NDEBUG2#undef NDEBUG3#endif4 5#include "json-schema-to-grammar.h"6 7#include "../src/unicode.h"8#include "../src/llama-grammar.h"9 10#include <nlohmann/json.hpp>11 12#include <cassert>13#include <string>14#include <vector>15 16using json = nlohmann::ordered_json;17 18static llama_grammar * build_grammar_with_root(const std::string & grammar_str, const char * grammar_root) {19    return llama_grammar_init_impl(nullptr, grammar_str.c_str(), grammar_root, false, nullptr, 0, nullptr, 0);20}21 22static llama_grammar * build_grammar(const std::string & grammar_str) {23    return build_grammar_with_root(grammar_str, "root");24}25 26static bool test_build_grammar_fails(const std::string & grammar_str) {27    fprintf(stderr, "⚫ Testing failure for grammar: %s\n", grammar_str.c_str());28    bool grammar_fails = false;29    llama_grammar * grammar = build_grammar(grammar_str);30    if (grammar != nullptr) {31        fprintf(stderr, "  ❌ Expected build failure, but succeeded\n");32    } else {33        grammar_fails = true;34        fprintf(stdout, "  ✅︎\n");35    }36    return grammar_fails;37}38 39struct token_and_piece {40    llama_token token;41    std::string piece;42};43 44// token() encodes a 32-bit ID as 5 bytes: a 0xff marker followed by the ID in big-endian order.45static std::string token(llama_token id) {46    return std::string{47        static_cast<char>(0xff),48        static_cast<char>((id >> 24) & 0xff),49        static_cast<char>((id >> 16) & 0xff),50        static_cast<char>((id >> 8) & 0xff),51        static_cast<char>(id & 0xff)52    };53}54 55// parse_tokens() parses the token encodes above and UTF-8 text.56static std::vector<token_and_piece> parse_tokens(const std::string & input) {57    std::vector<token_and_piece> result;58    result.reserve(input.size());59    size_t offset = 0;60    while (offset < input.size()) {61        try {62            if (static_cast<unsigned char>(input[offset]) == 0xff) {63                if (offset + 5 > input.size()) {64                    throw std::runtime_error("not enough bytes for token id");65                }66                uint32_t val =67                    (static_cast<unsigned char>(input[offset + 1]) << 24) |68                    (static_cast<unsigned char>(input[offset + 2]) << 16) |69                    (static_cast<unsigned char>(input[offset + 3]) << 8)  |70                    (static_cast<unsigned char>(input[offset + 4]));71                auto piece = "<[" + std::to_string(val) + "]>";72                result.push_back({static_cast<llama_token>(val), piece});73                offset += 5;74            } else {75                uint32_t cpt = unicode_cpt_from_utf8(input, offset);76                result.push_back({0, unicode_cpt_to_utf8(cpt)});77            }78        } catch (const std::invalid_argument & /*ex*/) {79            // Silently ignore invalid UTF-8 input to avoid leaking the exception beyond llama_tokenize80            ++offset;81            result.push_back({0, unicode_cpt_to_utf8(0xFFFD)}); // replacement character82        }83    }84    return result;85}86 87static bool match_string(const std::string & input, llama_grammar * grammar) {88    const auto parsed = parse_tokens(input);89 90    auto & stacks_cur = llama_grammar_get_stacks(grammar);91 92    for (const auto & in : parsed) {93        try {94            llama_grammar_accept_token(*grammar, in.token, in.piece);95        } catch (const std::runtime_error & /*e*/) {96            // normally this shouldn't get hit because of llama_grammar_apply97            return false;98        }99 100        if (stacks_cur.empty()) {101            // no stacks means that the grammar failed to match at this point102            return false;103        }104    }105 106    for (const auto & stack : stacks_cur) {107        if (stack.empty()) {108            // An empty stack means that the grammar has been completed109            return true;110        }111    }112 113    return false;114}115 116static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {117    fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());118    fflush(stderr);119 120    auto * grammar = build_grammar(grammar_str);121 122    // Save the original grammar stacks so that we can reset after every new string we want to test123    const llama_grammar_stacks stacks_org = llama_grammar_get_stacks(grammar); // copy124 125    llama_grammar_stacks & stacks_cur = llama_grammar_get_stacks(grammar);126 127    fprintf(stderr, "  🔵 Valid strings:\n");128 129    // Passing strings130    for (const auto & test_string : passing_strings) {131        fprintf(stderr, "    \"%s\" ", test_string.c_str());132        fflush(stderr);133 134        bool matched = match_string(test_string, grammar);135 136        if (!matched) {137            fprintf(stderr, "❌ (failed to match)\n");138 139            // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.140            // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf141            FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");142            if (grammar_file) {143                fprintf(grammar_file, "%s", grammar_str.c_str());144                fclose(grammar_file);145            }146 147            // DEBUG: Write the test string to test-grammar-integration.string.txt148            FILE* string_file = fopen("test-grammar-integration.string.txt", "w");149            if (string_file) {150                fprintf(string_file, "%s", test_string.c_str());151                fclose(string_file);152            }153 154            fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command:     ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");155        } else {156            fprintf(stdout, "✅︎\n");157        }158 159        assert(matched);160 161        // Reset the grammar stacks162        stacks_cur = stacks_org;163    }164 165    fprintf(stderr, "  🟠 Invalid strings:\n");166 167    // Failing strings168    for (const auto & test_string : failing_strings) {169        fprintf(stderr, "    \"%s\" ", test_string.c_str());170        fflush(stderr);171 172        bool matched = match_string(test_string, grammar);173 174        if (matched) {175            fprintf(stderr, "❌ (incorrectly matched)\n");176        } else {177            fprintf(stdout, "✅︎\n");178        }179        assert(!matched);180 181        // Reset the grammar stacks182        stacks_cur = stacks_org;183    }184 185    // Clean up allocated memory186    llama_grammar_free_impl(grammar);187}188static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {189    test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);190}191static void test_schema(const std::string & test_desc, const std::string & schema_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {192    test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str), true), passing_strings, failing_strings);193}194 195static void test_simple_grammar() {196    test_schema(197        "min 0",198        R"""({199            "type": "integer",200            "minimum": 0201        })""",202        // Passing strings203        {204            "0",205            "10",206            "12",207            "10000",208        },209        // Failing strings210        {211            "-1",212            "-10",213            "-10000",214            "-100000000000000000000000000000000",215            "100000000000000000000000000000000",216            "00",217            "01",218            "-0",219        }220    );221    test_schema(222        "min 2",223        // Schema224        R"""({225            "type": "integer",226            "minimum": 2227        })""",228        // Passing strings229        {230            "2",231            "3",232            "4",233            "10",234            "20",235            "1234567890000000",236        },237        // Failing strings238        {239            "0",240            "1",241            "-1",242            "-100",243            "0",244            "1",245            "01",246            "02",247            "12345678900000000",248        }249    );250    test_schema(251        "min 456",252        R"""({253            "type": "integer",254            "minimum": 456255        })""",256        // Passing strings257        {258            "456",259            "4560",260            "457",261            "460",262            "500",263        },264        // Failing strings265        {266            "455",267            "356",268            "50",269            "050",270            "-1",271            "-456",272        }273    );274    test_schema(275        "min -123",276        R"""({277            "type": "integer",278            "minimum": -123279        })""",280        // Passing strings281        {282            "-123",283            "-122",284            "-11",285            "-1",286            "0",287            "1",288            "123",289            "1234",290            "2345",291        },292        // Failing strings293        {294            "-1234",295            "-124",296        }297    );298 299    test_schema(300        "max 9999",301        // Schema302        R"""({303            "type": "integer",304            "maximum": 9999305        })""",306        // Passing strings307        {308            "-99999",309            "0",310            "9999",311        },312        // Failing strings313        {314            "10000",315            "99991",316        }317    );318    test_schema(319        "max -9999",320        // Schema321        R"""({322            "type": "integer",323            "maximum": -9999324        })""",325        // Passing strings326        {327            "-10000",328            "-9999",329        },330        // Failing strings331        {332            "-9998",333            "0",334            "9999",335        }336    );337    test_schema(338        "min 5 max 30",339        // Schema340        R"""({341            "type": "integer",342            "minimum": 5,343            "maximum": 30344        })""",345        // Passing strings346        {347            "5",348            "10",349            "30",350        },351        // Failing strings352        {353            "05",354            "4",355            "-1",356            "31",357            "123",358            "0123",359        }360    );361    test_schema(362        "min 1 max 900719925474091",363        // Schema364        R"""({365            "type": "integer",366            "exclusiveMinimum": 0,367            "maximum": 900719925474091368        })""",369        // Passing strings370        {371            "1",372            "2",373            "10",374            "900719925474090",375            "900719925474091",376        },377        // Failing strings378        {379            "0",380            "01",381            "900719925474092",382            "9007199254740910",383        }384    );385    test_schema(386        "min -1 max 1",387        R"""({388            "type": "integer",389            "minimum": -1,390            "maximum": 1391        })""",392        // Passing strings393        {394            "-1",395            "0",396            "1",397        },398        // Failing strings399        {400            "-11",401            "-10",402            "-2",403            "2",404            "10",405            "11",406        }407    );408    test_schema(409        "min -123 max 42",410        R"""({411            "type": "integer",412            "minimum": -123,413            "maximum": 42414        })""",415        // Passing strings416        {417            "-123",418            "-122",419            "-13",420            "-11",421            "-2",422            "-1",423            "0",424            "1",425            "5",426            "10",427            "39",428            "40",429            "42",430        },431        // Failing strings432        {433            "-0123",434            "-124",435            "-1123",436            "-200",437            "43",438            "123",439            "0123",440        }441    );442    test_schema(443        "exclusive min / max",444        // Schema445        R"""({446            "type": "integer",447            "exclusiveMinimum": 0,448            "exclusiveMaximum": 10000449        })""",450        // Passing strings451        {452            "1",453            "9999",454        },455        // Failing strings456        {457            "0",458            "01",459            "10000",460            "99999",461        }462    );463 464    // Test case for a simple grammar465    test_grammar(466        "simple grammar",467        R"""(468            root ::= expr469            expr ::= term ("+" term)*470            term ::= number471            number ::= [0-9]+)""",472        // Passing strings473        {474            "42",475            "1+2+3+4+5",476            "123+456",477        },478        // Failing strings479        {480            "+",481            "/ 3",482            "1+2+3+4+5+",483            "12a45",484        }485    );486 487    // Test case for a simple grammar with tokens488    test_grammar(489        "simple grammar with tokens",490        R"""(491            root ::= <[10]> content <[11]>492            content ::= (!<[11]>)*)""",493        // Passing strings494        {495            token(10) + "hello world" + token(11),496            token(10) + "text with " + token(12) + " other tokens " + token(13) + " mixed in" + token(11),497            token(10) + token(11),498            token(10) + token(12) + token(13) + token(14) + token(15) + token(11),499            token(10) + "a" + token(11),500        },501        // Failing strings502        {503            token(10) + "missing end token",504            token(10),505            "missing start token" + token(11),506            token(10) + token(11) + token(11),  // double end token507            token(11) + "wrong order" + token(10),508        }509    );510}511 512static void test_complex_grammar() {513    // Test case for a more complex grammar, with both failure strings and success strings514    test_grammar(515        "medium complexity grammar",516        // Grammar517        R"""(518            root ::= expression519            expression ::= term ws (("+"|"-") ws term)*520            term ::= factor ws (("*"|"/") ws factor)*521            factor ::= number | variable | "(" expression ")" | function-call522            number ::= [0-9]+523            variable ::= [a-zA-Z_][a-zA-Z0-9_]*524            function-call ::= variable ws "(" (expression ("," ws expression)*)? ")"525            ws ::= [ \t\n\r]?)""",526        // Passing strings527        {528            "42",529            "1*2*3*4*5",530            "x",531            "x+10",532            "x1+y2",533            "(a+b)*(c-d)",534            "func()",535            "func(x,y+2)",536            "a*(b+c)-d/e",537            "f(g(x),h(y,z))",538            "x + 10",539            "x1 + y2",540            "(a + b) * (c - d)",541            "func()",542            "func(x, y + 2)",543            "a * (b + c) - d / e",544            "f(g(x), h(y, z))",545            "123+456",546            "123*456*789-123/456+789*123",547            "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456"548        },549        // Failing strings550        {551            "+",552            "/ 3x",553            "x + + y",554            "a * / b",555            "func(,)",556            "func(x y)",557            "(a + b",558            "x + y)",559            "a + b * (c - d",560            "42 +",561            "x +",562            "x + 10 +",563            "(a + b) * (c - d",564            "func(",565            "func(x, y + 2",566            "a * (b + c) - d /",567            "f(g(x), h(y, z)",568            "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",569        }570    );571 572    // Test case for a more complex grammar with tokens573    test_grammar(574        "complex grammar with tokens",575        R"""(576            root ::= reasoning+ content tool-call*577            reasoning ::= <[10]> (!<[11]>)* <[11]>578            content ::= <[20]> (!<[21]>)* <[21]>579            tool-call ::= <[12]> name <[13]> args <[14]>580            name ::= (!<[13]>)+581            args ::= (!<[14]>)*)""",582        // Passing strings583        {584            token(10) + "I am thinking" + token(11) + token(20) + "hello world!" + token(21) + token(12) + "search" + token(13) + "query=test" + token(14),585            token(10) + "reasoning 1" + token(11) + token(10) + "reasoning 2" + token(11) + token(20) + token(21) + token(12) + "tool" + token(13) + token(14),586            token(10) + token(11) + token(20) + "content" + token(21),587            token(10) + "think" + token(12) + " nested" + token(11) + token(20) + token(10) + "more content" + token(21) + token(12) + "fn" + token(13) + "x=1,y=2" + token(14) + token(12) + "fn2" + token(13) + token(14),588            token(10) + "reasoning" + token(11) + token(10) + "more" + token(11) + token(10) + "even more" + token(11) + token(20) + "text" + token(21) + token(12) + "a" + token(13) + "b" + token(14) + token(12) + "c" + token(13) + "d" + token(14),589        },590        // Failing strings591        {592            token(20) + "content only" + token(21),593            token(10) + "no closing reasoning",594            token(10) + token(11) + token(20) + "no closing content",595            token(10) + token(11) + token(20) + token(21) + token(12) + "incomplete tool",596            token(10) + token(11) + token(11) + token(20) + token(21),597        }598    );599}600 601static void test_special_chars() {602    // A collection of tests to exercise special characters such as "."603    test_grammar(604        "special characters",605        // Grammar606        R"""(607            root ::= ... "abc" ...608            )""",609        // Passing strings610        {611            "abcabcabc",612            "aaaabcccc",613            // NOTE: Also ensures that multi-byte characters still count as a single character614            "🔵🟠✅abc❌🟠🔵"615        },616        // Failing strings617        {618            "aaabcccc",619            "aaaaabcccc",620            "aaaabccc",621            "aaaabccccc",622            "🔵🟠✅❌abc❌✅🟠🔵",623            "🔵🟠abc🟠🔵"624        }625    );626}627 628static void test_quantifiers() {629    // A collection of tests to exercise * + and ? quantifiers630 631    test_grammar(632        "* quantifier",633        // Grammar634        R"""(root ::= "a"*)""",635        // Passing strings636        {637            "",638            "a",639            "aaaaa",640            "aaaaaaaaaaaaaaaaaa",641            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"642        },643        // Failing strings644        {645            "b",646            "ab",647            "aab",648            "ba",649            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"650        }651    );652    test_grammar(653        "+ quantifier",654        // Grammar655        R"""(root ::= "a"+)""",656        // Passing strings657        {658            "a",659            "aaaaa",660            "aaaaaaaaaaaaaaaaaa",661            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"662        },663        // Failing strings664        {665            "",666            "b",667            "ab",668            "aab",669            "ba",670            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"671        }672    );673    test_grammar(674        "? quantifier",675        // Grammar676        R"""(root ::= "a"?)""",677        // Passing strings678        {679            "",680            "a"681        },682        // Failing strings683        {684            "b",685            "ab",686            "aa",687            "ba",688        }689    );690    test_grammar(691        "mixed quantifiers",692        // Grammar693        R"""(694            root ::= cons+ vowel* cons? (vowel cons)*695            vowel ::= [aeiouy]696            cons ::= [bcdfghjklmnpqrstvwxyz]697            )""",698        // Passing strings699        {700            "yes",701            "no",702            "noyes",703            "crwth",704            "four",705            "bryyyy",706        },707        // Failing strings708        {709            "yess",710            "yesno",711            "forty",712            "catyyy",713        }714    );715    test_grammar(716        "simple exact repetition",717        // Grammar718        R"""(719            root ::= [ab]{4}720        )""",721        // Passing strings722        {723            "aaaa",724            "bbbb",725            "abab",726        },727        // Failing strings728        {729            "a",730            "b",731            "aaaaa",732        }733    );734    test_grammar(735        "simple min repetition",736        // Grammar737        R"""(738            root ::= [ab]{4,}739        )""",740        // Passing strings741        {742            "aaaa",743            "aaaaab",744            "bbbb",745            "ababab",746        },747        // Failing strings748        {749            "",750            "aba",751        }752    );753    test_grammar(754        "simple max repetition",755        // Grammar756        R"""(757            root ::= [ab]{0,4}758        )""",759        // Passing strings760        {761            "",762            "a",763            "aa",764            "aaa",765            "aaab",766        },767        // Failing strings768        {769            "aaaaa",770        }771    );772    test_grammar(773        "min / max repetition",774        // Grammar775        R"""(776            root ::= ("0x" [A-F0-9]{2} " "?){3,5}777        )""",778        // Passing strings779        {780            "0xFF 0x12 0xAB",781            "0xFF 0x12 0xAB 0x00 0x00",782        },783        // Failing strings784        {785            "",786            "0xFF",787            "0xFF 0x12",788            "0xFF 0x12 0xAB 0x00 0x00 0x00",789        }790    );791    test_grammar(792        "segfault",793        // Grammar794        R"""(795            root ::= ( [x]* )*796        )""",797        // Passing strings798        {799            "",800            "x",801            "xx"802        },803        // Failing strings804        {805            "y",806            "yy"807        }808    );809}810 811static void test_failure_missing_root() {812    fprintf(stderr, "⚫ Testing missing root node:\n");813    // Test case for a grammar that is missing a root rule814    const std::string grammar_str = R"""(815        rot ::= expr816        expr ::= term ("+" term)*817        term ::= number818        number ::= [0-9]+)""";819 820    llama_grammar_parser parsed_grammar;821    parsed_grammar.parse(grammar_str.c_str());822 823    // Ensure we parsed correctly824    assert(!parsed_grammar.rules.empty());825 826    // Ensure we do NOT have a root node827    assert(parsed_grammar.symbol_ids.find("root") == parsed_grammar.symbol_ids.end());828    fprintf(stderr, "  ✅︎ Passed\n");829}830 831static void test_failure_missing_reference() {832    fprintf(stderr, "⚫ Testing missing reference node:\n");833 834    // Test case for a grammar that is missing a referenced rule835    const std::string grammar_str =836        R"""(root ::= expr837        expr ::= term ("+" term)*838        term ::= numero839        number ::= [0-9]+)""";840 841    fprintf(stderr, "    Expected error:  ");842 843    llama_grammar_parser parsed_grammar;844    parsed_grammar.parse(grammar_str.c_str());845 846    // Ensure we did NOT parsed correctly847    assert(parsed_grammar.rules.empty());848 849    fprintf(stderr, "    End of expected error.\n");850    fprintf(stderr, "  ✅︎ Passed\n");851}852 853static void test_failure_left_recursion() {854    fprintf(stderr, "⚫ Testing left recursion detection:\n");855 856    // Test simple left recursion detection857    const std::string simple_str = R"""(root ::= "a" | root "a")""";858    assert(test_build_grammar_fails(simple_str));859 860    // Test more complicated left recursion detection861    const std::string medium_str = R"""(862        root ::= asdf863        asdf ::= "a" | asdf "a"864        )""";865    assert(test_build_grammar_fails(medium_str));866 867    // Test even more complicated left recursion detection868    const std::string hard_str = R"""(869        root ::= asdf870        asdf ::= "a" | foo "b"871        foo ::= "c" | asdf "d" | "e")""";872    assert(test_build_grammar_fails(hard_str));873 874    // Test yet even more complicated left recursion detection875    const std::string hardest_str = R"""(876        root ::= asdf877        asdf ::= "a" | foo "b"878        foo ::= "c" | empty asdf "d" | "e"879        empty ::= "blah" | )""";880    assert(test_build_grammar_fails(hardest_str));881 882    fprintf(stderr, "  ✅︎ Passed\n");883}884 885static void test_failure_missing_root_symbol() {886    fprintf(stderr, "⚫ Testing missing root symbol:\n");887 888    const std::string grammar_str = R"""(889        root ::= "foobar"890    )""";891 892    llama_grammar * failure_result = build_grammar_with_root(grammar_str, "nonexistent");893    assert(failure_result == nullptr);894 895    fprintf(stderr, "  ✅︎ Passed\n");896}897 898static void test_custom_root_symbol_check() {899    fprintf(stderr, "⚫ Testing custom root symbol check:\n");900 901    const std::string custom_root_grammar_str = R"""(902        foobar ::= "foobar"903    )""";904 905    llama_grammar * failure_result = build_grammar_with_root(custom_root_grammar_str, "root");906    assert(failure_result == nullptr);907 908    llama_grammar * success_result = build_grammar_with_root(custom_root_grammar_str, "foobar");909    assert(success_result != nullptr);910    llama_grammar_free_impl(success_result);911 912    fprintf(stderr, "  ✅︎ Passed\n");913}914 915static void test_json_schema() {916    // Note that this is similar to the regular grammar tests,917    //  but we convert each json schema to a grammar before parsing.918    // Otherwise, this test structure is the same.919 920    test_schema(921        "empty schema (object)",922        // Schema923        R"""(924            {}925        )""",926        // Passing strings927        {928            R"""({})""",929            R"""({"foo": "bar"})""",930        },931        // Failing strings932        {933            "",934            "[]",935            "null",936            R"""("")""",937            "true",938        }939    );940 941    test_schema(942        "exotic formats (list)",943        // Schema944        R"""({945            "items": [946                { "format": "date" },947                { "format": "uuid" },948                { "format": "time" },949                { "format": "date-time" }950            ]951        })""",952        // Passing strings953        {954            // "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?955            // "[]", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?956            R"""(["2012-04-23", "12345678-1234-1234-1234-1234567890ab", "18:25:43.511Z", "2012-04-23T18:25:43.511Z"])""",957            //R"""(["2012-04-23","12345678-1234-1234-1234-1234567890ab"])""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?958            //R"""({"foo": "bar"})""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?959        },960        // Failing strings961        {962            R"""(["foo", "bar"])""",963            R"""(["12345678-1234-1234-1234-1234567890ab"])""",964        }965    );966 967    test_schema(968        "string",969        // Schema970        R"""({971            "type": "string"972        })""",973        // Passing strings974        {975            R"""("foo")""",976            R"""("bar")""",977            R"""("")""",978        },979        // Failing strings980        {981            R"""({})""",982            R"""("foo": "bar")""",983        }984    );985 986    test_schema(987        "string w/ min length 1",988        // Schema989        R"""({990            "type": "string",991            "minLength": 1992        })""",993        // Passing strings994        {995            R"""("foo")""",996            R"""("bar")""",997        },998        // Failing strings999        {1000            R"""("")""",1001            R"""({})""",1002            R"""("foo": "bar")""",1003        }1004    );1005 1006    test_schema(1007        "string w/ min length 3",1008        // Schema1009        R"""({1010                "type": "string",1011                "minLength": 31012        })""",1013        // Passing strings1014        {1015            R"""("foo")""",1016            R"""("bar")""",1017            R"""("foobar")""",1018        },1019        // Failing strings1020        {1021            R"""("")""",1022            R"""("f")""",1023            R"""("fo")""",1024        }1025    );1026 1027    test_schema(1028        "string w/ max length",1029        // Schema1030        R"""({1031            "type": "string",1032            "maxLength": 31033        })""",1034        // Passing strings1035        {1036            R"""("foo")""",1037            R"""("bar")""",1038            R"""("")""",1039            R"""("f")""",1040            R"""("fo")""",1041        },1042        // Failing strings1043        {1044            R"""("foobar")""",1045        }1046    );1047 1048    test_schema(1049        "string w/ min & max length",1050        // Schema1051        R"""({1052            "type": "string",1053            "minLength": 1,1054            "maxLength": 41055        })""",1056        // Passing strings1057        {1058            R"""("foo")""",1059            R"""("bar")""",1060            R"""("f")""",1061            R"""("barf")""",1062        },1063        // Failing strings1064        {1065            R"""("")""",1066            R"""("barfo")""",1067            R"""("foobar")""",1068        }1069    );1070 1071    test_schema(1072        "boolean",1073        // Schema1074        R"""({1075            "type": "boolean"1076        })""",1077        // Passing strings1078        {1079            "true",1080            "false",1081        },1082        // Failing strings1083        {1084            R"""("")""",1085            R"""("true")""",1086            R"""(True)""",1087            R"""(FALSE)""",1088        }1089    );1090 1091    test_schema(1092        "integer",1093        // Schema1094        R"""({1095            "type": "integer"1096        })""",1097        // Passing strings1098        {1099            R"""(0)""",1100            R"""(12345)""",1101            R"""(1234567890123456)""",1102        },1103        // Failing strings1104        {1105            R"""()""",1106            R"""(01)""",1107            R"""(007)""",1108            R"""(12345678901234567  )""",1109        }1110    );1111 1112    test_schema(1113        "string const",1114        // Schema1115        R"""({1116            "const": "foo"1117        })""",1118        // Passing strings1119        {1120            R"""("foo")""",1121        },1122        // Failing strings1123        {1124            R"""(foo)""",1125            R"""("bar")""",1126        }1127    );1128 1129    test_schema(1130        "non-string const",1131        // Schema1132        R"""({1133            "const": true1134        })""",1135        // Passing strings1136        {1137            R"""(true)""",1138        },1139        // Failing strings1140        {1141            R"""()""",1142            R"""(foo)""",1143            R"""("true")""",1144        }1145    );1146 1147    test_schema(1148        "non-string const",1149        // Schema1150        R"""({1151            "enum": ["red", "amber", "green", null, 42, ["foo"]]1152        })""",1153        // Passing strings1154        {1155            R"""("red")""",1156            R"""(null)""",1157            R"""(42)""",1158            R"""(["foo"])""",1159        },1160        // Failing strings1161        {1162            R"""()""",1163            R"""(420)""",1164            R"""(true)""",1165            R"""(foo)""",1166        }1167    );1168 1169    test_schema(1170        "simple pattern",1171        // Schema1172        R"""({1173            "pattern": "^[a-zA-Z0-9_-]*$"1174        })""",1175        // Passing strings1176        {1177            R"""("")""",1178            R"""("He_llo-12")""",1179        },1180        // Failing strings1181        {1182            R"""("!")""",1183            R"""("Hello World")""",1184        }1185    );1186 1187    test_schema(1188        "pattern with escapes",1189        // Schema1190        R"""({1191            "pattern": "^a\\^\\$\\.\\[\\]\\(\\)\\|\\{\\}\\*\\+\\?b$"1192        })""",1193        // Passing strings1194        {1195            R"""("a^$.[]()|{}*+?b")""",1196        },1197        // Failing strings1198        {1199            R"""("ab")""",1200        }

Showing the first 1,200 of 1494 lines. Download the file for the rest.