Brunobkr/llama.cpp_AlgMor24_github

1import { describe, it, expect, vitest } from "vitest";2import * as entities from "./decode.js";3 4describe("Decode test", () => {5    const testcases = [6        { input: "&amp;amp;", output: "&amp;" },7        { input: "&amp;#38;", output: "&#38;" },8        { input: "&amp;#x26;", output: "&#x26;" },9        { input: "&amp;#X26;", output: "&#X26;" },10        { input: "&#38;#38;", output: "&#38;" },11        { input: "&#x26;#38;", output: "&#38;" },12        { input: "&#X26;#38;", output: "&#38;" },13        { input: "&#x3a;", output: ":" },14        { input: "&#x3A;", output: ":" },15        { input: "&#X3a;", output: ":" },16        { input: "&#X3A;", output: ":" },17        { input: "&#", output: "&#" },18        { input: "&>", output: "&>" },19        { input: "id=770&#anchor", output: "id=770&#anchor" },20    ];21 22    for (const { input, output } of testcases) {23        it(`should XML decode ${input}`, () =>24            expect(entities.decodeXML(input)).toBe(output));25        it(`should HTML decode ${input}`, () =>26            expect(entities.decodeHTML(input)).toBe(output));27    }28 29    it("should HTML decode partial legacy entity", () => {30        expect(entities.decodeHTMLStrict("&timesbar")).toBe("&timesbar");31        expect(entities.decodeHTML("&timesbar")).toBe("×bar");32    });33 34    it("should HTML decode legacy entities according to spec", () =>35        expect(entities.decodeHTML("?&image_uri=1&ℑ=2&image=3")).toBe(36            "?&image_uri=1&ℑ=2&image=3",37        ));38 39    it("should back out of legacy entities", () =>40        expect(entities.decodeHTML("&ampa")).toBe("&a"));41 42    it("should not parse numeric entities in strict mode", () =>43        expect(entities.decodeHTMLStrict("&#55")).toBe("&#55"));44 45    it("should parse &nbsp followed by < (#852)", () =>46        expect(entities.decodeHTML("&nbsp<")).toBe("\u00A0<"));47 48    it("should decode trailing legacy entities", () => {49        expect(entities.decodeHTML("&timesbar;&timesbar")).toBe("⨱×bar");50    });51 52    it("should decode multi-byte entities", () => {53        expect(entities.decodeHTML("&NotGreaterFullEqual;")).toBe("≧̸");54    });55 56    it("should not decode legacy entities followed by text in attribute mode", () => {57        expect(58            entities.decodeHTML("&not", entities.DecodingMode.Attribute),59        ).toBe("¬");60 61        expect(62            entities.decodeHTML("&noti", entities.DecodingMode.Attribute),63        ).toBe("&noti");64 65        expect(66            entities.decodeHTML("&not=", entities.DecodingMode.Attribute),67        ).toBe("&not=");68 69        expect(entities.decodeHTMLAttribute("&notp")).toBe("&notp");70        expect(entities.decodeHTMLAttribute("&notP")).toBe("&notP");71        expect(entities.decodeHTMLAttribute("&not3")).toBe("&not3");72    });73});74 75describe("EntityDecoder", () => {76    it("should decode decimal entities", () => {77        const callback = vitest.fn();78        const decoder = new entities.EntityDecoder(79            entities.htmlDecodeTree,80            callback,81        );82 83        expect(decoder.write("&#5", 1)).toBe(-1);84        expect(decoder.write("8;", 0)).toBe(5);85 86        expect(callback).toHaveBeenCalledTimes(1);87        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 5);88    });89 90    it("should decode hex entities", () => {91        const callback = vitest.fn();92        const decoder = new entities.EntityDecoder(93            entities.htmlDecodeTree,94            callback,95        );96 97        expect(decoder.write("&#x3a;", 1)).toBe(6);98 99        expect(callback).toHaveBeenCalledTimes(1);100        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);101    });102 103    it("should decode named entities", () => {104        const callback = vitest.fn();105        const decoder = new entities.EntityDecoder(106            entities.htmlDecodeTree,107            callback,108        );109 110        expect(decoder.write("&amp;", 1)).toBe(5);111 112        expect(callback).toHaveBeenCalledTimes(1);113        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);114    });115 116    it("should decode legacy entities", () => {117        const callback = vitest.fn();118        const decoder = new entities.EntityDecoder(119            entities.htmlDecodeTree,120            callback,121        );122        decoder.startEntity(entities.DecodingMode.Legacy);123 124        expect(decoder.write("&amp", 1)).toBe(-1);125 126        expect(callback).toHaveBeenCalledTimes(0);127 128        expect(decoder.end()).toBe(4);129 130        expect(callback).toHaveBeenCalledTimes(1);131        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 4);132    });133 134    it("should decode named entity written character by character", () => {135        const callback = vitest.fn();136        const decoder = new entities.EntityDecoder(137            entities.htmlDecodeTree,138            callback,139        );140 141        for (const c of "amp") {142            expect(decoder.write(c, 0)).toBe(-1);143        }144        expect(decoder.write(";", 0)).toBe(5);145 146        expect(callback).toHaveBeenCalledTimes(1);147        expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);148    });149 150    it("should decode numeric entity written character by character", () => {151        const callback = vitest.fn();152        const decoder = new entities.EntityDecoder(153            entities.htmlDecodeTree,154            callback,155        );156 157        for (const c of "#x3a") {158            expect(decoder.write(c, 0)).toBe(-1);159        }160        expect(decoder.write(";", 0)).toBe(6);161 162        expect(callback).toHaveBeenCalledTimes(1);163        expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);164    });165 166    it("should decode hex entities across several chunks", () => {167        const callback = vitest.fn();168        const decoder = new entities.EntityDecoder(169            entities.htmlDecodeTree,170            callback,171        );172 173        for (const chunk of ["#x", "cf", "ff", "d"]) {174            expect(decoder.write(chunk, 0)).toBe(-1);175        }176 177        expect(decoder.write(";", 0)).toBe(9);178        expect(callback).toHaveBeenCalledTimes(1);179        expect(callback).toHaveBeenCalledWith(0xc_ff_fd, 9);180    });181 182    it("should not fail if nothing is written", () => {183        const callback = vitest.fn();184        const decoder = new entities.EntityDecoder(185            entities.htmlDecodeTree,186            callback,187        );188 189        expect(decoder.end()).toBe(0);190        expect(callback).toHaveBeenCalledTimes(0);191    });192 193    describe("errors", () => {194        it("should produce an error for a named entity without a semicolon", () => {195            const errorHandlers = {196                missingSemicolonAfterCharacterReference: vitest.fn(),197                absenceOfDigitsInNumericCharacterReference: vitest.fn(),198                validateNumericCharacterReference: vitest.fn(),199            };200            const callback = vitest.fn();201            const decoder = new entities.EntityDecoder(202                entities.htmlDecodeTree,203                callback,204                errorHandlers,205            );206 207            decoder.startEntity(entities.DecodingMode.Legacy);208            expect(decoder.write("&amp;", 1)).toBe(5);209            expect(callback).toHaveBeenCalledTimes(1);210            expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);211            expect(212                errorHandlers.missingSemicolonAfterCharacterReference,213            ).toHaveBeenCalledTimes(0);214 215            decoder.startEntity(entities.DecodingMode.Legacy);216            expect(decoder.write("&amp", 1)).toBe(-1);217            expect(decoder.end()).toBe(4);218 219            expect(callback).toHaveBeenCalledTimes(2);220            expect(callback).toHaveBeenLastCalledWith("&".charCodeAt(0), 4);221            expect(222                errorHandlers.missingSemicolonAfterCharacterReference,223            ).toHaveBeenCalledTimes(1);224        });225 226        it("should produce an error for a numeric entity without a semicolon", () => {227            const errorHandlers = {228                missingSemicolonAfterCharacterReference: vitest.fn(),229                absenceOfDigitsInNumericCharacterReference: vitest.fn(),230                validateNumericCharacterReference: vitest.fn(),231            };232            const callback = vitest.fn();233            const decoder = new entities.EntityDecoder(234                entities.htmlDecodeTree,235                callback,236                errorHandlers,237            );238 239            decoder.startEntity(entities.DecodingMode.Legacy);240            expect(decoder.write("&#x3a", 1)).toBe(-1);241            expect(decoder.end()).toBe(5);242 243            expect(callback).toHaveBeenCalledTimes(1);244            expect(callback).toHaveBeenCalledWith(0x3a, 5);245            expect(246                errorHandlers.missingSemicolonAfterCharacterReference,247            ).toHaveBeenCalledTimes(1);248            expect(249                errorHandlers.absenceOfDigitsInNumericCharacterReference,250            ).toHaveBeenCalledTimes(0);251            expect(252                errorHandlers.validateNumericCharacterReference,253            ).toHaveBeenCalledTimes(1);254            expect(255                errorHandlers.validateNumericCharacterReference,256            ).toHaveBeenCalledWith(0x3a);257        });258 259        it("should produce an error for numeric entities without digits", () => {260            const errorHandlers = {261                missingSemicolonAfterCharacterReference: vitest.fn(),262                absenceOfDigitsInNumericCharacterReference: vitest.fn(),263                validateNumericCharacterReference: vitest.fn(),264            };265            const callback = vitest.fn();266            const decoder = new entities.EntityDecoder(267                entities.htmlDecodeTree,268                callback,269                errorHandlers,270            );271 272            decoder.startEntity(entities.DecodingMode.Legacy);273            expect(decoder.write("&#", 1)).toBe(-1);274            expect(decoder.end()).toBe(0);275 276            expect(callback).toHaveBeenCalledTimes(0);277            expect(278                errorHandlers.missingSemicolonAfterCharacterReference,279            ).toHaveBeenCalledTimes(0);280            expect(281                errorHandlers.absenceOfDigitsInNumericCharacterReference,282            ).toHaveBeenCalledTimes(1);283            expect(284                errorHandlers.absenceOfDigitsInNumericCharacterReference,285            ).toHaveBeenCalledWith(2);286            expect(287                errorHandlers.validateNumericCharacterReference,288            ).toHaveBeenCalledTimes(0);289        });290 291        it("should produce an error for hex entities without digits", () => {292            const errorHandlers = {293                missingSemicolonAfterCharacterReference: vitest.fn(),294                absenceOfDigitsInNumericCharacterReference: vitest.fn(),295                validateNumericCharacterReference: vitest.fn(),296            };297            const callback = vitest.fn();298            const decoder = new entities.EntityDecoder(299                entities.htmlDecodeTree,300                callback,301                errorHandlers,302            );303 304            decoder.startEntity(entities.DecodingMode.Legacy);305            expect(decoder.write("&#x", 1)).toBe(-1);306            expect(decoder.end()).toBe(0);307 308            expect(callback).toHaveBeenCalledTimes(0);309            expect(310                errorHandlers.missingSemicolonAfterCharacterReference,311            ).toHaveBeenCalledTimes(0);312            expect(313                errorHandlers.absenceOfDigitsInNumericCharacterReference,314            ).toHaveBeenCalledTimes(1);315            expect(316                errorHandlers.validateNumericCharacterReference,317            ).toHaveBeenCalledTimes(0);318        });319    });320});321