Brunobkr/llama.cpp_AlgMor24_github

1import { readFileSync } from "node:fs";2import { describe, it, expect } from "vitest";3import * as entities from "./index.js";4import legacy from "../maps/legacy.json" with { type: "json" };5 6const levels = ["xml", "entities"];7 8describe("Documents", () => {9    const levelDocuments = levels10        .map((name) => new URL(`../maps/${name}.json`, import.meta.url))11        .map((url) => JSON.parse(readFileSync(url, "utf8")))12        .map((document, index) => [index, document]);13 14    for (const [level, document] of levelDocuments) {15        describe("Decode", () => {16            it(levels[level], () => {17                for (const entity of Object.keys(document)) {18                    for (let l = level; l < levels.length; l++) {19                        expect(entities.decode(`&${entity};`, l)).toBe(20                            document[entity],21                        );22                        expect(23                            entities.decode(`&${entity};`, { level: l }),24                        ).toBe(document[entity]);25                    }26                }27            });28        });29 30        describe("Decode strict", () => {31            it(levels[level], () => {32                for (const entity of Object.keys(document)) {33                    for (let l = level; l < levels.length; l++) {34                        expect(entities.decodeStrict(`&${entity};`, l)).toBe(35                            document[entity],36                        );37                        expect(38                            entities.decode(`&${entity};`, {39                                level: l,40                                mode: entities.DecodingMode.Strict,41                            }),42                        ).toBe(document[entity]);43                    }44                }45            });46        });47 48        describe("Encode", () => {49            it(levels[level], () => {50                for (const entity of Object.keys(document)) {51                    for (let l = level; l < levels.length; l++) {52                        const encoded = entities.encode(document[entity], l);53                        const decoded = entities.decode(encoded, l);54                        expect(decoded).toBe(document[entity]);55                    }56                }57            });58 59            it("should only encode non-ASCII values if asked", () =>60                expect(61                    entities.encode("Great #'s of 🎁", {62                        level,63                        mode: entities.EncodingMode.ASCII,64                    }),65                ).toBe("Great #&apos;s of &#x1f381;"));66        });67    }68 69    describe("Legacy", () => {70        const legacyMap: Record<string, string> = legacy;71        it("should decode", () => {72            for (const entity of Object.keys(legacyMap)) {73                expect(entities.decodeHTML(`&${entity}`)).toBe(74                    legacyMap[entity],75                );76                expect(77                    entities.decodeStrict(`&${entity}`, {78                        level: entities.EntityLevel.HTML,79                        mode: entities.DecodingMode.Legacy,80                    }),81                ).toBe(legacyMap[entity]);82            }83        });84    });85});86 87const astral = [88    ["1d306", "\uD834\uDF06"],89    ["1d11e", "\uD834\uDD1E"],90];91 92const astralSpecial = [93    ["80", "\u20AC"],94    ["110000", "\uFFFD"],95];96 97describe("Astral entities", () => {98    for (const [c, value] of astral) {99        it(`should decode ${value}`, () =>100            expect(entities.decode(`&#x${c};`)).toBe(value));101 102        it(`should encode ${value}`, () =>103            expect(entities.encode(value)).toBe(`&#x${c};`));104 105        it(`should escape ${value}`, () =>106            expect(entities.escape(value)).toBe(`&#x${c};`));107    }108 109    for (const [c, value] of astralSpecial) {110        it(`should decode special \\u${c}`, () =>111            expect(entities.decode(`&#x${c};`)).toBe(value));112    }113});114 115describe("Escape", () => {116    it("should always decode ASCII chars", () => {117        for (let index = 0; index < 0x7f; index++) {118            const c = String.fromCharCode(index);119            expect(entities.decodeXML(entities.escape(c))).toBe(c);120        }121    });122 123    it("should keep UTF8 characters", () =>124        expect(entities.escapeUTF8('ß < "ü"')).toBe(`ß &lt; &quot;ü&quot;`));125});126