Add initial 11ty website structure

This commit is contained in:
Marc Froehlich committed 2026-07-06 01:04:53 +02:00
1 parent 62b6bbdcbc
commit 80214bf42d
1378 files changed
+204162

No files matched your search

+81
View File
@@ -0,0 +1,81 @@
// Adapted from https://github.com/mathiasbynens/he/blob/36afe179392226cf1b6ccdb16ebbb7a5a844d93a/src/he.js#L106-L134
const decodeMap = new Map([
[0, 65_533],
// C1 Unicode control character reference replacements
[128, 8364],
[130, 8218],
[131, 402],
[132, 8222],
[133, 8230],
[134, 8224],
[135, 8225],
[136, 710],
[137, 8240],
[138, 352],
[139, 8249],
[140, 338],
[142, 381],
[145, 8216],
[146, 8217],
[147, 8220],
[148, 8221],
[149, 8226],
[150, 8211],
[151, 8212],
[152, 732],
[153, 8482],
[154, 353],
[155, 8250],
[156, 339],
[158, 382],
[159, 376],
]);
/**
* Polyfill for `String.fromCodePoint`. It is used to create a string from a Unicode code point.
*/
export const fromCodePoint: (...codePoints: number[]) => string =
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, n/no-unsupported-features/es-builtins
String.fromCodePoint ??
function (codePoint: number): string {
let output = "";
if (codePoint > 0xff_ff) {
codePoint -= 0x1_00_00;
output += String.fromCharCode(
((codePoint >>> 10) & 0x3_ff) | 0xd8_00,
);
codePoint = 0xdc_00 | (codePoint & 0x3_ff);
}
output += String.fromCharCode(codePoint);
return output;
};
/**
* Replace the given code point with a replacement character if it is a
* surrogate or is outside the valid range. Otherwise return the code
* point unchanged.
*/
export function replaceCodePoint(codePoint: number): number {
if (
(codePoint >= 0xd8_00 && codePoint <= 0xdf_ff) ||
codePoint > 0x10_ff_ff
) {
return 0xff_fd;
}
return decodeMap.get(codePoint) ?? codePoint;
}
/**
* Replace the code point if relevant, then convert it to a string.
*
* @deprecated Use `fromCodePoint(replaceCodePoint(codePoint))` instead.
* @param codePoint The code point to decode.
* @returns The decoded code point.
*/
export function decodeCodePoint(codePoint: number): string {
return fromCodePoint(replaceCodePoint(codePoint));
}
+320
View File
@@ -0,0 +1,320 @@
import { describe, it, expect, vitest } from "vitest";
import * as entities from "./decode.js";
describe("Decode test", () => {
const testcases = [
{ input: "&amp;amp;", output: "&amp;" },
{ input: "&amp;#38;", output: "&#38;" },
{ input: "&amp;#x26;", output: "&#x26;" },
{ input: "&amp;#X26;", output: "&#X26;" },
{ input: "&#38;#38;", output: "&#38;" },
{ input: "&#x26;#38;", output: "&#38;" },
{ input: "&#X26;#38;", output: "&#38;" },
{ input: "&#x3a;", output: ":" },
{ input: "&#x3A;", output: ":" },
{ input: "&#X3a;", output: ":" },
{ input: "&#X3A;", output: ":" },
{ input: "&#", output: "&#" },
{ input: "&>", output: "&>" },
{ input: "id=770&#anchor", output: "id=770&#anchor" },
];
for (const { input, output } of testcases) {
it(`should XML decode ${input}`, () =>
expect(entities.decodeXML(input)).toBe(output));
it(`should HTML decode ${input}`, () =>
expect(entities.decodeHTML(input)).toBe(output));
}
it("should HTML decode partial legacy entity", () => {
expect(entities.decodeHTMLStrict("&timesbar")).toBe("&timesbar");
expect(entities.decodeHTML("&timesbar")).toBe("×bar");
});
it("should HTML decode legacy entities according to spec", () =>
expect(entities.decodeHTML("?&image_uri=1&ℑ=2&image=3")).toBe(
"?&image_uri=1&ℑ=2&image=3",
));
it("should back out of legacy entities", () =>
expect(entities.decodeHTML("&ampa")).toBe("&a"));
it("should not parse numeric entities in strict mode", () =>
expect(entities.decodeHTMLStrict("&#55")).toBe("&#55"));
it("should parse &nbsp followed by < (#852)", () =>
expect(entities.decodeHTML("&nbsp<")).toBe("\u00A0<"));
it("should decode trailing legacy entities", () => {
expect(entities.decodeHTML("&timesbar;&timesbar")).toBe("⨱×bar");
});
it("should decode multi-byte entities", () => {
expect(entities.decodeHTML("&NotGreaterFullEqual;")).toBe("≧̸");
});
it("should not decode legacy entities followed by text in attribute mode", () => {
expect(
entities.decodeHTML("&not", entities.DecodingMode.Attribute),
).toBe("¬");
expect(
entities.decodeHTML("&noti", entities.DecodingMode.Attribute),
).toBe("&noti");
expect(
entities.decodeHTML("&not=", entities.DecodingMode.Attribute),
).toBe("&not=");
expect(entities.decodeHTMLAttribute("&notp")).toBe("&notp");
expect(entities.decodeHTMLAttribute("&notP")).toBe("&notP");
expect(entities.decodeHTMLAttribute("&not3")).toBe("&not3");
});
});
describe("EntityDecoder", () => {
it("should decode decimal entities", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
expect(decoder.write("&#5", 1)).toBe(-1);
expect(decoder.write("8;", 0)).toBe(5);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 5);
});
it("should decode hex entities", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
expect(decoder.write("&#x3a;", 1)).toBe(6);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);
});
it("should decode named entities", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
expect(decoder.write("&amp;", 1)).toBe(5);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
});
it("should decode legacy entities", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&amp", 1)).toBe(-1);
expect(callback).toHaveBeenCalledTimes(0);
expect(decoder.end()).toBe(4);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 4);
});
it("should decode named entity written character by character", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
for (const c of "amp") {
expect(decoder.write(c, 0)).toBe(-1);
}
expect(decoder.write(";", 0)).toBe(5);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
});
it("should decode numeric entity written character by character", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
for (const c of "#x3a") {
expect(decoder.write(c, 0)).toBe(-1);
}
expect(decoder.write(";", 0)).toBe(6);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(":".charCodeAt(0), 6);
});
it("should decode hex entities across several chunks", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
for (const chunk of ["#x", "cf", "ff", "d"]) {
expect(decoder.write(chunk, 0)).toBe(-1);
}
expect(decoder.write(";", 0)).toBe(9);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(0xc_ff_fd, 9);
});
it("should not fail if nothing is written", () => {
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
);
expect(decoder.end()).toBe(0);
expect(callback).toHaveBeenCalledTimes(0);
});
describe("errors", () => {
it("should produce an error for a named entity without a semicolon", () => {
const errorHandlers = {
missingSemicolonAfterCharacterReference: vitest.fn(),
absenceOfDigitsInNumericCharacterReference: vitest.fn(),
validateNumericCharacterReference: vitest.fn(),
};
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
errorHandlers,
);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&amp;", 1)).toBe(5);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith("&".charCodeAt(0), 5);
expect(
errorHandlers.missingSemicolonAfterCharacterReference,
).toHaveBeenCalledTimes(0);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&amp", 1)).toBe(-1);
expect(decoder.end()).toBe(4);
expect(callback).toHaveBeenCalledTimes(2);
expect(callback).toHaveBeenLastCalledWith("&".charCodeAt(0), 4);
expect(
errorHandlers.missingSemicolonAfterCharacterReference,
).toHaveBeenCalledTimes(1);
});
it("should produce an error for a numeric entity without a semicolon", () => {
const errorHandlers = {
missingSemicolonAfterCharacterReference: vitest.fn(),
absenceOfDigitsInNumericCharacterReference: vitest.fn(),
validateNumericCharacterReference: vitest.fn(),
};
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
errorHandlers,
);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&#x3a", 1)).toBe(-1);
expect(decoder.end()).toBe(5);
expect(callback).toHaveBeenCalledTimes(1);
expect(callback).toHaveBeenCalledWith(0x3a, 5);
expect(
errorHandlers.missingSemicolonAfterCharacterReference,
).toHaveBeenCalledTimes(1);
expect(
errorHandlers.absenceOfDigitsInNumericCharacterReference,
).toHaveBeenCalledTimes(0);
expect(
errorHandlers.validateNumericCharacterReference,
).toHaveBeenCalledTimes(1);
expect(
errorHandlers.validateNumericCharacterReference,
).toHaveBeenCalledWith(0x3a);
});
it("should produce an error for numeric entities without digits", () => {
const errorHandlers = {
missingSemicolonAfterCharacterReference: vitest.fn(),
absenceOfDigitsInNumericCharacterReference: vitest.fn(),
validateNumericCharacterReference: vitest.fn(),
};
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
errorHandlers,
);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&#", 1)).toBe(-1);
expect(decoder.end()).toBe(0);
expect(callback).toHaveBeenCalledTimes(0);
expect(
errorHandlers.missingSemicolonAfterCharacterReference,
).toHaveBeenCalledTimes(0);
expect(
errorHandlers.absenceOfDigitsInNumericCharacterReference,
).toHaveBeenCalledTimes(1);
expect(
errorHandlers.absenceOfDigitsInNumericCharacterReference,
).toHaveBeenCalledWith(2);
expect(
errorHandlers.validateNumericCharacterReference,
).toHaveBeenCalledTimes(0);
});
it("should produce an error for hex entities without digits", () => {
const errorHandlers = {
missingSemicolonAfterCharacterReference: vitest.fn(),
absenceOfDigitsInNumericCharacterReference: vitest.fn(),
validateNumericCharacterReference: vitest.fn(),
};
const callback = vitest.fn();
const decoder = new entities.EntityDecoder(
entities.htmlDecodeTree,
callback,
errorHandlers,
);
decoder.startEntity(entities.DecodingMode.Legacy);
expect(decoder.write("&#x", 1)).toBe(-1);
expect(decoder.end()).toBe(0);
expect(callback).toHaveBeenCalledTimes(0);
expect(
errorHandlers.missingSemicolonAfterCharacterReference,
).toHaveBeenCalledTimes(0);
expect(
errorHandlers.absenceOfDigitsInNumericCharacterReference,
).toHaveBeenCalledTimes(1);
expect(
errorHandlers.validateNumericCharacterReference,
).toHaveBeenCalledTimes(0);
});
});
});
+620
View File
@@ -0,0 +1,620 @@
import { htmlDecodeTree } from "./generated/decode-data-html.js";
import { xmlDecodeTree } from "./generated/decode-data-xml.js";
import { replaceCodePoint, fromCodePoint } from "./decode-codepoint.js";
const enum CharCodes {
NUM = 35, // "#"
SEMI = 59, // ";"
EQUALS = 61, // "="
ZERO = 48, // "0"
NINE = 57, // "9"
LOWER_A = 97, // "a"
LOWER_F = 102, // "f"
LOWER_X = 120, // "x"
LOWER_Z = 122, // "z"
UPPER_A = 65, // "A"
UPPER_F = 70, // "F"
UPPER_Z = 90, // "Z"
}
/** Bit that needs to be set to convert an upper case ASCII character to lower case */
const TO_LOWER_BIT = 0b10_0000;
export enum BinTrieFlags {
VALUE_LENGTH = 0b1100_0000_0000_0000,
BRANCH_LENGTH = 0b0011_1111_1000_0000,
JUMP_TABLE = 0b0000_0000_0111_1111,
}
function isNumber(code: number): boolean {
return code >= CharCodes.ZERO && code <= CharCodes.NINE;
}
function isHexadecimalCharacter(code: number): boolean {
return (
(code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_F) ||
(code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_F)
);
}
function isAsciiAlphaNumeric(code: number): boolean {
return (
(code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_Z) ||
(code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_Z) ||
isNumber(code)
);
}
/**
* Checks if the given character is a valid end character for an entity in an attribute.
*
* Attribute values that aren't terminated properly aren't parsed, and shouldn't lead to a parser error.
* See the example in https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state
*/
function isEntityInAttributeInvalidEnd(code: number): boolean {
return code === CharCodes.EQUALS || isAsciiAlphaNumeric(code);
}
const enum EntityDecoderState {
EntityStart,
NumericStart,
NumericDecimal,
NumericHex,
NamedEntity,
}
export enum DecodingMode {
/** Entities in text nodes that can end with any character. */
Legacy = 0,
/** Only allow entities terminated with a semicolon. */
Strict = 1,
/** Entities in attributes have limitations on ending characters. */
Attribute = 2,
}
/**
* Producers for character reference errors as defined in the HTML spec.
*/
export interface EntityErrorProducer {
missingSemicolonAfterCharacterReference(): void;
absenceOfDigitsInNumericCharacterReference(
consumedCharacters: number,
): void;
validateNumericCharacterReference(code: number): void;
}
/**
* Token decoder with support of writing partial entities.
*/
export class EntityDecoder {
constructor(
/** The tree used to decode entities. */
private readonly decodeTree: Uint16Array,
/**
* The function that is called when a codepoint is decoded.
*
* For multi-byte named entities, this will be called multiple times,
* with the second codepoint, and the same `consumed` value.
*
* @param codepoint The decoded codepoint.
* @param consumed The number of bytes consumed by the decoder.
*/
private readonly emitCodePoint: (cp: number, consumed: number) => void,
/** An object that is used to produce errors. */
private readonly errors?: EntityErrorProducer | undefined,
) {}
/** The current state of the decoder. */
private state = EntityDecoderState.EntityStart;
/** Characters that were consumed while parsing an entity. */
private consumed = 1;
/**
* The result of the entity.
*
* Either the result index of a numeric entity, or the codepoint of a
* numeric entity.
*/
private result = 0;
/** The current index in the decode tree. */
private treeIndex = 0;
/** The number of characters that were consumed in excess. */
private excess = 1;
/** The mode in which the decoder is operating. */
private decodeMode = DecodingMode.Strict;
/** Resets the instance to make it reusable. */
startEntity(decodeMode: DecodingMode): void {
this.decodeMode = decodeMode;
this.state = EntityDecoderState.EntityStart;
this.result = 0;
this.treeIndex = 0;
this.excess = 1;
this.consumed = 1;
}
/**
* Write an entity to the decoder. This can be called multiple times with partial entities.
* If the entity is incomplete, the decoder will return -1.
*
* Mirrors the implementation of `getDecoder`, but with the ability to stop decoding if the
* entity is incomplete, and resume when the next string is written.
*
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The offset at which the entity begins. Should be 0 if this is not the first call.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
write(input: string, offset: number): number {
switch (this.state) {
case EntityDecoderState.EntityStart: {
if (input.charCodeAt(offset) === CharCodes.NUM) {
this.state = EntityDecoderState.NumericStart;
this.consumed += 1;
return this.stateNumericStart(input, offset + 1);
}
this.state = EntityDecoderState.NamedEntity;
return this.stateNamedEntity(input, offset);
}
case EntityDecoderState.NumericStart: {
return this.stateNumericStart(input, offset);
}
case EntityDecoderState.NumericDecimal: {
return this.stateNumericDecimal(input, offset);
}
case EntityDecoderState.NumericHex: {
return this.stateNumericHex(input, offset);
}
case EntityDecoderState.NamedEntity: {
return this.stateNamedEntity(input, offset);
}
}
}
/**
* Switches between the numeric decimal and hexadecimal states.
*
* Equivalent to the `Numeric character reference state` in the HTML spec.
*
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNumericStart(input: string, offset: number): number {
if (offset >= input.length) {
return -1;
}
if ((input.charCodeAt(offset) | TO_LOWER_BIT) === CharCodes.LOWER_X) {
this.state = EntityDecoderState.NumericHex;
this.consumed += 1;
return this.stateNumericHex(input, offset + 1);
}
this.state = EntityDecoderState.NumericDecimal;
return this.stateNumericDecimal(input, offset);
}
private addToNumericResult(
input: string,
start: number,
end: number,
base: number,
): void {
if (start !== end) {
const digitCount = end - start;
this.result =
this.result * Math.pow(base, digitCount) +
Number.parseInt(input.substr(start, digitCount), base);
this.consumed += digitCount;
}
}
/**
* Parses a hexadecimal numeric entity.
*
* Equivalent to the `Hexademical character reference state` in the HTML spec.
*
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNumericHex(input: string, offset: number): number {
const startIndex = offset;
while (offset < input.length) {
const char = input.charCodeAt(offset);
if (isNumber(char) || isHexadecimalCharacter(char)) {
offset += 1;
} else {
this.addToNumericResult(input, startIndex, offset, 16);
return this.emitNumericEntity(char, 3);
}
}
this.addToNumericResult(input, startIndex, offset, 16);
return -1;
}
/**
* Parses a decimal numeric entity.
*
* Equivalent to the `Decimal character reference state` in the HTML spec.
*
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNumericDecimal(input: string, offset: number): number {
const startIndex = offset;
while (offset < input.length) {
const char = input.charCodeAt(offset);
if (isNumber(char)) {
offset += 1;
} else {
this.addToNumericResult(input, startIndex, offset, 10);
return this.emitNumericEntity(char, 2);
}
}
this.addToNumericResult(input, startIndex, offset, 10);
return -1;
}
/**
* Validate and emit a numeric entity.
*
* Implements the logic from the `Hexademical character reference start
* state` and `Numeric character reference end state` in the HTML spec.
*
* @param lastCp The last code point of the entity. Used to see if the
* entity was terminated with a semicolon.
* @param expectedLength The minimum number of characters that should be
* consumed. Used to validate that at least one digit
* was consumed.
* @returns The number of characters that were consumed.
*/
private emitNumericEntity(lastCp: number, expectedLength: number): number {
// Ensure we consumed at least one digit.
if (this.consumed <= expectedLength) {
this.errors?.absenceOfDigitsInNumericCharacterReference(
this.consumed,
);
return 0;
}
// Figure out if this is a legit end of the entity
if (lastCp === CharCodes.SEMI) {
this.consumed += 1;
} else if (this.decodeMode === DecodingMode.Strict) {
return 0;
}
this.emitCodePoint(replaceCodePoint(this.result), this.consumed);
if (this.errors) {
if (lastCp !== CharCodes.SEMI) {
this.errors.missingSemicolonAfterCharacterReference();
}
this.errors.validateNumericCharacterReference(this.result);
}
return this.consumed;
}
/**
* Parses a named entity.
*
* Equivalent to the `Named character reference state` in the HTML spec.
*
* @param input The string containing the entity (or a continuation of the entity).
* @param offset The current offset.
* @returns The number of characters that were consumed, or -1 if the entity is incomplete.
*/
private stateNamedEntity(input: string, offset: number): number {
const { decodeTree } = this;
let current = decodeTree[this.treeIndex];
// The mask is the number of bytes of the value, including the current byte.
let valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;
for (; offset < input.length; offset++, this.excess++) {
const char = input.charCodeAt(offset);
this.treeIndex = determineBranch(
decodeTree,
current,
this.treeIndex + Math.max(1, valueLength),
char,
);
if (this.treeIndex < 0) {
return this.result === 0 ||
// If we are parsing an attribute
(this.decodeMode === DecodingMode.Attribute &&
// We shouldn't have consumed any characters after the entity,
(valueLength === 0 ||
// And there should be no invalid characters.
isEntityInAttributeInvalidEnd(char)))
? 0
: this.emitNotTerminatedNamedEntity();
}
current = decodeTree[this.treeIndex];
valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;
// If the branch is a value, store it and continue
if (valueLength !== 0) {
// If the entity is terminated by a semicolon, we are done.
if (char === CharCodes.SEMI) {
return this.emitNamedEntityData(
this.treeIndex,
valueLength,
this.consumed + this.excess,
);
}
// If we encounter a non-terminated (legacy) entity while parsing strictly, then ignore it.
if (this.decodeMode !== DecodingMode.Strict) {
this.result = this.treeIndex;
this.consumed += this.excess;
this.excess = 0;
}
}
}
return -1;
}
/**
* Emit a named entity that was not terminated with a semicolon.
*
* @returns The number of characters consumed.
*/
private emitNotTerminatedNamedEntity(): number {
const { result, decodeTree } = this;
const valueLength =
(decodeTree[result] & BinTrieFlags.VALUE_LENGTH) >> 14;
this.emitNamedEntityData(result, valueLength, this.consumed);
this.errors?.missingSemicolonAfterCharacterReference();
return this.consumed;
}
/**
* Emit a named entity.
*
* @param result The index of the entity in the decode tree.
* @param valueLength The number of bytes in the entity.
* @param consumed The number of characters consumed.
*
* @returns The number of characters consumed.
*/
private emitNamedEntityData(
result: number,
valueLength: number,
consumed: number,
): number {
const { decodeTree } = this;
this.emitCodePoint(
valueLength === 1
? decodeTree[result] & ~BinTrieFlags.VALUE_LENGTH
: decodeTree[result + 1],
consumed,
);
if (valueLength === 3) {
// For multi-byte values, we need to emit the second byte.
this.emitCodePoint(decodeTree[result + 2], consumed);
}
return consumed;
}
/**
* Signal to the parser that the end of the input was reached.
*
* Remaining data will be emitted and relevant errors will be produced.
*
* @returns The number of characters consumed.
*/
end(): number {
switch (this.state) {
case EntityDecoderState.NamedEntity: {
// Emit a named entity if we have one.
return this.result !== 0 &&
(this.decodeMode !== DecodingMode.Attribute ||
this.result === this.treeIndex)
? this.emitNotTerminatedNamedEntity()
: 0;
}
// Otherwise, emit a numeric entity if we have one.
case EntityDecoderState.NumericDecimal: {
return this.emitNumericEntity(0, 2);
}
case EntityDecoderState.NumericHex: {
return this.emitNumericEntity(0, 3);
}
case EntityDecoderState.NumericStart: {
this.errors?.absenceOfDigitsInNumericCharacterReference(
this.consumed,
);
return 0;
}
case EntityDecoderState.EntityStart: {
// Return 0 if we have no entity.
return 0;
}
}
}
}
/**
* Creates a function that decodes entities in a string.
*
* @param decodeTree The decode tree.
* @returns A function that decodes entities in a string.
*/
function getDecoder(decodeTree: Uint16Array) {
let returnValue = "";
const decoder = new EntityDecoder(
decodeTree,
(data) => (returnValue += fromCodePoint(data)),
);
return function decodeWithTrie(
input: string,
decodeMode: DecodingMode,
): string {
let lastIndex = 0;
let offset = 0;
while ((offset = input.indexOf("&", offset)) >= 0) {
returnValue += input.slice(lastIndex, offset);
decoder.startEntity(decodeMode);
const length = decoder.write(
input,
// Skip the "&"
offset + 1,
);
if (length < 0) {
lastIndex = offset + decoder.end();
break;
}
lastIndex = offset + length;
// If `length` is 0, skip the current `&` and continue.
offset = length === 0 ? lastIndex + 1 : lastIndex;
}
const result = returnValue + input.slice(lastIndex);
// Make sure we don't keep a reference to the final string.
returnValue = "";
return result;
};
}
/**
* Determines the branch of the current node that is taken given the current
* character. This function is used to traverse the trie.
*
* @param decodeTree The trie.
* @param current The current node.
* @param nodeIdx The index right after the current node and its value.
* @param char The current character.
* @returns The index of the next node, or -1 if no branch is taken.
*/
export function determineBranch(
decodeTree: Uint16Array,
current: number,
nodeIndex: number,
char: number,
): number {
const branchCount = (current & BinTrieFlags.BRANCH_LENGTH) >> 7;
const jumpOffset = current & BinTrieFlags.JUMP_TABLE;
// Case 1: Single branch encoded in jump offset
if (branchCount === 0) {
return jumpOffset !== 0 && char === jumpOffset ? nodeIndex : -1;
}
// Case 2: Multiple branches encoded in jump table
if (jumpOffset) {
const value = char - jumpOffset;
return value < 0 || value >= branchCount
? -1
: decodeTree[nodeIndex + value] - 1;
}
// Case 3: Multiple branches encoded in dictionary
// Binary search for the character.
let lo = nodeIndex;
let hi = lo + branchCount - 1;
while (lo <= hi) {
const mid = (lo + hi) >>> 1;
const midValue = decodeTree[mid];
if (midValue < char) {
lo = mid + 1;
} else if (midValue > char) {
hi = mid - 1;
} else {
return decodeTree[mid + branchCount];
}
}
return -1;
}
const htmlDecoder = /* #__PURE__ */ getDecoder(htmlDecodeTree);
const xmlDecoder = /* #__PURE__ */ getDecoder(xmlDecodeTree);
/**
* Decodes an HTML string.
*
* @param htmlString The string to decode.
* @param mode The decoding mode.
* @returns The decoded string.
*/
export function decodeHTML(
htmlString: string,
mode: DecodingMode = DecodingMode.Legacy,
): string {
return htmlDecoder(htmlString, mode);
}
/**
* Decodes an HTML string in an attribute.
*
* @param htmlAttribute The string to decode.
* @returns The decoded string.
*/
export function decodeHTMLAttribute(htmlAttribute: string): string {
return htmlDecoder(htmlAttribute, DecodingMode.Attribute);
}
/**
* Decodes an HTML string, requiring all entities to be terminated by a semicolon.
*
* @param htmlString The string to decode.
* @returns The decoded string.
*/
export function decodeHTMLStrict(htmlString: string): string {
return htmlDecoder(htmlString, DecodingMode.Strict);
}
/**
* Decodes an XML string, requiring all entities to be terminated by a semicolon.
*
* @param xmlString The string to decode.
* @returns The decoded string.
*/
export function decodeXML(xmlString: string): string {
return xmlDecoder(xmlString, DecodingMode.Strict);
}
// Re-export for use by eg. htmlparser2
export { htmlDecodeTree } from "./generated/decode-data-html.js";
export { xmlDecodeTree } from "./generated/decode-data-xml.js";
export {
decodeCodePoint,
replaceCodePoint,
fromCodePoint,
} from "./decode-codepoint.js";
+78
View File
@@ -0,0 +1,78 @@
import { describe, it, expect } from "vitest";
import * as entities from "./index.js";
describe("Encode->decode test", () => {
const testcases = [
{
input: "asdf & ÿ ü '",
xml: "asdf &amp; &#xff; &#xfc; &apos;",
html: "asdf &amp; &yuml; &uuml; &apos;",
},
{
input: "&#38;",
xml: "&amp;#38;",
html: "&amp;&num;38&semi;",
},
];
for (const { input, xml, html } of testcases) {
const encodedXML = entities.encodeXML(input);
it(`should XML encode ${input}`, () => expect(encodedXML).toBe(xml));
it(`should default to XML encode ${input}`, () =>
expect(entities.encode(input)).toBe(xml));
it(`should XML decode ${encodedXML}`, () =>
expect(entities.decodeXML(encodedXML)).toBe(input));
it(`should default to XML encode ${encodedXML}`, () =>
expect(entities.decode(encodedXML)).toBe(input));
it(`should default strict to XML encode ${encodedXML}`, () =>
expect(entities.decodeStrict(encodedXML)).toBe(input));
const encodedHTML5 = entities.encodeHTML5(input);
it(`should HTML5 encode ${input}`, () =>
expect(encodedHTML5).toBe(html));
it(`should HTML5 decode ${encodedHTML5}`, () =>
expect(entities.decodeHTML(encodedHTML5)).toBe(input));
it("should encode emojis", () =>
expect(entities.encodeHTML5("😄🍾🥳💥😇")).toBe(
"&#x1f604;&#x1f37e;&#x1f973;&#x1f4a5;&#x1f607;",
));
}
it("should encode data URIs (issue #16)", () => {
const data =
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAALAAABAAEAAAIBRAA7";
expect(entities.decode(entities.encode(data))).toBe(data);
});
it("should HTML encode all ASCII characters", () => {
for (let index = 0; index < 128; index++) {
const char = String.fromCharCode(index);
const encoded = entities.encodeHTML(char);
const decoded = entities.decodeHTML(encoded);
expect(decoded).toBe(char);
}
});
it("should encode trailing parts of entities", () =>
expect(entities.encodeHTML("\uD835")).toBe("&#xd835;"));
it("should encode surrogate pair with first surrogate equivalent of entity, without corresponding entity", () =>
expect(entities.encodeHTML("\u{1D4A4}")).toBe("&#x1d4a4;"));
});
describe("encodeNonAsciiHTML", () => {
it("should encode all non-ASCII characters", () =>
expect(entities.encodeNonAsciiHTML("<test> #123! übermaßen")).toBe(
"&lt;test&gt; #123! &uuml;berma&szlig;en",
));
it("should encode emojis", () =>
expect(entities.encodeNonAsciiHTML("😄🍾🥳💥😇")).toBe(
"&#x1f604;&#x1f37e;&#x1f973;&#x1f4a5;&#x1f607;",
));
it("should encode chars above surrogates", () =>
expect(entities.encodeNonAsciiHTML("♒️♓️♈️♉️♊️♋️♌️♍️♎️♏️♐️♑️")).toBe(
"&#x2652;&#xfe0f;&#x2653;&#xfe0f;&#x2648;&#xfe0f;&#x2649;&#xfe0f;&#x264a;&#xfe0f;&#x264b;&#xfe0f;&#x264c;&#xfe0f;&#x264d;&#xfe0f;&#x264e;&#xfe0f;&#x264f;&#xfe0f;&#x2650;&#xfe0f;&#x2651;&#xfe0f;",
));
});
+77
View File
@@ -0,0 +1,77 @@
import { htmlTrie } from "./generated/encode-html.js";
import { xmlReplacer, getCodePoint } from "./escape.js";
const htmlReplacer = /[\t\n\f!-,./:-@[-`{-}\u0080-\uFFFF]/g;
/**
* Encodes all characters in the input using HTML entities. This includes
* characters that are valid ASCII characters in HTML documents, such as `#`.
*
* To get a more compact output, consider using the `encodeNonAsciiHTML`
* function, which will only encode characters that are not valid in HTML
* documents, as well as non-ASCII characters.
*
* If a character has no equivalent entity, a numeric hexadecimal reference
* (eg. `&#xfc;`) will be used.
*/
export function encodeHTML(input: string): string {
return encodeHTMLTrieRe(htmlReplacer, input);
}
/**
* Encodes all non-ASCII characters, as well as characters not valid in HTML
* documents using HTML entities. This function will not encode characters that
* are valid in HTML documents, such as `#`.
*
* If a character has no equivalent entity, a numeric hexadecimal reference
* (eg. `&#xfc;`) will be used.
*/
export function encodeNonAsciiHTML(input: string): string {
return encodeHTMLTrieRe(xmlReplacer, input);
}
function encodeHTMLTrieRe(regExp: RegExp, input: string): string {
let returnValue = "";
let lastIndex = 0;
let match;
while ((match = regExp.exec(input)) !== null) {
const { index } = match;
returnValue += input.substring(lastIndex, index);
const char = input.charCodeAt(index);
let next = htmlTrie.get(char);
if (typeof next === "object") {
// We are in a branch. Try to match the next char.
if (index + 1 < input.length) {
const nextChar = input.charCodeAt(index + 1);
const value =
typeof next.n === "number"
? next.n === nextChar
? next.o
: undefined
: next.n.get(nextChar);
if (value !== undefined) {
returnValue += value;
lastIndex = regExp.lastIndex += 1;
continue;
}
}
next = next.v;
}
// We might have a tree node without a value; skip and use a numeric entity.
if (next === undefined) {
const cp = getCodePoint(input, index);
returnValue += `&#x${cp.toString(16)};`;
// Increase by 1 if we have a surrogate pair
lastIndex = regExp.lastIndex += Number(cp !== char);
} else {
returnValue += next;
lastIndex = index + 1;
}
}
return returnValue + input.substr(lastIndex);
}
+14
View File
@@ -0,0 +1,14 @@
import { describe, it, expect } from "vitest";
import * as entities from "./index.js";
describe("escape HTML", () => {
it("should escape HTML attribute values", () =>
expect(entities.escapeAttribute('<a " attr > & value \u00A0!')).toBe(
"<a &quot; attr > &amp; value &nbsp;!",
));
it("should escape HTML text", () =>
expect(entities.escapeText('<a " text > & value \u00A0!')).toBe(
'&lt;a " text &gt; &amp; value &nbsp;!',
));
});
+148
View File
@@ -0,0 +1,148 @@
export const xmlReplacer: RegExp = /["$&'<>\u0080-\uFFFF]/g;
const xmlCodeMap = new Map([
[34, "&quot;"],
[38, "&amp;"],
[39, "&apos;"],
[60, "&lt;"],
[62, "&gt;"],
]);
// For compatibility with node < 4, we wrap `codePointAt`
export const getCodePoint: (c: string, index: number) => number =
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
String.prototype.codePointAt == null
? (c: string, index: number): number =>
(c.charCodeAt(index) & 0xfc_00) === 0xd8_00
? (c.charCodeAt(index) - 0xd8_00) * 0x4_00 +
c.charCodeAt(index + 1) -
0xdc_00 +
0x1_00_00
: c.charCodeAt(index)
: // http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
(input: string, index: number): number => input.codePointAt(index)!;
/**
* Encodes all non-ASCII characters, as well as characters not valid in XML
* documents using XML entities.
*
* If a character has no equivalent entity, a
* numeric hexadecimal reference (eg. `&#xfc;`) will be used.
*/
export function encodeXML(input: string): string {
let returnValue = "";
let lastIndex = 0;
let match;
while ((match = xmlReplacer.exec(input)) !== null) {
const { index } = match;
const char = input.charCodeAt(index);
const next = xmlCodeMap.get(char);
if (next === undefined) {
returnValue += `${input.substring(lastIndex, index)}&#x${getCodePoint(
input,
index,
).toString(16)};`;
// Increase by 1 if we have a surrogate pair
lastIndex = xmlReplacer.lastIndex += Number(
(char & 0xfc_00) === 0xd8_00,
);
} else {
returnValue += input.substring(lastIndex, index) + next;
lastIndex = index + 1;
}
}
return returnValue + input.substr(lastIndex);
}
/**
* Encodes all non-ASCII characters, as well as characters not valid in XML
* documents using numeric hexadecimal reference (eg. `&#xfc;`).
*
* Have a look at `escapeUTF8` if you want a more concise output at the expense
* of reduced transportability.
*
* @param data String to escape.
*/
export const escape: typeof encodeXML = encodeXML;
/**
* Creates a function that escapes all characters matched by the given regular
* expression using the given map of characters to escape to their entities.
*
* @param regex Regular expression to match characters to escape.
* @param map Map of characters to escape to their entities.
*
* @returns Function that escapes all characters matched by the given regular
* expression using the given map of characters to escape to their entities.
*/
function getEscaper(
regex: RegExp,
map: Map<number, string>,
): (data: string) => string {
return function escape(data: string): string {
let match;
let lastIndex = 0;
let result = "";
while ((match = regex.exec(data))) {
if (lastIndex !== match.index) {
result += data.substring(lastIndex, match.index);
}
// We know that this character will be in the map.
result += map.get(match[0].charCodeAt(0))!;
// Every match will be of length 1
lastIndex = match.index + 1;
}
return result + data.substring(lastIndex);
};
}
/**
* Encodes all characters not valid in XML documents using XML entities.
*
* Note that the output will be character-set dependent.
*
* @param data String to escape.
*/
export const escapeUTF8: (data: string) => string = /* #__PURE__ */ getEscaper(
/["&'<>]/g,
xmlCodeMap,
);
/**
* Encodes all characters that have to be escaped in HTML attributes,
* following {@link https://html.spec.whatwg.org/multipage/parsing.html#escapingString}.
*
* @param data String to escape.
*/
export const escapeAttribute: (data: string) => string =
/* #__PURE__ */ getEscaper(
/["&\u00A0]/g,
new Map([
[34, "&quot;"],
[38, "&amp;"],
[160, "&nbsp;"],
]),
);
/**
* Encodes all characters that have to be escaped in HTML text,
* following {@link https://html.spec.whatwg.org/multipage/parsing.html#escapingString}.
*
* @param data String to escape.
*/
export const escapeText: (data: string) => string = /* #__PURE__ */ getEscaper(
/[&<>\u00A0]/g,
new Map([
[38, "&amp;"],
[60, "&lt;"],
[62, "&gt;"],
[160, "&nbsp;"],
]),
);
+10
View File
@@ -0,0 +1,10 @@
{
"rules": {
"multiline-comment-style": 0,
"capitalized-comments": 0,
"unicorn/escape-case": 0,
"unicorn/no-hex-escape": 0,
"unicorn/numeric-separators-style": 0,
"unicorn/prefer-spread": 0
}
}
+8
View File
@@ -0,0 +1,8 @@
// Generated using scripts/write-decode-map.ts
export const htmlDecodeTree: Uint16Array = /* #__PURE__ */ new Uint16Array(
// prettier-ignore
/* #__PURE__ */ "\u1d41<\xd5\u0131\u028a\u049d\u057b\u05d0\u0675\u06de\u07a2\u07d6\u080f\u0a4a\u0a91\u0da1\u0e6d\u0f09\u0f26\u10ca\u1228\u12e1\u1415\u149d\u14c3\u14df\u1525\0\0\0\0\0\0\u156b\u16cd\u198d\u1c12\u1ddd\u1f7e\u2060\u21b0\u228d\u23c0\u23fb\u2442\u2824\u2912\u2d08\u2e48\u2fce\u3016\u32ba\u3639\u37ac\u38fe\u3a28\u3a71\u3ae0\u3b2e\u0800EMabcfglmnoprstu\\bfms\x7f\x84\x8b\x90\x95\x98\xa6\xb3\xb9\xc8\xcflig\u803b\xc6\u40c6P\u803b&\u4026cute\u803b\xc1\u40c1reve;\u4102\u0100iyx}rc\u803b\xc2\u40c2;\u4410r;\uc000\ud835\udd04rave\u803b\xc0\u40c0pha;\u4391acr;\u4100d;\u6a53\u0100gp\x9d\xa1on;\u4104f;\uc000\ud835\udd38plyFunction;\u6061ing\u803b\xc5\u40c5\u0100cs\xbe\xc3r;\uc000\ud835\udc9cign;\u6254ilde\u803b\xc3\u40c3ml\u803b\xc4\u40c4\u0400aceforsu\xe5\xfb\xfe\u0117\u011c\u0122\u0127\u012a\u0100cr\xea\xf2kslash;\u6216\u0176\xf6\xf8;\u6ae7ed;\u6306y;\u4411\u0180crt\u0105\u010b\u0114ause;\u6235noullis;\u612ca;\u4392r;\uc000\ud835\udd05pf;\uc000\ud835\udd39eve;\u42d8c\xf2\u0113mpeq;\u624e\u0700HOacdefhilorsu\u014d\u0151\u0156\u0180\u019e\u01a2\u01b5\u01b7\u01ba\u01dc\u0215\u0273\u0278\u027ecy;\u4427PY\u803b\xa9\u40a9\u0180cpy\u015d\u0162\u017aute;\u4106\u0100;i\u0167\u0168\u62d2talDifferentialD;\u6145leys;\u612d\u0200aeio\u0189\u018e\u0194\u0198ron;\u410cdil\u803b\xc7\u40c7rc;\u4108nint;\u6230ot;\u410a\u0100dn\u01a7\u01adilla;\u40b8terDot;\u40b7\xf2\u017fi;\u43a7rcle\u0200DMPT\u01c7\u01cb\u01d1\u01d6ot;\u6299inus;\u6296lus;\u6295imes;\u6297o\u0100cs\u01e2\u01f8kwiseContourIntegral;\u6232eCurly\u0100DQ\u0203\u020foubleQuote;\u601duote;\u6019\u0200lnpu\u021e\u0228\u0247\u0255on\u0100;e\u0225\u0226\u6237;\u6a74\u0180git\u022f\u0236\u023aruent;\u6261nt;\u622fourIntegral;\u622e\u0100fr\u024c\u024e;\u6102oduct;\u6210nterClockwiseContourIntegral;\u6233oss;\u6a2fcr;\uc000\ud835\udc9ep\u0100;C\u0284\u0285\u62d3ap;\u624d\u0580DJSZacefios\u02a0\u02ac\u02b0\u02b4\u02b8\u02cb\u02d7\u02e1\u02e6\u0333\u048d\u0100;o\u0179\u02a5trahd;\u6911cy;\u4402cy;\u4405cy;\u440f\u0180grs\u02bf\u02c4\u02c7ger;\u6021r;\u61a1hv;\u6ae4\u0100ay\u02d0\u02d5ron;\u410e;\u4414l\u0100;t\u02dd\u02de\u6207a;\u4394r;\uc000\ud835\udd07\u0100af\u02eb\u0327\u0100cm\u02f0\u0322ritical\u0200ADGT\u0300\u0306\u0316\u031ccute;\u40b4o\u0174\u030b\u030d;\u42d9bleAcute;\u42ddrave;\u4060ilde;\u42dcond;\u62c4ferentialD;\u6146\u0470\u033d\0\0\0\u0342\u0354\0\u0405f;\uc000\ud835\udd3b\u0180;DE\u0348\u0349\u034d\u40a8ot;\u60dcqual;\u6250ble\u0300CDLRUV\u0363\u0372\u0382\u03cf\u03e2\u03f8ontourIntegra\xec\u0239o\u0274\u0379\0\0\u037b\xbb\u0349nArrow;\u61d3\u0100eo\u0387\u03a4ft\u0180ART\u0390\u0396\u03a1rrow;\u61d0ightArrow;\u61d4e\xe5\u02cang\u0100LR\u03ab\u03c4eft\u0100AR\u03b3\u03b9rrow;\u67f8ightArrow;\u67faightArrow;\u67f9ight\u0100AT\u03d8\u03derrow;\u61d2ee;\u62a8p\u0241\u03e9\0\0\u03efrrow;\u61d1ownArrow;\u61d5erticalBar;\u6225n\u0300ABLRTa\u0412\u042a\u0430\u045e\u047f\u037crrow\u0180;BU\u041d\u041e\u0422\u6193ar;\u6913pArrow;\u61f5reve;\u4311eft\u02d2\u043a\0\u0446\0\u0450ightVector;\u6950eeVector;\u695eector\u0100;B\u0459\u045a\u61bdar;\u6956ight\u01d4\u0467\0\u0471eeVector;\u695fector\u0100;B\u047a\u047b\u61c1ar;\u6957ee\u0100;A\u0486\u0487\u62a4rrow;\u61a7\u0100ct\u0492\u0497r;\uc000\ud835\udc9frok;\u4110\u0800NTacdfglmopqstux\u04bd\u04c0\u04c4\u04cb\u04de\u04e2\u04e7\u04ee\u04f5\u0521\u052f\u0536\u0552\u055d\u0560\u0565G;\u414aH\u803b\xd0\u40d0cute\u803b\xc9\u40c9\u0180aiy\u04d2\u04d7\u04dcron;\u411arc\u803b\xca\u40ca;\u442dot;\u4116r;\uc000\ud835\udd08rave\u803b\xc8\u40c8ement;\u6208\u0100ap\u04fa\u04fecr;\u4112ty\u0253\u0506\0\0\u0512mallSquare;\u65fberySmallSquare;\u65ab\u0100gp\u0526\u052aon;\u4118f;\uc000\ud835\udd3csilon;\u4395u\u0100ai\u053c\u0549l\u0100;T\u0542\u0543\u6a75ilde;\u6242librium;\u61cc\u0100ci\u0557\u055ar;\u6130m;\u6a73a;\u4397ml\u803b\xcb\u40cb\u0100ip\u056a\u056fsts;\u6203onentialE;\u6147\u0280cfios\u0585\u0588\u058d\u05b2\u05ccy;\u4424r;\uc000\ud835\udd09lled\u0253\u0597\0\0\u05a3mallSquare;\u65fcerySmallSquare;\u65aa\u0370\u05ba\0\u05bf\0\0\u05c4f;\uc000\ud835\udd3dAll;\u6200riertrf;\u6131c\xf2\u05cb\u0600JTabcdfgorst\u05e8\u05ec\u05ef\u05fa\u0600\u0612\u0616\u061b\u061d\u0623\u066c\u0672cy;\u4403\u803b>\u403emma\u0100;d\u05f7\u05f8\u4393;\u43dcreve;\u411e\u0180eiy\u0607\u060c\u0610dil;\u4122rc;\u411c;\u4413ot;\u4120r;\uc000\ud835\udd0a;\u62d9pf;\uc000\ud835\udd3eeater\u0300EFGLST\u0635\u0644\u064e\u0656\u065b\u0666qual\u0100;L\u063e\u063f\u6265ess;\u62dbullEqual;\u6267reater;\u6aa2ess;\u6277lantEqual;\u6a7eilde;\u6273cr;\uc000\ud835\udca2;\u626b\u0400Aacfiosu\u0685\u068b\u0696\u069b\u069e\u06aa\u06be\u06caRDcy;\u442a\u0100ct\u0690\u0694ek;\u42c7;\u405eirc;\u4124r;\u610clbertSpace;\u610b\u01f0\u06af\0\u06b2f;\u610dizontalLine;\u6500\u0100ct\u06c3\u06c5\xf2\u06a9rok;\u4126mp\u0144\u06d0\u06d8ownHum\xf0\u012fqual;\u624f\u0700EJOacdfgmnostu\u06fa\u06fe\u0703\u0707\u070e\u071a\u071e\u0721\u0728\u0744\u0778\u078b\u078f\u0795cy;\u4415lig;\u4132cy;\u4401cute\u803b\xcd\u40cd\u0100iy\u0713\u0718rc\u803b\xce\u40ce;\u4418ot;\u4130r;\uLine truncated
.split("")
.map((c) => c.charCodeAt(0)),
);
+8
View File
@@ -0,0 +1,8 @@
// Generated using scripts/write-decode-map.ts
export const xmlDecodeTree: Uint16Array = /* #__PURE__ */ new Uint16Array(
// prettier-ignore
/* #__PURE__ */ "\u0200aglq\t\x15\x18\x1b\u026d\x0f\0\0\x12p;\u4026os;\u4027t;\u403et;\u403cuot;\u4022"
.split("")
.map((c) => c.charCodeAt(0)),
);
+17
View File
@@ -0,0 +1,17 @@
// Generated using scripts/write-encode-map.ts
type EncodeTrieNode =
| string
| { v?: string; n: number | Map<number, EncodeTrieNode>; o?: string };
function restoreDiff<T extends ReadonlyArray<[number, EncodeTrieNode]>>(
array: T,
): T {
for (let index = 1; index < array.length; index++) {
array[index][0] += array[index - 1][0] + 1;
}
return array;
}
// prettier-ignore
export const htmlTrie: Map<number,EncodeTrieNode> = /* #__PURE__ */ new Map<number,EncodeTrieNode>(/* #__PURE__ */restoreDiff([[9,"&Tab;"],[0,"&NewLine;"],[22,"&excl;"],[0,"&quot;"],[0,"&num;"],[0,"&dollar;"],[0,"&percnt;"],[0,"&amp;"],[0,"&apos;"],[0,"&lpar;"],[0,"&rpar;"],[0,"&ast;"],[0,"&plus;"],[0,"&comma;"],[1,"&period;"],[0,"&sol;"],[10,"&colon;"],[0,"&semi;"],[0,{v:"&lt;",n:8402,o:"&nvlt;"}],[0,{v:"&equals;",n:8421,o:"&bne;"}],[0,{v:"&gt;",n:8402,o:"&nvgt;"}],[0,"&quest;"],[0,"&commat;"],[26,"&lbrack;"],[0,"&bsol;"],[0,"&rbrack;"],[0,"&Hat;"],[0,"&lowbar;"],[0,"&DiacriticalGrave;"],[5,{n:106,o:"&fjlig;"}],[20,"&lbrace;"],[0,"&verbar;"],[0,"&rbrace;"],[34,"&nbsp;"],[0,"&iexcl;"],[0,"&cent;"],[0,"&pound;"],[0,"&curren;"],[0,"&yen;"],[0,"&brvbar;"],[0,"&sect;"],[0,"&die;"],[0,"&copy;"],[0,"&ordf;"],[0,"&laquo;"],[0,"&not;"],[0,"&shy;"],[0,"&circledR;"],[0,"&macr;"],[0,"&deg;"],[0,"&PlusMinus;"],[0,"&sup2;"],[0,"&sup3;"],[0,"&acute;"],[0,"&micro;"],[0,"&para;"],[0,"&centerdot;"],[0,"&cedil;"],[0,"&sup1;"],[0,"&ordm;"],[0,"&raquo;"],[0,"&frac14;"],[0,"&frac12;"],[0,"&frac34;"],[0,"&iquest;"],[0,"&Agrave;"],[0,"&Aacute;"],[0,"&Acirc;"],[0,"&Atilde;"],[0,"&Auml;"],[0,"&angst;"],[0,"&AElig;"],[0,"&Ccedil;"],[0,"&Egrave;"],[0,"&Eacute;"],[0,"&Ecirc;"],[0,"&Euml;"],[0,"&Igrave;"],[0,"&Iacute;"],[0,"&Icirc;"],[0,"&Iuml;"],[0,"&ETH;"],[0,"&Ntilde;"],[0,"&Ograve;"],[0,"&Oacute;"],[0,"&Ocirc;"],[0,"&Otilde;"],[0,"&Ouml;"],[0,"&times;"],[0,"&Oslash;"],[0,"&Ugrave;"],[0,"&Uacute;"],[0,"&Ucirc;"],[0,"&Uuml;"],[0,"&Yacute;"],[0,"&THORN;"],[0,"&szlig;"],[0,"&agrave;"],[0,"&aacute;"],[0,"&acirc;"],[0,"&atilde;"],[0,"&auml;"],[0,"&aring;"],[0,"&aelig;"],[0,"&ccedil;"],[0,"&egrave;"],[0,"&eacute;"],[0,"&ecirc;"],[0,"&euml;"],[0,"&igrave;"],[0,"&iacute;"],[0,"&icirc;"],[0,"&iuml;"],[0,"&eth;"],[0,"&ntilde;"],[0,"&ograve;"],[0,"&oacute;"],[0,"&ocirc;"],[0,"&otilde;"],[0,"&ouml;"],[0,"&div;"],[0,"&oslash;"],[0,"&ugrave;"],[0,"&uacute;"],[0,"&ucirc;"],[0,"&uuml;"],[0,"&yacute;"],[0,"&thorn;"],[0,"&yuml;"],[0,"&Amacr;"],[0,"&amacr;"],[0,"&Abreve;"],[0,"&abreve;"],[0,"&Aogon;"],[0,"&aogon;"],[0,"&Cacute;"],[0,"&cacute;"],[0,"&Ccirc;"],[0,"&ccirc;"],[0,"&Cdot;"],[0,"&cdot;"],[0,"&Ccaron;"],[0,"&ccaron;"],[0,"&Dcaron;"],[0,"&dcaron;"],[0,"&Dstrok;"],[0,"&dstrok;"],[0,"&Emacr;"],[0,"&emacr;"],[2,"&Edot;"],[0,"&edot;"],[0,"&Eogon;"],[0,"&eogon;"],[0,"&Ecaron;"],[0,"&ecaron;"],[0,"&Gcirc;"],[0,"&gcirc;"],[0,"&Gbreve;"],[0,"&gbreve;"],[0,"&Gdot;"],[0,"&gdot;"],[0,"&Gcedil;"],[1,"&Hcirc;"],[0,"&hcirc;"],[0,"&Hstrok;"],[0,"&hstrok;"],[0,"&Itilde;"],[0,"&itilde;"],[0,"&Imacr;"],[0,"&imacr;"],[2,"&Iogon;"],[0,"&iogon;"],[0,"&Idot;"],[0,"&imath;"],[0,"&IJlig;"],[0,"&ijlig;"],[0,"&Jcirc;"],[0,"&jcirc;"],[0,"&Kcedil;"],[0,"&kcedil;"],[0,"&kgreen;"],[0,"&Lacute;"],[0,"&lacute;"],[0,"&Lcedil;"],[0,"&lcedil;"],[0,"&Lcaron;"],[0,"&lcaron;"],[0,"&Lmidot;"],[0,"&lmidot;"],[0,"&Lstrok;"],[0,"&lstrok;"],[0,"&Nacute;"],[0,"&nacute;"],[0,"&Ncedil;"],[0,"&ncedil;"],[0,"&Ncaron;"],[0,"&ncaron;"],[0,"&napos;"],[0,"&ENG;"],[0,"&eng;"],[0,"&Omacr;"],[0,"&omacr;"],[2,"&Odblac;"],[0,"&odblac;"],[0,"&OElig;"],[0,"&oelig;"],[0,"&Racute;"],[0,"&racute;"],[0,"&Rcedil;"],[0,"&rcedil;"],[0,"&Rcaron;"],[0,"&rcaron;"],[0,"&Sacute;"],[0,"&sacute;"],[0,"&Scirc;"],[0,"&scirc;"],[0,"&Scedil;"],[0,"&scedil;"],[0,"&Scaron;"],[0,"&scaron;"],[0,"&Tcedil;"],[0,"&tcedil;"],[0,"&Tcaron;"],[0,"&tcaron;"],[0,"&Tstrok;"],[0,"&tstrok;"],[0,"&Utilde;"],[0,"&utilde;"],[0,"&Umacr;"],[0,"&umacr;"],[0,"&Ubreve;"],[0,"&ubreve;"],[0,"&Uring;"],[0,"&uring;"],[0,"&Udblac;"],[0,"&udblac;"],[0,"&Uogon;"],[0,"&uogon;"],[0,"&Wcirc;"],[0,"&wcirc;"],[0,"&Ycirc;"],[0,"&ycirc;"],[0,"&Yuml;"],[0,"&Zacute;"],[0,"&zacute;"],[0,"&Zdot;"],[0,"&zdot;"],[0,"&Zcaron;"],[0,"&zcaron;"],[19,"&fnof;"],[34,"&imped;"],[63,"&gacute;"],[65,"&jmath;"],[142,"&circ;"],[0,"&caron;"],[16,"&breve;"],[0,"&DiacriticalDot;"],[0,"&ring;"],[0,"&ogon;"],[0,"&DiacriticalTilde;"],[0,"&dblac;"],[51,"&DownBreve;"],[127,"&Alpha;"],[0,"&Beta;"],[0,"&Gamma;"],[0,"&Delta;"],[0,"&Epsilon;"],[0,"&Zeta;"],[0,"&Eta;"],[0,"&Theta;"],[0,"&Iota;"],[0,"&Kappa;"],[0,"&Lambda;"],[0,"&Mu;"],[0,"&Nu;"],[0,"&Xi;"],[0,"&Omicron;"],[0,"&Pi;"],[0,"&Rho;"],[1,"&Sigma;"],[0,"&Tau;"],[0,"&Upsilon;"],[0,"&Phi;"],[0,"&Chi;"],[0,"&Psi;"],[0,"&ohm;"],[7,"&alpha;"],[0,"&beta;"],[0,"&gamma;"],[0,"&delta;"],[0,"&epsi;"],[0,"&zeta;"],[0,"&eta;"],[0,"&theta;"],[0,"&iota;"],[0,"&kappa;"],[0,"&lambda;"],[0,"&mu;"],[0,"&nu;"],[0,"&xi;"],[0,"&omicron;"],[0,"&pi;"],[0,"&rho;"],[0,"&sigmaf;"],[0,"&sigma;"],[0,"&tau;"],[0,"&upsi;"],[0,"&phi;"],[0,"&chi;"],[0,"&psi;"],[0,"&omega;"],[7,"&thetasym;"],[0,"&Upsi;"],[2,"&phiv;"],[0,"&piv;"],[5,"&Gammad;"],[0,"&digamma;"],[18,"&kappav;"],[0,"&rhov;"],[3,"&epsiv;"],[0,"&backepsilon;"],[10,"&IOcy;"],[0,"&DJcy;"],[0,"&GJcy;"],[0,"&Jukcy;"],[0,"&DScy;"],[0,"&Iukcy;"],[0,"&YIcy;"],[0,"&Jsercy;"],[0,"&LJcy;"],[0,"&NJcy;"],[0,"&TSHcy;"],[0,"&KJcy;"],[1,"&Ubrcy;"],[0,"&DZcy;"],[0,"&Acy;"],[0,"&Bcy;"],[0,"&Vcy;"],[0,"&Gcy;"],[0,"&Line truncated
+125
View File
@@ -0,0 +1,125 @@
import { readFileSync } from "node:fs";
import { describe, it, expect } from "vitest";
import * as entities from "./index.js";
import legacy from "../maps/legacy.json" with { type: "json" };
const levels = ["xml", "entities"];
describe("Documents", () => {
const levelDocuments = levels
.map((name) => new URL(`../maps/${name}.json`, import.meta.url))
.map((url) => JSON.parse(readFileSync(url, "utf8")))
.map((document, index) => [index, document]);
for (const [level, document] of levelDocuments) {
describe("Decode", () => {
it(levels[level], () => {
for (const entity of Object.keys(document)) {
for (let l = level; l < levels.length; l++) {
expect(entities.decode(`&${entity};`, l)).toBe(
document[entity],
);
expect(
entities.decode(`&${entity};`, { level: l }),
).toBe(document[entity]);
}
}
});
});
describe("Decode strict", () => {
it(levels[level], () => {
for (const entity of Object.keys(document)) {
for (let l = level; l < levels.length; l++) {
expect(entities.decodeStrict(`&${entity};`, l)).toBe(
document[entity],
);
expect(
entities.decode(`&${entity};`, {
level: l,
mode: entities.DecodingMode.Strict,
}),
).toBe(document[entity]);
}
}
});
});
describe("Encode", () => {
it(levels[level], () => {
for (const entity of Object.keys(document)) {
for (let l = level; l < levels.length; l++) {
const encoded = entities.encode(document[entity], l);
const decoded = entities.decode(encoded, l);
expect(decoded).toBe(document[entity]);
}
}
});
it("should only encode non-ASCII values if asked", () =>
expect(
entities.encode("Great #'s of 🎁", {
level,
mode: entities.EncodingMode.ASCII,
}),
).toBe("Great #&apos;s of &#x1f381;"));
});
}
describe("Legacy", () => {
const legacyMap: Record<string, string> = legacy;
it("should decode", () => {
for (const entity of Object.keys(legacyMap)) {
expect(entities.decodeHTML(`&${entity}`)).toBe(
legacyMap[entity],
);
expect(
entities.decodeStrict(`&${entity}`, {
level: entities.EntityLevel.HTML,
mode: entities.DecodingMode.Legacy,
}),
).toBe(legacyMap[entity]);
}
});
});
});
const astral = [
["1d306", "\uD834\uDF06"],
["1d11e", "\uD834\uDD1E"],
];
const astralSpecial = [
["80", "\u20AC"],
["110000", "\uFFFD"],
];
describe("Astral entities", () => {
for (const [c, value] of astral) {
it(`should decode ${value}`, () =>
expect(entities.decode(`&#x${c};`)).toBe(value));
it(`should encode ${value}`, () =>
expect(entities.encode(value)).toBe(`&#x${c};`));
it(`should escape ${value}`, () =>
expect(entities.escape(value)).toBe(`&#x${c};`));
}
for (const [c, value] of astralSpecial) {
it(`should decode special \\u${c}`, () =>
expect(entities.decode(`&#x${c};`)).toBe(value));
}
});
describe("Escape", () => {
it("should always decode ASCII chars", () => {
for (let index = 0; index < 0x7f; index++) {
const c = String.fromCharCode(index);
expect(entities.decodeXML(entities.escape(c))).toBe(c);
}
});
it("should keep UTF8 characters", () =>
expect(entities.escapeUTF8('ß < "ü"')).toBe(`ß &lt; &quot;ü&quot;`));
});
+188
View File
@@ -0,0 +1,188 @@
import { decodeXML, decodeHTML, DecodingMode } from "./decode.js";
import { encodeHTML, encodeNonAsciiHTML } from "./encode.js";
import {
encodeXML,
escapeUTF8,
escapeAttribute,
escapeText,
} from "./escape.js";
/** The level of entities to support. */
export enum EntityLevel {
/** Support only XML entities. */
XML = 0,
/** Support HTML entities, which are a superset of XML entities. */
HTML = 1,
}
export enum EncodingMode {
/**
* The output is UTF-8 encoded. Only characters that need escaping within
* XML will be escaped.
*/
UTF8,
/**
* The output consists only of ASCII characters. Characters that need
* escaping within HTML, and characters that aren't ASCII characters will
* be escaped.
*/
ASCII,
/**
* Encode all characters that have an equivalent entity, as well as all
* characters that are not ASCII characters.
*/
Extensive,
/**
* Encode all characters that have to be escaped in HTML attributes,
* following {@link https://html.spec.whatwg.org/multipage/parsing.html#escapingString}.
*/
Attribute,
/**
* Encode all characters that have to be escaped in HTML text,
* following {@link https://html.spec.whatwg.org/multipage/parsing.html#escapingString}.
*/
Text,
}
export interface DecodingOptions {
/**
* The level of entities to support.
* @default {@link EntityLevel.XML}
*/
level?: EntityLevel;
/**
* Decoding mode. If `Legacy`, will support legacy entities not terminated
* with a semicolon (`;`).
*
* Always `Strict` for XML. For HTML, set this to `true` if you are parsing
* an attribute value.
*
* The deprecated `decodeStrict` function defaults this to `Strict`.
*
* @default {@link DecodingMode.Legacy}
*/
mode?: DecodingMode | undefined;
}
/**
* Decodes a string with entities.
*
* @param input String to decode.
* @param options Decoding options.
*/
export function decode(
input: string,
options: DecodingOptions | EntityLevel = EntityLevel.XML,
): string {
const level = typeof options === "number" ? options : options.level;
if (level === EntityLevel.HTML) {
const mode = typeof options === "object" ? options.mode : undefined;
return decodeHTML(input, mode);
}
return decodeXML(input);
}
/**
* Decodes a string with entities. Does not allow missing trailing semicolons for entities.
*
* @param input String to decode.
* @param options Decoding options.
* @deprecated Use `decode` with the `mode` set to `Strict`.
*/
export function decodeStrict(
input: string,
options: DecodingOptions | EntityLevel = EntityLevel.XML,
): string {
const normalizedOptions =
typeof options === "number" ? { level: options } : options;
normalizedOptions.mode ??= DecodingMode.Strict;
return decode(input, normalizedOptions);
}
/**
* Options for `encode`.
*/
export interface EncodingOptions {
/**
* The level of entities to support.
* @default {@link EntityLevel.XML}
*/
level?: EntityLevel;
/**
* Output format.
* @default {@link EncodingMode.Extensive}
*/
mode?: EncodingMode;
}
/**
* Encodes a string with entities.
*
* @param input String to encode.
* @param options Encoding options.
*/
export function encode(
input: string,
options: EncodingOptions | EntityLevel = EntityLevel.XML,
): string {
const { mode = EncodingMode.Extensive, level = EntityLevel.XML } =
typeof options === "number" ? { level: options } : options;
switch (mode) {
case EncodingMode.UTF8: {
return escapeUTF8(input);
}
case EncodingMode.Attribute: {
return escapeAttribute(input);
}
case EncodingMode.Text: {
return escapeText(input);
}
case EncodingMode.ASCII: {
return level === EntityLevel.HTML
? encodeNonAsciiHTML(input)
: encodeXML(input);
}
// eslint-disable-next-line unicorn/no-useless-switch-case
case EncodingMode.Extensive:
default: {
return level === EntityLevel.HTML
? encodeHTML(input)
: encodeXML(input);
}
}
}
export {
encodeXML,
escape,
escapeUTF8,
escapeAttribute,
escapeText,
} from "./escape.js";
export {
encodeHTML,
encodeNonAsciiHTML,
// Legacy aliases (deprecated)
encodeHTML as encodeHTML4,
encodeHTML as encodeHTML5,
} from "./encode.js";
export {
EntityDecoder,
DecodingMode,
decodeXML,
decodeHTML,
decodeHTMLStrict,
decodeHTMLAttribute,
// Legacy aliases (deprecated)
decodeHTML as decodeHTML4,
decodeHTML as decodeHTML5,
decodeHTMLStrict as decodeHTML4Strict,
decodeHTMLStrict as decodeHTML5Strict,
decodeXML as decodeXMLStrict,
} from "./decode.js";