Brunobkr/llama.cpp_AlgMor24_github

1/** @import { ValidatedCompileOptions } from '#compiler' */2/** @import { Processed } from '../preprocess/public.js' */3/** @import { SourceMap } from 'magic-string' */4/** @import { Source } from '../preprocess/private.js' */5/** @import { DecodedSourceMap, SourceMapSegment, RawSourceMap } from '@jridgewell/remapping' */6import remapping from '@jridgewell/remapping';7import { push_array } from './push_array.js';8 9/**10 * @param {string} s11 */12function last_line_length(s) {13	return s.length - s.lastIndexOf('\n') - 1;14}15// mutate map in-place16 17/**18 * @param {DecodedSourceMap} map19 * @param {{ line: number; column: number; }} offset20 * @param {number} source_index21 */22export function sourcemap_add_offset(map, offset, source_index) {23	if (map.mappings.length == 0) return;24	for (let line = 0; line < map.mappings.length; line++) {25		const segment_list = map.mappings[line];26		for (let segment = 0; segment < segment_list.length; segment++) {27			const seg = segment_list[segment];28			// shift only segments that belong to component source file29			if (seg[1] === source_index) {30				// also ensures that seg.length >= 431				// shift column if it points at the first line32				if (seg[2] === 0) {33					/** @type {any} */ (seg[3]) += offset.column;34				}35				// shift line36				/** @type {any} */ (seg[2]) += offset.line;37			}38		}39	}40}41 42/**43 * @template T44 * @param {T[]} this_table45 * @param {T[]} other_table46 * @returns {[T[], number[], boolean, boolean]}47 */48function merge_tables(this_table, other_table) {49	const new_table = this_table.slice();50	const idx_map = [];51	other_table = other_table || [];52	let val_changed = false;53	for (const [other_idx, other_val] of other_table.entries()) {54		const this_idx = this_table.indexOf(other_val);55		if (this_idx >= 0) {56			idx_map[other_idx] = this_idx;57		} else {58			const new_idx = new_table.length;59			new_table[new_idx] = other_val;60			idx_map[other_idx] = new_idx;61			val_changed = true;62		}63	}64	let idx_changed = val_changed;65	if (val_changed) {66		if (idx_map.find((val, idx) => val != idx) === undefined) {67			// idx_map is identity map [0, 1, 2, 3, 4, ....]68			idx_changed = false;69		}70	}71	return [new_table, idx_map, val_changed, idx_changed];72}73const regex_line_token = /([^\w\s]|\s+)/g;74/** */75export class MappedCode {76	/**77	 * @type {string}78	 */79	string = /** @type {any} */ (undefined);80 81	/**82	 * @type {DecodedSourceMap}83	 */84	map = /** @type {any} */ (undefined);85 86	/**87	 * @param {string} string88	 * @param {DecodedSourceMap | null} map89	 */90	constructor(string = '', map = null) {91		this.string = string;92		if (map) {93			this.map = map;94		} else {95			this.map = {96				version: 3,97				mappings: [],98				sources: [],99				names: []100			};101		}102	}103	/**104	 * concat in-place (mutable), return this (chainable)105	 * will also mutate the `other` object106	 * @param {MappedCode} other107	 * @returns {MappedCode}108	 */109	concat(other) {110		// noop: if one is empty, return the other111		if (other.string == '') return this;112		if (this.string == '') {113			this.string = other.string;114			this.map = other.map;115			return this;116		}117		// compute last line length before mutating118		const column_offset = last_line_length(this.string);119		this.string += other.string;120		const m1 = this.map;121		const m2 = other.map;122		if (m2.mappings.length == 0) return this;123		// combine sources and names124		const [sources, new_source_idx, sources_changed, sources_idx_changed] = merge_tables(125			m1.sources,126			m2.sources127		);128		const [names, new_name_idx, names_changed, names_idx_changed] = merge_tables(129			m1.names,130			m2.names131		);132		if (sources_changed) m1.sources = sources;133		if (names_changed) m1.names = names;134		// unswitched loops are faster135		if (sources_idx_changed && names_idx_changed) {136			for (let line = 0; line < m2.mappings.length; line++) {137				const segment_list = m2.mappings[line];138				for (let segment = 0; segment < segment_list.length; segment++) {139					const seg = segment_list[segment];140					// @ts-ignore141					if (seg[1] >= 0) seg[1] = new_source_idx[seg[1]];142					// @ts-ignore143					if (seg[4] >= 0) seg[4] = new_name_idx[seg[4]];144				}145			}146		} else if (sources_idx_changed) {147			for (let line = 0; line < m2.mappings.length; line++) {148				const segment_list = m2.mappings[line];149				for (let segment = 0; segment < segment_list.length; segment++) {150					const seg = segment_list[segment];151					// @ts-ignore152					if (seg[1] >= 0) seg[1] = new_source_idx[seg[1]];153				}154			}155		} else if (names_idx_changed) {156			for (let line = 0; line < m2.mappings.length; line++) {157				const segment_list = m2.mappings[line];158				for (let segment = 0; segment < segment_list.length; segment++) {159					const seg = segment_list[segment];160					// @ts-ignore161					if (seg[4] >= 0) seg[4] = new_name_idx[seg[4]];162				}163			}164		}165		// combine the mappings166		// combine167		// 1. last line of first map168		// 2. first line of second map169		// columns of 2 must be shifted170		if (m2.mappings.length > 0 && column_offset > 0) {171			const first_line = m2.mappings[0];172			for (let i = 0; i < first_line.length; i++) {173				first_line[i][0] += column_offset;174			}175		}176		// combine last line + first line177		push_array(178			m1.mappings[m1.mappings.length - 1],179			/** @type {SourceMapSegment[]} */ (m2.mappings.shift())180		);181		// append other lines182		push_array(m1.mappings, m2.mappings);183		return this;184	}185 186	/**187	 * @static188	 * @param {string} string189	 * @param {DecodedSourceMap} [map]190	 * @returns {MappedCode}191	 */192	static from_processed(string, map) {193		const line_count = string.split('\n').length;194		if (map) {195			// ensure that count of source map mappings lines196			// is equal to count of generated code lines197			// (some tools may produce less)198			const missing_lines = line_count - map.mappings.length;199			for (let i = 0; i < missing_lines; i++) {200				map.mappings.push([]);201			}202			return new MappedCode(string, map);203		}204		if (string == '') return new MappedCode();205		map = { version: 3, names: [], sources: [], mappings: [] };206		// add empty SourceMapSegment[] for every line207		for (let i = 0; i < line_count; i++) map.mappings.push([]);208		return new MappedCode(string, map);209	}210 211	/**212	 * @static213	 * @param {Source} opts214	 * @returns {MappedCode}215	 */216	static from_source({ source, file_basename, get_location }) {217		/**218		 * @type {{ line: number; column: number; }}219		 */220		let offset = get_location(0);221		if (!offset) offset = { line: 0, column: 0 };222 223		/**224		 * @type {DecodedSourceMap}225		 */226		const map = { version: 3, names: [], sources: [file_basename], mappings: [] };227		if (source == '') return new MappedCode(source, map);228		// we create a high resolution identity map here,229		// we know that it will eventually be merged with svelte's map,230		// at which stage the resolution will decrease.231		const line_list = source.split('\n');232		for (let line = 0; line < line_list.length; line++) {233			map.mappings.push([]);234			const token_list = line_list[line].split(regex_line_token);235			for (let token = 0, column = 0; token < token_list.length; token++) {236				if (token_list[token] == '') continue;237				map.mappings[line].push([column, 0, offset.line + line, column]);238				column += token_list[token].length;239			}240		}241		// shift columns in first line242		const segment_list = map.mappings[0];243		for (let segment = 0; segment < segment_list.length; segment++) {244			// @ts-ignore245			segment_list[segment][3] += offset.column;246		}247		return new MappedCode(source, map);248	}249}250 251// browser vs node.js252const b64enc =253	typeof window !== 'undefined' && typeof btoa === 'function'254		? /** @param {string} str */ (str) => btoa(unescape(encodeURIComponent(str)))255		: /** @param {string} str */ (str) => Buffer.from(str).toString('base64');256const b64dec =257	typeof window !== 'undefined' && typeof atob === 'function'258		? atob259		: /** @param {any} a */ (a) => Buffer.from(a, 'base64').toString();260 261/**262 * @param {string} filename Basename of the input file263 * @param {Array<DecodedSourceMap | RawSourceMap>} sourcemap_list264 */265export function combine_sourcemaps(filename, sourcemap_list) {266	if (sourcemap_list.length == 0) return null;267	let map_idx = 1;268	const map =269		sourcemap_list.slice(0, -1).find((m) => m.sources.length !== 1) === undefined270			? remapping(271					// use array interface272					// only the oldest sourcemap can have multiple sources273					sourcemap_list,274					() => null,275					true // skip optional field `sourcesContent`276				)277			: remapping(278					// use loader interface279					sourcemap_list[0], // last map280					(sourcefile) => {281						// TODO the equality check assumes that the preprocessor map has the input file as a relative path in sources,282						// e.g. when the input file is `src/foo/bar.svelte`, then sources is expected to contain just `bar.svelte`.283						// Therefore filename also needs to be the basename of the path. This feels brittle, investigate how we can284						// harden this (without breaking other tooling that assumes this behavior).285						if (sourcefile === filename && sourcemap_list[map_idx]) {286							return sourcemap_list[map_idx++]; // idx 1, 2, ...287							// bundle file = branch node288						} else {289							return null; // source file = leaf node290						}291					},292					true293				);294	if (!map.file) delete map.file; // skip optional field `file`295	// When source maps are combined and the leading map is empty, sources is not set.296	// Add the filename to the empty array in this case.297	// Further improvements to remapping may help address this as well https://github.com/ampproject/remapping/issues/116298	if (!map.sources.length) map.sources = [filename];299	return map;300}301 302/**303 * @param {string} filename304 * @param {SourceMap} svelte_map305 * @param {string | DecodedSourceMap | RawSourceMap} preprocessor_map_input306 * @returns {SourceMap}307 */308function apply_preprocessor_sourcemap(filename, svelte_map, preprocessor_map_input) {309	if (!svelte_map || !preprocessor_map_input) return svelte_map;310	const preprocessor_map =311		typeof preprocessor_map_input === 'string'312			? JSON.parse(preprocessor_map_input)313			: preprocessor_map_input;314	const result_map = combine_sourcemaps(filename, [svelte_map, preprocessor_map]);315	// Svelte expects a SourceMap which includes toUrl and toString. Instead of wrapping our output in a class,316	// we just tack on the extra properties.317	Object.defineProperties(result_map, {318		toString: {319			enumerable: false,320			value: function toString() {321				return JSON.stringify(this);322			}323		},324		toUrl: {325			enumerable: false,326			value: function toUrl() {327				return 'data:application/json;charset=utf-8;base64,' + b64enc(this.toString());328			}329		}330	});331	return /** @type {any} */ (result_map);332}333const regex_data_uri = /data:(?:application|text)\/json;(?:charset[:=]\S+?;)?base64,(\S*)/;334// parse attached sourcemap in processed.code335 336/**337 * @param {Processed} processed338 * @param {'script' | 'style'} tag_name339 * @returns {void}340 */341export function parse_attached_sourcemap(processed, tag_name) {342	const r_in = '[#@]\\s*sourceMappingURL\\s*=\\s*(\\S*)';343	const regex =344		tag_name == 'script'345			? new RegExp('(?://' + r_in + ')|(?:/\\*' + r_in + '\\s*\\*/)$')346			: new RegExp('/\\*' + r_in + '\\s*\\*/$');347 348	/**349	 * @param {any} message350	 */351	function log_warning(message) {352		// code_start: help to find preprocessor353		const code_start =354			processed.code.length < 100 ? processed.code : processed.code.slice(0, 100) + ' [...]';355		// eslint-disable-next-line no-console356		console.warn(`warning: ${message}. processed.code = ${JSON.stringify(code_start)}`);357	}358	processed.code = processed.code.replace(regex, (_, match1, match2) => {359		const map_url = tag_name == 'script' ? match1 || match2 : match1;360		const map_data = (map_url.match(regex_data_uri) || [])[1];361		if (map_data) {362			// sourceMappingURL is data URL363			if (processed.map) {364				log_warning(365					'Not implemented. ' +366						'Found sourcemap in both processed.code and processed.map. ' +367						'Please update your preprocessor to return only one sourcemap.'368				);369				// ignore attached sourcemap370				return '';371			}372			processed.map = b64dec(map_data); // use attached sourcemap373			return ''; // remove from processed.code374		}375		// sourceMappingURL is path or URL376		if (!processed.map) {377			log_warning(378				`Found sourcemap path ${JSON.stringify(379					map_url380				)} in processed.code, but no sourcemap data. ` +381					'Please update your preprocessor to return sourcemap data directly.'382			);383		}384		// ignore sourcemap path385		return ''; // remove from processed.code386	});387}388 389/**390 * @param {{ code: string, map: SourceMap}} result391 * @param {ValidatedCompileOptions} options392 * @param {string} source_name393 */394export function merge_with_preprocessor_map(result, options, source_name) {395	if (options.sourcemap) {396		const file_basename = get_basename(options.filename);397		// The preprocessor map is expected to contain `sources: [basename_of_filename]`, but our own398		// map may contain a different file name. Patch our map beforehand to align sources so merging399		// with the preprocessor map works correctly.400		result.map.sources = [file_basename];401		Object.assign(402			result.map,403			apply_preprocessor_sourcemap(404				file_basename,405				result.map,406				/** @type {any} */ (options.sourcemap)407			)408		);409		// After applying the preprocessor map, we need to do the inverse and make the sources410		// relative to the input file again in case the output code is in a different directory.411		if (file_basename !== source_name) {412			result.map.sources = result.map.sources.map(413				/** @param {string} source */ (source) => get_relative_path(source_name, source)414			);415		}416	}417}418 419/**420 * @param {string} from421 * @param {string} to422 */423function get_relative_path(from, to) {424	// Don't use node's utils here to ensure the compiler is usable in a browser environment425	const from_parts = from.split(/[/\\]/);426	const to_parts = to.split(/[/\\]/);427	from_parts.pop(); // get dirname428	while (from_parts[0] === to_parts[0]) {429		from_parts.shift();430		to_parts.shift();431	}432	if (from_parts.length) {433		let i = from_parts.length;434		while (i--) from_parts[i] = '..';435	}436	return from_parts.concat(to_parts).join('/');437}438 439/**440 * Like node's `basename`, but doesn't use it to ensure the compiler is usable in a browser environment441 * @param {string} filename442 */443export function get_basename(filename) {444	return /** @type {string} */ (filename.split(/[/\\]/).pop());445}446 447/**448 * @param {string} filename449 * @param {string | undefined} output_filename450 * @param {string} fallback451 */452export function get_source_name(filename, output_filename, fallback) {453	return output_filename ? get_relative_path(output_filename, filename) : get_basename(filename);454}455