<commit_before>import { NativeConfig, NativeParser } from './ParserLib';

import { Namespace } from '../Namespace';
import { ParserNamespace } from './ParserNamespace';
import { TokenSpace } from '../tokenizer/TokenSpace';
import { TokenSet } from '../tokenizer/TokenSet';
import { InternalToken } from './InternalToken';
import { TokenKind, MemberToken, OpenToken, CloseToken, EmittedToken, StringToken } from './Token';
import { Parser } from './Parser';
import { XModuleTable } from './JSX';

export interface ParserOptions {
	parseUnknown?: boolean;
	omitDefaults?: boolean;
}

export interface TokenTbl {
	[ prefix: string ]: {
		uri: string,
		elements?: string[],
		attributes?: string[]
	}
}

export interface Registry {
	tokens: { [ name: string ]: MemberToken };
	elements: { [ id: number ]: string };
	attributes: { [ id: number ]: string };
}

/** Parser configuration for quickly instantiating new parsers.
  * Each parser instance holds a new, cloned copy. */

export class ParserConfig {

	/** XML parser configuration.
	  * @param config Parent object for cloning.
	  * @param native Reference to C++ object. For internal use only. */
	constructor(config?: ParserOptions | ParserConfig, native?: NativeConfig | null) {
		if(config instanceof ParserConfig) {
			this.isLinked = true;

			this.options = config.options;

			this.uriSpace = config.uriSpace;
			this.prefixSpace = config.prefixSpace;
			this.elementSpace = config.elementSpace;
			this.attributeSpace = config.attributeSpace;

			this.xmlnsToken = config.xmlnsToken;

			this.emptyPrefixToken = config.emptyPrefixToken;
			this.xmlnsPrefixToken = config.xmlnsPrefixToken;
			this.processingPrefixToken = config.processingPrefixToken;

			this.uriSet = config.uriSet;
			this.prefixSet = config.prefixSet;

			this.namespaceList = config.namespaceList;
			this.namespaceTbl = config.namespaceTbl;
			this.maxNamespace = config.maxNamespace;

			this.nsMapper = config.nsMapper;
		} else {
			this.options = config || {};

			this.uriSpace = new TokenSpace(TokenKind.uri);
			this.prefixSpace = new TokenSpace(TokenKind.prefix);
			this.elementSpace = new TokenSpace(TokenKind.element);
			this.attributeSpace = new TokenSpace(TokenKind.attribute);

			this.xmlnsToken = this.attributeSpace.createToken('xmlns');

			this.uriSet = new TokenSet(this.uriSpace);
			this.prefixSet = new TokenSet(this.prefixSpace);

			this.namespaceList = [];
			this.namespaceTbl = {};
			this.maxNamespace = 0;
		}

		// this.clonedNamespaceCount = this.maxNamespace;

		if(!native) {
			this.emptyPrefixToken = this.prefixSet.createToken('');
			this.xmlnsPrefixToken = this.prefixSet.createToken('xmlns');
			this.processingPrefixToken = this.prefixSet.createToken('?');

			native = new NativeConfig(this.xmlnsToken.id, this.emptyPrefixToken.id, this.xmlnsPrefixToken.id, this.processingPrefixToken.id);
		}

		this.native = native;

		if(!this.isLinked && !this.options.omitDefaults) {
			this.bindNamespace(Namespace.processing);
			this.bindNamespace(Namespace.unknown);
			this.bindNamespace(Namespace.xml1998);
		}
	}

	unlink() {
		if(!this.isLinked) return;
		this.isLinked = false;

		this.uriSpace = new TokenSpace(TokenKind.uri, this.uriSpace);
		this.prefixSpace = new TokenSpace(TokenKind.prefix, this.prefixSpace);
		this.elementSpace = new TokenSpace(TokenKind.element, this.elementSpace);
		this.attributeSpace = new TokenSpace(TokenKind.attribute, this.attributeSpace);

		this.uriSet = new TokenSet(this.uriSpace, this.uriSet);
		this.prefixSet = new TokenSet(this.prefixSpace, this.prefixSet);

		const namespaceList = this.namespaceList.slice(0);
		let num = namespaceList.length;

		while(num--) {
			let ns = namespaceList[num];

			// This just skips namespace 0 which never exists
			// (see ParserConfig.cc).
			if(ns) {
				ns = new ParserNamespace(ns, this);
				ns.id = num;
				namespaceList[num] = ns;
			}
		}

		const namespaceTbl: { [ name: string ]: ParserNamespace } = {};
		for(let key of Object.keys(this.namespaceTbl)) {
			namespaceTbl[key] = namespaceList[this.namespaceTbl[key].id];
		}

		this.namespaceList = namespaceList;
		this.namespaceTbl = namespaceTbl;
	}

	createParser() {
		const nativeParser = new NativeParser(this.native);
		const config = new ParserConfig(this, nativeParser.getConfig());

		return(new Parser(config, nativeParser));
	}

	parseSync(data: string) {
		return(this.createParser().parseSync(data));
	}

	getNamespace(uri: string) {
		const ns = this.namespaceTbl[uri];
		return(ns && ns.base);
	}

	addNamespace(nsBase: Namespace) {
		let uri = (this.nsMapper && this.nsMapper(nsBase.uri)) || nsBase.uri;
		let nsParser = this.namespaceTbl[uri];

		if(nsParser) return(nsParser.id);

		if(this.isLinked) this.unlink();

		nsBase.uri = uri;
		nsParser = new ParserNamespace(nsBase, this);
		nsParser.id = this.native.addNamespace(nsParser.registerNative());

		this.namespaceList[nsParser.id] = nsParser;
		this.namespaceTbl[uri] = nsParser;

		if(nsBase.id > this.maxNamespace) this.maxNamespace = nsBase.id;

		if(nsBase.defaultPrefix) this.addPrefix(nsBase.defaultPrefix);
		this.addUri(uri, nsParser);

		return(nsParser.id);
	}

	bindNamespace(nsBase: Namespace, prefix?: string, parser?: Parser) {
		this.addNamespace(nsBase);

		let uri = nsBase.uri;
		let nsParser = this.namespaceTbl[uri];

		prefix = prefix || nsParser.base.defaultPrefix;

		(parser || this).bindPrefix(
			this.addPrefix(prefix),
			this.addUri(uri, nsParser)
		);

		return(nsParser.id);<commit_msg>Allow binding a default namespace even if it suggests a prefix.<commit_after>import { NativeConfig, NativeParser } from './ParserLib';

import { Namespace } from '../Namespace';
import { ParserNamespace } from './ParserNamespace';
import { TokenSpace } from '../tokenizer/TokenSpace';
import { TokenSet } from '../tokenizer/TokenSet';
import { InternalToken } from './InternalToken';
import { TokenKind, MemberToken, OpenToken, CloseToken, EmittedToken, StringToken } from './Token';
import { Parser } from './Parser';
import { XModuleTable } from './JSX';

export interface ParserOptions {
	parseUnknown?: boolean;
	omitDefaults?: boolean;
}

export interface TokenTbl {
	[ prefix: string ]: {
		uri: string,
		elements?: string[],
		attributes?: string[]
	}
}

export interface Registry {
	tokens: { [ name: string ]: MemberToken };
	elements: { [ id: number ]: string };
	attributes: { [ id: number ]: string };
}

/** Parser configuration for quickly instantiating new parsers.
  * Each parser instance holds a new, cloned copy. */

export class ParserConfig {

	/** XML parser configuration.
	  * @param config Parent object for cloning.
	  * @param native Reference to C++ object. For internal use only. */
	constructor(config?: ParserOptions | ParserConfig, native?: NativeConfig | null) {
		if(config instanceof ParserConfig) {
			this.isLinked = true;

			this.options = config.options;

			this.uriSpace = config.uriSpace;
			this.prefixSpace = config.prefixSpace;
			this.elementSpace = config.elementSpace;
			this.attributeSpace = config.attributeSpace;

			this.xmlnsToken = config.xmlnsToken;

			this.emptyPrefixToken = config.emptyPrefixToken;
			this.xmlnsPrefixToken = config.xmlnsPrefixToken;
			this.processingPrefixToken = config.processingPrefixToken;

			this.uriSet = config.uriSet;
			this.prefixSet = config.prefixSet;

			this.namespaceList = config.namespaceList;
			this.namespaceTbl = config.namespaceTbl;
			this.maxNamespace = config.maxNamespace;

			this.nsMapper = config.nsMapper;
		} else {
			this.options = config || {};

			this.uriSpace = new TokenSpace(TokenKind.uri);
			this.prefixSpace = new TokenSpace(TokenKind.prefix);
			this.elementSpace = new TokenSpace(TokenKind.element);
			this.attributeSpace = new TokenSpace(TokenKind.attribute);

			this.xmlnsToken = this.attributeSpace.createToken('xmlns');

			this.uriSet = new TokenSet(this.uriSpace);
			this.prefixSet = new TokenSet(this.prefixSpace);

			this.namespaceList = [];
			this.namespaceTbl = {};
			this.maxNamespace = 0;
		}

		// this.clonedNamespaceCount = this.maxNamespace;

		if(!native) {
			this.emptyPrefixToken = this.prefixSet.createToken('');
			this.xmlnsPrefixToken = this.prefixSet.createToken('xmlns');
			this.processingPrefixToken = this.prefixSet.createToken('?');

			native = new NativeConfig(this.xmlnsToken.id, this.emptyPrefixToken.id, this.xmlnsPrefixToken.id, this.processingPrefixToken.id);
		}

		this.native = native;

		if(!this.isLinked && !this.options.omitDefaults) {
			this.bindNamespace(Namespace.processing);
			this.bindNamespace(Namespace.unknown);
			this.bindNamespace(Namespace.xml1998);
		}
	}

	unlink() {
		if(!this.isLinked) return;
		this.isLinked = false;

		this.uriSpace = new TokenSpace(TokenKind.uri, this.uriSpace);
		this.prefixSpace = new TokenSpace(TokenKind.prefix, this.prefixSpace);
		this.elementSpace = new TokenSpace(TokenKind.element, this.elementSpace);
		this.attributeSpace = new TokenSpace(TokenKind.attribute, this.attributeSpace);

		this.uriSet = new TokenSet(this.uriSpace, this.uriSet);
		this.prefixSet = new TokenSet(this.prefixSpace, this.prefixSet);

		const namespaceList = this.namespaceList.slice(0);
		let num = namespaceList.length;

		while(num--) {
			let ns = namespaceList[num];

			// This just skips namespace 0 which never exists
			// (see ParserConfig.cc).
			if(ns) {
				ns = new ParserNamespace(ns, this);
				ns.id = num;
				namespaceList[num] = ns;
			}
		}

		const namespaceTbl: { [ name: string ]: ParserNamespace } = {};
		for(let key of Object.keys(this.namespaceTbl)) {
			namespaceTbl[key] = namespaceList[this.namespaceTbl[key].id];
		}

		this.namespaceList = namespaceList;
		this.namespaceTbl = namespaceTbl;
	}

	createParser() {
		const nativeParser = new NativeParser(this.native);
		const config = new ParserConfig(this, nativeParser.getConfig());

		return(new Parser(config, nativeParser));
	}

	parseSync(data: string) {
		return(this.createParser().parseSync(data));
	}

	getNamespace(uri: string) {
		const ns = this.namespaceTbl[uri];
		return(ns && ns.base);
	}

	addNamespace(nsBase: Namespace) {
		let uri = (this.nsMapper && this.nsMapper(nsBase.uri)) || nsBase.uri;
		let nsParser = this.namespaceTbl[uri];

		if(nsParser) return(nsParser.id);

		if(this.isLinked) this.unlink();

		nsBase.uri = uri;
		nsParser = new ParserNamespace(nsBase, this);
		nsParser.id = this.native.addNamespace(nsParser.registerNative());

		this.namespaceList[nsParser.id] = nsParser;
		this.namespaceTbl[uri] = nsParser;

		if(nsBase.id > this.maxNamespace) this.maxNamespace = nsBase.id;

		if(nsBase.defaultPrefix) this.addPrefix(nsBase.defaultPrefix);
		this.addUri(uri, nsParser);

		return(nsParser.id);
	}

	bindNamespace(nsBase: Namespace, prefix?: string, parser?: Parser) {
		this.addNamespace(nsBase);

		let uri = nsBase.uri;
		let nsParser = this.namespaceTbl[uri];

		if(!prefix && prefix != '') prefix = nsParser.base.defaultPrefix;

		(parser || this).bindPrefix(
			this.addPrefix(prefix),
			this.addUri(uri, nsParser)
		);

		return(nsParser.id);
{'fraction_non_alphanumeric': 0.12950937950937952, 'fraction_numerical': 0.0012626262626262627, 'mean_word_length': 4.333814333814334, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7218543', 'n_tokens_mistral': 3816, 'n_tokens_neox': 3407, 'n_words': 720}