<commit_before>// This file is part of cxml, copyright (c) 2016 BusFaster Ltd.
// Released under the MIT license, see LICENSE.

import * as stream from 'stream';
import * as Promise from 'bluebird';
import * as sax from 'sax';

import {Context} from './Context';
import {Namespace} from './Namespace';
import {Type, TypeClass, HandlerInstance} from './Type';
import {Member} from './Member';
import {State} from './State';
import {defaultContext} from '../importer/JS';

var converterTbl: { [type: string]: (item: string) => any } = {
	Date: ((item: string) => {
		var dateParts = item.match(/([0-9]+)-([0-9]+)-([0-9]+)(?:T([0-9]+):([0-9]+):([0-9]+)(\.[0-9]+)?)?(?:Z|([+-][0-9]+):([0-9]+))?/);
		var offset = +(dateParts[8] || '0') * 60 + +(dateParts[9] || '0');

		var date = new Date(
			+dateParts[1],
			+dateParts[2] - 1,
			+dateParts[3],
			+(dateParts[4] || '0'),
			+(dateParts[5] || '0'),
			+(dateParts[6] || '0'),
			+(dateParts[7] || '0') * 1000
		);

		offset += date.getTimezoneOffset();
		date.setTime(date.getTime() - offset * 60000);

		return(date);
	}),
	boolean: ((item: string) => !!item),
	string: ((item: string) => item),
	number: ((item: string) => +item)
};

export class Parser {
	constructor(namespace: any, context?: Context) {
		this.context = context || defaultContext;
		this.namespace = namespace._cxml[0];
	}

	attach<CustomHandler extends HandlerInstance>(handler: { new(): CustomHandler; }) {
		var proto = handler.prototype as CustomHandler;
		var realHandler = (handler as TypeClass).type.handler;
		var realProto = realHandler.prototype as CustomHandler;

		for(var key of Object.keys(proto)) {
			realProto[key] = proto[key];
		}

		realHandler.custom = true;
		if(proto.before) realHandler.before = true;
		if(proto.after) realHandler.after = true;
	}

	parse(stream: stream.Readable) {
		var xml = sax.createStream(true, { position: true });
		var type = this.namespace.doc.getType();

		var state = new State(null, null, type, new type.handler());
		var rootState = state;
		state.addNamespace('', this.namespace);

		xml.on('opentag', (node: sax.Tag) => {
			var attrTbl = node.attributes;
			var attr: string;
			var nodePrefix = '';
			var name = node.name;
			var splitter = name.indexOf(':');
			var item: HandlerInstance = null;

			if(splitter >= 0) {
				nodePrefix = name.substr(0, splitter);
				name = name.substr(splitter + 1);<commit_msg>Always import xml namespace.<commit_after>// This file is part of cxml, copyright (c) 2016 BusFaster Ltd.
// Released under the MIT license, see LICENSE.

import * as stream from 'stream';
import * as Promise from 'bluebird';
import * as sax from 'sax';

import {Context} from './Context';
import {Namespace} from './Namespace';
import {Type, TypeClass, HandlerInstance} from './Type';
import {Member} from './Member';
import {State} from './State';
import {defaultContext} from '../importer/JS';

var converterTbl: { [type: string]: (item: string) => any } = {
	Date: ((item: string) => {
		var dateParts = item.match(/([0-9]+)-([0-9]+)-([0-9]+)(?:T([0-9]+):([0-9]+):([0-9]+)(\.[0-9]+)?)?(?:Z|([+-][0-9]+):([0-9]+))?/);
		var offset = +(dateParts[8] || '0') * 60 + +(dateParts[9] || '0');

		var date = new Date(
			+dateParts[1],
			+dateParts[2] - 1,
			+dateParts[3],
			+(dateParts[4] || '0'),
			+(dateParts[5] || '0'),
			+(dateParts[6] || '0'),
			+(dateParts[7] || '0') * 1000
		);

		offset += date.getTimezoneOffset();
		date.setTime(date.getTime() - offset * 60000);

		return(date);
	}),
	boolean: ((item: string) => !!item),
	string: ((item: string) => item),
	number: ((item: string) => +item)
};

export class Parser {
	constructor(namespace: any, context?: Context) {
		this.context = context || defaultContext;
		this.namespace = namespace._cxml[0];
	}

	attach<CustomHandler extends HandlerInstance>(handler: { new(): CustomHandler; }) {
		var proto = handler.prototype as CustomHandler;
		var realHandler = (handler as TypeClass).type.handler;
		var realProto = realHandler.prototype as CustomHandler;

		for(var key of Object.keys(proto)) {
			realProto[key] = proto[key];
		}

		realHandler.custom = true;
		if(proto.before) realHandler.before = true;
		if(proto.after) realHandler.after = true;
	}

	parse(stream: stream.Readable) {
		var xml = sax.createStream(true, { position: true });
		var type = this.namespace.doc.getType();

		var xmlSpace = this.context.registerNamespace('http://www.w3.org/XML/1998/namespace');
		var state = new State(null, null, type, new type.handler());
		var rootState = state;

		state.addNamespace('', this.namespace);
		if(xmlSpace) state.addNamespace('xml', xmlSpace);

		xml.on('opentag', (node: sax.Tag) => {
			var attrTbl = node.attributes;
			var attr: string;
			var nodePrefix = '';
			var name = node.name;
			var splitter = name.indexOf(':');
			var item: HandlerInstance = null;

			if(splitter >= 0) {
				nodePrefix = name.substr(0, splitter);
				name = name.substr(splitter + 1);
{'fraction_non_alphanumeric': 0.19688700222356983, 'fraction_numerical': 0.022437841115827774, 'mean_word_length': 4.043832823649337, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1631288', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1674, 'n_words': 434}