<commit_before>class Handler {
  constructor(callback, options) {
    this.callback = callback;
    this.obj = {};
    this.stack = [this.obj];
    const defaultOptions = {
      normalizeWhitespace: true,
    };
    this.options = Object.assign(defaultOptions, options);
  }
  onopentag(name, attrs) {
    const ref = this.last();
    const obj = {
      $: attrs || {},
      _: '',
    };
    if (ref[name]) {
      if (!Array.isArray(ref[name])) {
        ref[name] = [ref[name]];
      }
      ref[name].push(obj);
    } else {
      ref[name] = obj;
    }
    this.stack.push(obj);
  }
  onclosetag() {
    this.stack.pop();
  }
  ontext(text) {
    if (this.options.normalizeWhitespace) {
      this.last()._ = text.replace(/\s+/g, ' ').trim();
    } else {
      this.last()._ = text;
    }
  }
  onend() {
    this.callback(null, this.obj);
  }<commit_msg>now the text is appended to the end<commit_after>class Handler {
  constructor(callback, options) {
    this.callback = callback;
    this.obj = { _: '' };
    this.stack = [this.obj];
    const defaultOptions = {
      normalizeWhitespace: true,
    };
    this.options = Object.assign(defaultOptions, options);
  }
  onopentag(name, attrs) {
    const ref = this.last();
    const obj = {
      $: attrs || {},
      _: '',
    };
    if (ref[name]) {
      if (!Array.isArray(ref[name])) {
        ref[name] = [ref[name]];
      }
      ref[name].push(obj);
    } else {
      ref[name] = obj;
    }
    this.stack.push(obj);
  }
  onclosetag() {
    this.stack.pop();
  }
  ontext(text) {
    const last = this.last();
    if (this.options.normalizeWhitespace) {
      last._ += text.replace(/\s+/g, ' ').trim();
    } else {
      last._ += text;
    }
  }
  onend() {
    this.callback(null, this.obj);
  }
{'fraction_non_alphanumeric': 0.18007878446820483, 'fraction_numerical': 0.0, 'mean_word_length': 2.175, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2043219', 'n_tokens_mistral': 640, 'n_tokens_neox': 633, 'n_words': 108}