<commit_before>import { parser as saxParser } from 'sax';
import Queue from './queue.js';

function isAllowedType(type) {
  switch(type) {
    case 'text': case 'opentag': case 'closetag': case 'doctype':
    case 'processinginstruction': case 'attribute': case 'comment':
    case 'opencdata': case 'closecdata': case 'opennamespace':
    case 'closenamespace': return true;
    default: return false;
  }
}

class XMLPulley {
  constructor(xml, options) {
    if(!options) options = {};
    let types = options.types || ['opentag', 'closetag', 'text'];
    let queue = this.queue = new Queue();
    let parser = saxParser(true, {
      xmlns: options.xmlns,
      position: false
    });
    let skipWS = options.skipWhitespaceOnly;
    let trim = options.trim, normalize = options.normalize;
    function textOpts(t) {
      if(trim)
        t = t.trim();
      if(normalize)
        t = t.replace(/\s+/g, " ")
      return t;
    }
    let text = null, rawText = null;
    let flushText = () => {
      if(text !== null) {
        this.queue.enqueue({
          type: 'text',
          text: text,
          rawText: rawText
        });
        text = rawText = null;
      }
    };
    parser.onerror = (err) => {
      throw err;
    };
    types.forEach((type) => {
      if(type === 'text') {
        parser.ontext = (t) => {
          if(!skipWS || /\S/.test(t)) {
            let pt = textOpts(t);
            if(text) {
              text += pt; rawText += t;
            } else {
              text = pt; rawText = t;
            }
          }
        };
        parser.oncdata = (t) => {
          if(text) {
            text += t; rawText += t;
          } else {
            text = rawText = t;
          }
        };
      } else if(type === 'comment') {
        parser.oncomment = (t) => {
          flushText();
          queue.enqueue({
            type: 'comment',
            text: textOpts(t),
            rawText: t
          });
        }
      } else if(type === 'doctype') {
        parser.ondoctype = (t) => {
          flushText();
          queue.enqueue({
            type: 'doctype',
            text: t
          });
        }
      } else if(type === 'closetag') {
        parser.onclosetag = (t) => {
          flushText();
          queue.enqueue({
            type: 'closetag',
            name: t
          });
        }
      } else if(type === 'opencdata' || type === 'closecdata' ||
                type === 'opennamespace' || type === 'closenamespace') {
        parser['on'+type] = (data) => {
          flushText();
          queue.enqueue({ type: type });
        }
      } else if(isAllowedType(type)) {
        parser['on'+type] = (data) => {
          flushText();
          data.type = type;
          queue.enqueue(data);
        }
      } else {
        throw new Error(`${type} isn't an allowed type!`);
      }
    });
    parser.write(xml).close();
    flushText();
  }
  next() {
    return this.queue.dequeue();
  }
  peek() {
    return this.queue.peek();
  }
  expect(type, error) {
    let out = this.peek();
    assertType(out, type, error);
    this.next();
    return out;
  }
  expectName(name, type, wrongNameError, wrongTypeError) {
    type = type || 'opentag';
    let out = this.peek();
    assertType(out, type, wrongTypeError || wrongNameError);
    assertName(out, name, wrongNameError);
    this.next();
    return out;
  }
  nextAll(callback, endType, tagName) {
    endType = endType || 'closetag';
    let node;
    while((node = this.peek()) && node.type !== endType) {
      if(callback(this))
        break;
    }
    if(tagName)
      assertName(this.next(), tagName);
  }
  expectAll(callback, type, endType, tagName) {
    endType = endType || 'closetag';
    let node;
    while((node = this.peek()) && node.type !== endType) {
      assertType(node, type);
      if(callback(this))
        break;
    }
    if(tagName)
      assertName(this.next(), tagName);
  }
}

export function makePulley(xml, options) {
  return new XMLPulley(xml, options);
}

export function assertType(node, type, error) {
  if(node === undefined) {
    throw new Error(`Expected ${type}; got end of file!`);
  } else if(node.type !== type) {
    throw new Error(`Expected ${type}; got ${node.type}: ${nodeRepr(node)}!`);
  }
}
export function assertName(tag, name, error) {
  if(tag.name !== name) {
    throw error ||
          new Error(`${tag.type} had name ${tag.name} instead of ${name}!`);
  }
}

export function nodeRepr(tag) {
  switch(tag.type) {
    case 'text': case 'comment': return `"${tag.text}"`;
    case 'opentag': return `<${tag.name}>`;
    case 'closetag': return `</${tag.name}>`;
    case 'doctype': return `<!${tag.text}>`;
    case 'processinginstruction': return `<?${tag.name} ${tag.body}?>`;
    case 'attribute': return `${tag.name}="${tag.value}"`;<commit_msg>Removed tagName from nextAll() and expectAll()<commit_after>import { parser as saxParser } from 'sax';
import Queue from './queue.js';

function isAllowedType(type) {
  switch(type) {
    case 'text': case 'opentag': case 'closetag': case 'doctype':
    case 'processinginstruction': case 'attribute': case 'comment':
    case 'opencdata': case 'closecdata': case 'opennamespace':
    case 'closenamespace': return true;
    default: return false;
  }
}

class XMLPulley {
  constructor(xml, options) {
    if(!options) options = {};
    let types = options.types || ['opentag', 'closetag', 'text'];
    let queue = this.queue = new Queue();
    let parser = saxParser(true, {
      xmlns: options.xmlns,
      position: false
    });
    let skipWS = options.skipWhitespaceOnly;
    let trim = options.trim, normalize = options.normalize;
    function textOpts(t) {
      if(trim)
        t = t.trim();
      if(normalize)
        t = t.replace(/\s+/g, " ")
      return t;
    }
    let text = null, rawText = null;
    let flushText = () => {
      if(text !== null) {
        this.queue.enqueue({
          type: 'text',
          text: text,
          rawText: rawText
        });
        text = rawText = null;
      }
    };
    parser.onerror = (err) => {
      throw err;
    };
    types.forEach((type) => {
      if(type === 'text') {
        parser.ontext = (t) => {
          if(!skipWS || /\S/.test(t)) {
            let pt = textOpts(t);
            if(text) {
              text += pt; rawText += t;
            } else {
              text = pt; rawText = t;
            }
          }
        };
        parser.oncdata = (t) => {
          if(text) {
            text += t; rawText += t;
          } else {
            text = rawText = t;
          }
        };
      } else if(type === 'comment') {
        parser.oncomment = (t) => {
          flushText();
          queue.enqueue({
            type: 'comment',
            text: textOpts(t),
            rawText: t
          });
        }
      } else if(type === 'doctype') {
        parser.ondoctype = (t) => {
          flushText();
          queue.enqueue({
            type: 'doctype',
            text: t
          });
        }
      } else if(type === 'closetag') {
        parser.onclosetag = (t) => {
          flushText();
          queue.enqueue({
            type: 'closetag',
            name: t
          });
        }
      } else if(type === 'opencdata' || type === 'closecdata' ||
                type === 'opennamespace' || type === 'closenamespace') {
        parser['on'+type] = (data) => {
          flushText();
          queue.enqueue({ type: type });
        }
      } else if(isAllowedType(type)) {
        parser['on'+type] = (data) => {
          flushText();
          data.type = type;
          queue.enqueue(data);
        }
      } else {
        throw new Error(`${type} isn't an allowed type!`);
      }
    });
    parser.write(xml).close();
    flushText();
  }
  next() {
    return this.queue.dequeue();
  }
  peek() {
    return this.queue.peek();
  }
  expect(type, error) {
    let out = this.peek();
    assertType(out, type, error);
    this.next();
    return out;
  }
  expectName(name, type, wrongNameError, wrongTypeError) {
    type = type || 'opentag';
    let out = this.peek();
    assertType(out, type, wrongTypeError || wrongNameError);
    assertName(out, name, wrongNameError);
    this.next();
    return out;
  }
  nextAll(callback, endType) {
    endType = endType || 'closetag';
    let node;
    while((node = this.peek()) && node.type !== endType) {
      if(callback(this))
        break;
    }
  }
  expectAll(callback, type, endType) {
    endType = endType || 'closetag';
    let node;
    while((node = this.peek()) && node.type !== endType) {
      assertType(node, type);
      if(callback(this))
        break;
    }
  }
}

export function makePulley(xml, options) {
  return new XMLPulley(xml, options);
}

export function assertType(node, type, error) {
  if(node === undefined) {
    throw new Error(`Expected ${type}; got end of file!`);
  } else if(node.type !== type) {
    throw new Error(`Expected ${type}; got ${node.type}: ${nodeRepr(node)}!`);
  }
}
export function assertName(tag, name, error) {
  if(tag.name !== name) {
    throw error ||
          new Error(`${tag.type} had name ${tag.name} instead of ${name}!`);
  }
}

export function nodeRepr(tag) {
  switch(tag.type) {
    case 'text': case 'comment': return `"${tag.text}"`;
    case 'opentag': return `<${tag.name}>`;
    case 'closetag': return `</${tag.name}>`;
    case 'doctype': return `<!${tag.text}>`;
    case 'processinginstruction': return `<?${tag.name} ${tag.body}?>`;
    case 'attribute': return `${tag.name}="${tag.value}"`;
{'fraction_non_alphanumeric': 0.1691842900302115, 'fraction_numerical': 0.0, 'mean_word_length': 2.0303030303030303, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2178026', 'n_tokens_mistral': 3122, 'n_tokens_neox': 3112, 'n_words': 714}