<commit_before>export class NullDelimiterParser<T extends { [name: string]: string }> {
  private readonly keys = new Array<string>()
  public readonly format: string = ''

  public constructor(fields: T, delimiterString = '%x00') {
    for (const [key, value] of Object.entries(fields)) {
      this.keys.push(key)
      this.format = `${this.format}${value}${delimiterString}`
    }
  }

  public *parse(output: string): Iterable<T> {
    const { keys } = this

    let head = 0
    let tail = 0
    let fieldIndex = 0
    let entry = {} as T

    while (head < output.length && (tail = output.indexOf('\0', head)) !== -1) {
      entry[keys[fieldIndex % keys.length]] = output.substring(head, tail)

      head = tail + 1
      fieldIndex++

      if (fieldIndex % keys.length === 0) {
        yield entry
        entry = {} as T

        if (head < output.length) {
          if (output[head] !== '\0' && output[head] !== '\n') {
            const char = output.charCodeAt(head)
            throw new Error(
              `Unexpected character at end of record: '0x${char.toString(16)}'`
            )
          }
          head++
        }
      }
    }
  }
}<commit_msg>Iterable was a pie in the sky idea anyway<commit_after>export class NullDelimiterParser<T extends { [name: string]: string }> {
  private readonly keys = new Array<string>()
  public readonly format: string = ''

  public constructor(fields: T, delimiterString = '%x00') {
    for (const [key, value] of Object.entries(fields)) {
      this.keys.push(key)
      this.format = `${this.format}${value}${delimiterString}`
    }
  }

  public parse(output: string): T[] {
    const { keys } = this
    const entries = new Array<T>()

    let head = 0
    let tail = 0
    let fieldIndex = 0
    let entry = {} as T

    while (head < output.length && (tail = output.indexOf('\0', head)) !== -1) {
      entry[keys[fieldIndex % keys.length]] = output.substring(head, tail)

      head = tail + 1
      fieldIndex++

      if (fieldIndex % keys.length === 0) {
        entries.push(entry)
        entry = {} as T

        if (head < output.length) {
          if (output[head] !== '\0' && output[head] !== '\n') {
            const char = output.charCodeAt(head)
            throw new Error(
              `Unexpected character at end of record: '0x${char.toString(16)}'`
            )
          }
          head++
        }
      }
    }
    return entries
  }
}
{'fraction_non_alphanumeric': 0.14591039868475134, 'fraction_numerical': 0.010686395396629675, 'mean_word_length': 2.1858638743455496, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4878668', 'n_tokens_mistral': 756, 'n_tokens_neox': 734, 'n_words': 205}