<commit_before>let binding

if (process.env.SUPERSTRING_USE_BROWSER_VERSION) {
  binding = require('./browser');

  const {TextBuffer} = binding
  const {search, searchSync} = TextBuffer.prototype

  TextBuffer.prototype.searchSync = function (pattern) {
    if (pattern.source) pattern = pattern.source
    const result = searchSync.call(this, pattern)
    if (typeof result === 'string') {
      throw new Error(result);
    } else {
      return result
    }
  }

  TextBuffer.prototype.search = function (pattern) {
    return new Promise(resolve => resolve(this.searchSync(pattern)))
  }

} else {
  try {
    binding = require('./build/Release/superstring.node')
  } catch (e1) {
    try {
      binding = require('./build/Debug/superstring.node')
    } catch (e2) {
      throw e1
    }
  }

  const {TextBuffer, TextWriter, TextReader} = binding
  const {load, reload, save, search} = TextBuffer.prototype

  for (const methodName of ['load', 'reload']) {
    const method = TextBuffer.prototype[methodName]
    TextBuffer.prototype[methodName] = function (source, encoding, progressCallback) {
      if (typeof encoding !== 'string') {
        progressCallback = encoding
        encoding = 'UTF8'
      }

      encoding = normalizeEncoding(encoding)

      return new Promise((resolve, reject) => {
        const completionCallback = (error, result) => {
          error ? reject(error) : resolve(result)
        }

        if (typeof source === 'string') {
          const filePath = source
          method.call(this, completionCallback, progressCallback, filePath, encoding)
        } else {
          const stream = source
          const writer = new TextWriter(encoding)
          stream.on('data', (data) => writer.write(data))
          stream.on('error', reject)
          stream.on('end', () => {
            writer.end()
            method.call(this, completionCallback, progressCallback, writer)
          })
        }
      })
    }
  }

  TextBuffer.prototype.save = function (destination, encoding = 'UTF8') {
    const CHUNK_SIZE = 10 * 1024

    encoding = normalizeEncoding(encoding)

    return new Promise((resolve, reject) => {
      if (typeof destination === 'string') {
        const filePath = destination
        save.call(this, filePath, encoding, (error) => {
          error ? reject(error) : resolve()
        })
      } else {
        const stream = destination
        stream.on('error', reject)

        const reader = new TextReader(this, encoding)
        const buffer = Buffer.allocUnsafe(CHUNK_SIZE)
        writeToStream()

        function writeToStream () {
          const bytesRead = reader.read(buffer)
          if (bytesRead > 0) {
            stream.write(buffer.slice(0, bytesRead), writeToStream)
          } else {
            stream.end(() => {
              reader.end()
              resolve()
            })
          }
        }
      }
    })
  }

  TextBuffer.prototype.search = function (pattern) {
    return new Promise((resolve, reject) => {
      search.call(this, pattern, (error, result) => {
        error ?
          reject(error) :
          resolve(result)
      })
    })
  }
}

function normalizeEncoding(encoding) {
  return encoding.toUpperCase()
    .replace(/[^A-Z\d]/g, '')
    .replace(/^(UTF|UCS|ISO|WINDOWS|KOI8)(\w)/, '$1-$2')
    .replace(/^(ISO-8859)(\d)/, '$1-$2')
    .replace(/^(SHIFT)(\w)/, '$1_$2')
}

module.exports = {
  TextBuffer: binding.TextBuffer,
  Patch: binding.Patch,
  MarkerIndex: binding.MarkerIndex,
  BufferOffsetIndex: binding.BufferOffsetIndex,
}<commit_msg>Normalize EUC-JP encoding name in a way that works w/ win-iconv<commit_after>let binding

if (process.env.SUPERSTRING_USE_BROWSER_VERSION) {
  binding = require('./browser');

  const {TextBuffer} = binding
  const {search, searchSync} = TextBuffer.prototype

  TextBuffer.prototype.searchSync = function (pattern) {
    if (pattern.source) pattern = pattern.source
    const result = searchSync.call(this, pattern)
    if (typeof result === 'string') {
      throw new Error(result);
    } else {
      return result
    }
  }

  TextBuffer.prototype.search = function (pattern) {
    return new Promise(resolve => resolve(this.searchSync(pattern)))
  }

} else {
  try {
    binding = require('./build/Release/superstring.node')
  } catch (e1) {
    try {
      binding = require('./build/Debug/superstring.node')
    } catch (e2) {
      throw e1
    }
  }

  const {TextBuffer, TextWriter, TextReader} = binding
  const {load, reload, save, search} = TextBuffer.prototype

  for (const methodName of ['load', 'reload']) {
    const method = TextBuffer.prototype[methodName]
    TextBuffer.prototype[methodName] = function (source, encoding, progressCallback) {
      if (typeof encoding !== 'string') {
        progressCallback = encoding
        encoding = 'UTF8'
      }

      encoding = normalizeEncoding(encoding)

      return new Promise((resolve, reject) => {
        const completionCallback = (error, result) => {
          error ? reject(error) : resolve(result)
        }

        if (typeof source === 'string') {
          const filePath = source
          method.call(this, completionCallback, progressCallback, filePath, encoding)
        } else {
          const stream = source
          const writer = new TextWriter(encoding)
          stream.on('data', (data) => writer.write(data))
          stream.on('error', reject)
          stream.on('end', () => {
            writer.end()
            method.call(this, completionCallback, progressCallback, writer)
          })
        }
      })
    }
  }

  TextBuffer.prototype.save = function (destination, encoding = 'UTF8') {
    const CHUNK_SIZE = 10 * 1024

    encoding = normalizeEncoding(encoding)

    return new Promise((resolve, reject) => {
      if (typeof destination === 'string') {
        const filePath = destination
        save.call(this, filePath, encoding, (error) => {
          error ? reject(error) : resolve()
        })
      } else {
        const stream = destination
        stream.on('error', reject)

        const reader = new TextReader(this, encoding)
        const buffer = Buffer.allocUnsafe(CHUNK_SIZE)
        writeToStream()

        function writeToStream () {
          const bytesRead = reader.read(buffer)
          if (bytesRead > 0) {
            stream.write(buffer.slice(0, bytesRead), writeToStream)
          } else {
            stream.end(() => {
              reader.end()
              resolve()
            })
          }
        }
      }
    })
  }

  TextBuffer.prototype.search = function (pattern) {
    return new Promise((resolve, reject) => {
      search.call(this, pattern, (error, result) => {
        error ?
          reject(error) :
          resolve(result)
      })
    })
  }
}

function normalizeEncoding(encoding) {
  return encoding.toUpperCase()
    .replace(/[^A-Z\d]/g, '')
    .replace(/^(UTF|UCS|ISO|WINDOWS|KOI8|EUC)(\w)/, '$1-$2')
    .replace(/^(ISO-8859)(\d)/, '$1-$2')
    .replace(/^(SHIFT)(\w)/, '$1_$2')
}

module.exports = {
  TextBuffer: binding.TextBuffer,
  Patch: binding.Patch,
  MarkerIndex: binding.MarkerIndex,
  BufferOffsetIndex: binding.BufferOffsetIndex,
}
{'fraction_non_alphanumeric': 0.13364823562100583, 'fraction_numerical': 0.0066685190330647405, 'mean_word_length': 2.4150853889943074, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6007495', 'n_tokens_mistral': 2109, 'n_tokens_neox': 2032, 'n_words': 458}