<commit_before>declare const Components: any

import _kuroshiro = require('kuroshiro')
import _kuromojiLoader = require('kuromoji/src/loader/NodeDictionaryLoader')
import { debug } from '../debug.ts'
import { Preferences as Prefs } from '../prefs.ts'

_kuromojiLoader.prototype.loadArrayBuffer = function(url, callback) { // tslint:disable-line:only-arrow-functions
  url = url.replace(/^resource:\/\/?/, 'resource://') // kuromoji replaces the double-slash for some reason
  debug('kuromoji: loading', url)
  const xhr = Components.classes['@mozilla.org/xmlextras/xmlhttprequest;1'].createInstance()

  xhr.open('GET', url.replace(/\.gz$/, ''), true)
  xhr.responseType = 'arraybuffer'

  xhr.onload = function() {
    debug('kuromoji: loaded', url, this.status)
    const err = this.status > 0 && this.status !== 200 // tslint:disable-line:no-magic-numbers
    callback(err ? new Error(xhr.statusText) : null, err ? null : this.response)
  }

  xhr.onerror = function(err) { // tslint:disable-line:only-arrow-functions
    err = new Error(`could not load ${url}: ${err}`)
    debug('kuromoji: load failed', url, err)
    callback(err, null)
  }

  xhr.send()
}

export let kuroshiro = new class {
  public enabled = false

  public init() {
    debug('kuroshiro: initializing...')

    return new Promise((resolve, reject) => {
      if (!Prefs.get('kuroshiro')) {
        debug('kuroshiro: disabled')
        resolve()
        return
      }

      _kuroshiro.init({ dicPath: 'resource://zotero-better-bibtex/kuromoji' }, err => {
        if (err) {
          debug('kuroshiro: initializing failed')
          reject(err)
        } else {
          debug('kuroshiro: ready')
          this.enabled = true
          resolve()
        }
      })
    })
  }

  public convert(str, options) {
    if (!this.enabled) throw new Error('kuroshoro not initialized')
    if (!str) return str
    if (_kuroshiro.hasHiragana(str) || _kuroshiro.hasKatakana(str) || _kuroshiro.hasKanji(str)) return _kuroshiro.convert(str, options)
    return str
  }
}<commit_msg>faster kuroshiro test<commit_after>declare const Components: any

import _kuroshiro = require('kuroshiro')
import _kuromojiLoader = require('kuromoji/src/loader/NodeDictionaryLoader')
import { debug } from '../debug.ts'
import { Preferences as Prefs } from '../prefs.ts'

_kuromojiLoader.prototype.loadArrayBuffer = function(url, callback) { // tslint:disable-line:only-arrow-functions
  url = url.replace(/^resource:\/\/?/, 'resource://') // kuromoji replaces the double-slash for some reason
  debug('kuromoji: loading', url)
  const xhr = Components.classes['@mozilla.org/xmlextras/xmlhttprequest;1'].createInstance()

  xhr.open('GET', url.replace(/\.gz$/, ''), true)
  xhr.responseType = 'arraybuffer'

  xhr.onload = function() {
    debug('kuromoji: loaded', url, this.status)
    const err = this.status > 0 && this.status !== 200 // tslint:disable-line:no-magic-numbers
    callback(err ? new Error(xhr.statusText) : null, err ? null : this.response)
  }

  xhr.onerror = function(err) { // tslint:disable-line:only-arrow-functions
    err = new Error(`could not load ${url}: ${err}`)
    debug('kuromoji: load failed', url, err)
    callback(err, null)
  }

  xhr.send()
}

export let kuroshiro = new class {
  public enabled = false

  public init() {
    debug('kuroshiro: initializing...')

    return new Promise((resolve, reject) => {
      if (!Prefs.get('kuroshiro')) {
        debug('kuroshiro: disabled')
        resolve()
        return
      }

      _kuroshiro.init({ dicPath: 'resource://zotero-better-bibtex/kuromoji' }, err => {
        if (err) {
          debug('kuroshiro: initializing failed')
          reject(err)
        } else {
          debug('kuroshiro: ready')
          this.enabled = true
          resolve()
        }
      })
    })
  }

  public convert(str, options) {
    if (!this.enabled) throw new Error('kuroshoro not initialized')
    if (!str) return str
    if (this.isJapanese(str)) return _kuroshiro.convert(str, options)
    return str
  }

  private isJapanese(str) {
    for(const c of str) {
      if (_kuroshiro.isKanji(c)) return true
      if (_kuroshiro.isHiragana(c)) return true
      if (_kuroshiro.isKatakana(c)) return true
    }
    return false
  }
}
{'fraction_non_alphanumeric': 0.14774057597752283, 'fraction_numerical': 0.0023413720440177946, 'mean_word_length': 3.111645813282002, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5809247', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1414, 'n_words': 308}