<commit_before>declare const Zotero: any
declare const window: any

import ETA = require('node-eta')
import { kuroshiro } from './key-manager/kuroshiro'

import * as log from './debug'
import { timeout } from './timeout'
import { flash } from './flash'
import { Events } from './events'
import { arXiv } from './arXiv'

import * as ZoteroDB from './db/zotero'

import { getItemsAsync } from './get-items-async'

import { Preferences as Prefs } from './prefs'
import * as Citekey from './key-manager/get-set'
import { Formatter } from './key-manager/formatter'
import { DB } from './db/main'
import { AutoExport } from './auto-export'

// export singleton: https://k94n.com/es6-modules-single-instance-pattern
export let KeyManager = new class { // tslint:disable-line:variable-name
  public keys: any
  public query: {
    field: { extra?: number }
    type: {
      note?: number,
      attachment?: number
    }
  }

  private itemObserverDelay: number = Prefs.get('itemObserverDelay')
  private scanning: any[]

  public async pin(ids, inspireHEP = false) {
    ids = this.expandSelection(ids)
    log.debug('KeyManager.pin', ids)

    const inspireSearch = 'http://inspirehep.net/search?of=recjson&ot=system_control_number&p='

    for (const item of await getItemsAsync(ids)) {
      if (item.isNote() || item.isAttachment()) continue

      const parsed = Citekey.get(item.getField('extra'))
      if (parsed.pinned && !inspireHEP) continue

      try {
        let citekey

        if (inspireHEP) {
          let key = item.getField('DOI')
          if (!key) {
            for (const line of parsed.extra.split('\n')) {
              if (line.toLowerCase().startsWith('DOI:')) {
                key = line.substr('DOI:'.length).trim()
                if (key) break
              }

              const arxiv = arXiv.parse(line.trim())
              if (arxiv) {
                key = arxiv.id
                break
              }
            }
          }

          if (!key) throw new Error(`No DOI or arXiv ID for ${item.getField('title')}`)

          const results = JSON.parse((await Zotero.HTTP.request('GET', inspireSearch + encodeURIComponent(key))).responseText)
          if (results.length !== 1) throw new Error(`Expected 1 inspire result for ${item.getField('title')}, got ${results.length}`)

          citekey = results[0].system_control_number.find(i => i.institute.endsWith('TeX') && i.value).value

        } else {

          citekey = this.get(item.id).citekey || this.update(item)
        }

        item.setField('extra', Citekey.set(parsed.extra, citekey))
        await item.saveTx() // this should cause an update and key registration
      } catch (err) {
        log.error('KeyManager.pin', err)
      }
    }
  }

  public async unpin(ids) {
    ids = this.expandSelection(ids)
    log.debug('KeyManager.unpin', ids)

    for (const item of await getItemsAsync(ids)) {
      if (item.isNote() || item.isAttachment()) continue<commit_msg>cleanup<commit_after>declare const Zotero: any
declare const window: any

import ETA = require('node-eta')
import { kuroshiro } from './key-manager/kuroshiro'

import * as log from './debug'
import { timeout } from './timeout'
import { flash } from './flash'
import { Events } from './events'
import { arXiv } from './arXiv'
import { extract as varExtract } from './var-extract'

import * as ZoteroDB from './db/zotero'

import { getItemsAsync } from './get-items-async'

import { Preferences as Prefs } from './prefs'
import * as Citekey from './key-manager/get-set'
import { Formatter } from './key-manager/formatter'
import { DB } from './db/main'
import { AutoExport } from './auto-export'

// export singleton: https://k94n.com/es6-modules-single-instance-pattern
export let KeyManager = new class { // tslint:disable-line:variable-name
  public keys: any
  public query: {
    field: { extra?: number }
    type: {
      note?: number,
      attachment?: number
    }
  }

  private itemObserverDelay: number = Prefs.get('itemObserverDelay')
  private scanning: any[]

  public async pin(ids, inspireHEP = false) {
    ids = this.expandSelection(ids)
    log.debug('KeyManager.pin', ids)

    const inspireSearch = 'http://inspirehep.net/search?of=recjson&ot=system_control_number&p='

    for (const item of await getItemsAsync(ids)) {
      if (item.isNote() || item.isAttachment()) continue

      try {
        let parsed
        let citekey

        if (inspireHEP) {
          parsed = varExtract({ extra: item.getField('extra') })
          const key = parsed.extraKeys.csl.DOI || item.getField('DOI') || (arXiv.parse(parsed.extraKeys.kv.arxiv) || { id: null}).id
          if (!key) throw new Error(`No DOI or arXiv ID for ${item.getField('title')}`)

          const results = JSON.parse((await Zotero.HTTP.request('GET', inspireSearch + encodeURIComponent(key))).responseText)
          if (results.length !== 1) throw new Error(`Expected 1 inspire result for ${item.getField('title')}, got ${results.length}`)

          citekey = results[0].system_control_number.find(i => i.institute.endsWith('TeX') && i.value).value

        } else {
          parsed = Citekey.get(item.getField('extra'))
          if (parsed.pinned) continue

          citekey = this.get(item.id).citekey || this.update(item)
        }

        item.setField('extra', Citekey.set(parsed.extra, citekey))
        await item.saveTx() // this should cause an update and key registration
      } catch (err) {
        log.error('KeyManager.pin', err)
      }
    }
  }

  public async unpin(ids) {
    ids = this.expandSelection(ids)
    log.debug('KeyManager.unpin', ids)

    for (const item of await getItemsAsync(ids)) {
      if (item.isNote() || item.isAttachment()) continue
{'fraction_non_alphanumeric': 0.14188718662952646, 'fraction_numerical': 0.0020891364902506965, 'mean_word_length': 2.772160210111622, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4973529', 'n_tokens_mistral': 1812, 'n_tokens_neox': 1735, 'n_words': 447}