<commit_before>/* @flow */

import Promise from 'bluebird'
import async from 'async'
import crypto from 'crypto'
import fs from 'fs'

import type { Callback } from '../utils/func'

// Get checksum for given file
const computeChecksum = (filePath: string, callback: Callback) => {
  const stream = fs.createReadStream(filePath)
  const checksum = crypto.createHash('md5')
  checksum.setEncoding('base64')
  stream.on('end', function () {
    checksum.end()
    callback(null, checksum.read())
  })
  stream.on('error', function (err) {
    checksum.end()
    callback(err)
  })
  stream.pipe(checksum)
}

const retryComputeChecksum = (filePath: string, callback: Callback) => {
  async.retry({
    times: 5,
    // retry after 1, 2, 4, 8, 16 seconds
    interval: (count) => 500 * Math.pow(2, count),
    errorFilter: err => err.code == 'EBUSY'
  }, (cb) => { computeChecksum(filePath, cb) }, callback)
}

export type Checksumer = {
  push: (filePath: string) => Promise<string>,
  kill: () => void
}

export const init = (): Checksumer => {
  // Use a queue for checksums to avoid computing many checksums at the
  // same time. It's better for performance (hard disk are faster with
  // linear readings).
  const queue = Promise.promisifyAll(async.queue(retryComputeChecksum))

  return {
    push (filePath: string): Promise<string> {
      return queue.pushAsync(filePath)
    },

    kill: queue.kill
  }
}
<commit_msg>core/local/checksumer: Fix err code equality comparison (should be strict)<commit_after>/* @flow */

import Promise from 'bluebird'
import async from 'async'
import crypto from 'crypto'
import fs from 'fs'

import type { Callback } from '../utils/func'

// Get checksum for given file
const computeChecksum = (filePath: string, callback: Callback) => {
  const stream = fs.createReadStream(filePath)
  const checksum = crypto.createHash('md5')
  checksum.setEncoding('base64')
  stream.on('end', function () {
    checksum.end()
    callback(null, checksum.read())
  })
  stream.on('error', function (err) {
    checksum.end()
    callback(err)
  })
  stream.pipe(checksum)
}

const retryComputeChecksum = (filePath: string, callback: Callback) => {
  async.retry({
    times: 5,
    // retry after 1, 2, 4, 8, 16 seconds
    interval: (count) => 500 * Math.pow(2, count),
    errorFilter: err => err.code === 'EBUSY'
  }, (cb) => { computeChecksum(filePath, cb) }, callback)
}

export type Checksumer = {
  push: (filePath: string) => Promise<string>,
  kill: () => void
}

export const init = (): Checksumer => {
  // Use a queue for checksums to avoid computing many checksums at the
  // same time. It's better for performance (hard disk are faster with
  // linear readings).
  const queue = Promise.promisifyAll(async.queue(retryComputeChecksum))

  return {
    push (filePath: string): Promise<string> {
      return queue.pushAsync(filePath)
    },

    kill: queue.kill
  }
}

{'fraction_non_alphanumeric': 0.13951890034364262, 'fraction_numerical': 0.009621993127147767, 'mean_word_length': 3.6576, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4298600', 'n_tokens_mistral': 954, 'n_tokens_neox': 931, 'n_words': 272}