<commit_before>
import app from 'app'
import path from 'path'
import fstream from 'fstream'
import Promise from 'bluebird'

import {partial} from 'underscore'
import needle from 'needle'

import extract from '../tasks/extract'
import os from './os'
let log = require('./log')('ibrew')

let self = {
  formulas: {
    '7za': {
      format: 'executable',
      check: () => {
        if (self.os() === 'linux') {
          // TODO: add link to doc page too
          let msg = '7-zip missing: 7za must be in $PATH\n(Try installing p7zip-full)'
          throw new Error(msg)
        }
      },
      version_check: {
        args: [],
        parser: /p7zip Version ([0-9.]*)/
      }
    },
    'butler': {
      format: '7z',
      version_check: {
        args: ['version'],
        parser: /butler version ([0-9a-z.v]*)/
      }
    }
  },

  bin_path: () => {
    return path.join(app.getPath('userData'), 'bin')
  },

  root_url: () => {
    return 'https://misc.amos.me'
  },

  ext: () => {
    if (os.platform() === 'win32') {
      return '.exe'
    } else {
      return ''
    }
  },

  /** platform in go format */
  os: () => {
    let result = os.platform()
    if (result === 'win32') {
      return 'windows'
    }
    return result
  },

  /** arch in go format */
  arch: () => {
    let result = os.arch()
    if (result === 'x64') {
      return 'amd64'
    } else if (result === 'ia32') {
      return '386'
    } else {
      return 'unknown'
    }
  },

  normalize_version: (version) => {
    return version.replace(/^v/, '')
  },

  version_equal: (a, b) => {
    return self.normalize_version(a) === self.normalize_version(b)
  },

  archive_name: (name) => {
    let formula = self.formulas[name]

    if (formula.format === '7z') {
      return `${name}.7z`
    } else if (formula.format === 'executable') {
      return `${name}${self.ext()}`
    } else {
      throw new Error(`Unknown formula format: ${formula.format}`)
    }
  },

  download_to_file: (url, file) => {
    let req = needle.get(url)
    let sink = fstream.Writer({
      path: file,
      mode: 0o777
    })
    req.pipe(sink)

    return new Promise((resolve, reject) => {
      sink.on('close', resolve)
      req.on('error', reject)
    })
  },

  get_latest_version: (channel) => {
    return new Promise((resolve, reject) => {
      let url = `${channel}/LATEST`
      needle.get(url, (err, res) => {
        if (err || res.statusCode !== 200) {
          return reject(err || `status code: ${res.statusCode}`)
        }
        let version = res.body.toString('utf8').replace(/\s/g, '')
        resolve(version)
      })
    })
  },

  fetch: async (opts, name) => {
    let noop = () => null
    let {onstatus = noop} = opts

    let formula = self.formulas[name]
    if (!formula) throw new Error(`Unknown formula: ${name}`)

    let channel = `${self.root_url()}/${name}/${self.os()}-${self.arch()}`

    let download_version = async (version) => {
      let archive_name = self.archive_name(name)
      let archive_path = path.join(self.bin_path(), archive_name)
      let archive_url = `${channel}/${version}/${archive_name}`
      onstatus(`Downloading ${name} ${version}`, 'download')
      log(opts, `downloading ${name} ${version} from ${archive_url}`)

      await self.download_to_file(archive_path, archive_path)

      if (formula.format === 'executable') {
        log(opts, `executable formula, no extract step`)
      } else {
        log(opts, `${formula.format} formula, extracting`)
        onstatus(`Installing ${name}`, 'install')
        await extract.extract({ archive_path, dest_path: self.bin_path() })
      }
    }

    onstatus(`Making sure we have the latest of everything...`, 'stopwatch')
    let get_latest_version = partial(self.get_latest_version, channel)

    let check = formula.version_check
<commit_msg>woops<commit_after>
import app from 'app'
import path from 'path'
import fstream from 'fstream'
import Promise from 'bluebird'

import {partial} from 'underscore'
import needle from 'needle'

import extract from '../tasks/extract'
import os from './os'
let log = require('./log')('ibrew')

let self = {
  formulas: {
    '7za': {
      format: 'executable',
      check: () => {
        if (self.os() === 'linux') {
          // TODO: add link to doc page too
          let msg = '7-zip missing: 7za must be in $PATH\n(Try installing p7zip-full)'
          throw new Error(msg)
        }
      },
      version_check: {
        args: [],
        parser: /p7zip Version ([0-9.]*)/
      }
    },
    'butler': {
      format: '7z',
      version_check: {
        args: ['version'],
        parser: /butler version ([0-9a-z.v]*)/
      }
    }
  },

  bin_path: () => {
    return path.join(app.getPath('userData'), 'bin')
  },

  root_url: () => {
    return 'https://misc.amos.me'
  },

  ext: () => {
    if (os.platform() === 'win32') {
      return '.exe'
    } else {
      return ''
    }
  },

  /** platform in go format */
  os: () => {
    let result = os.platform()
    if (result === 'win32') {
      return 'windows'
    }
    return result
  },

  /** arch in go format */
  arch: () => {
    let result = os.arch()
    if (result === 'x64') {
      return 'amd64'
    } else if (result === 'ia32') {
      return '386'
    } else {
      return 'unknown'
    }
  },

  normalize_version: (version) => {
    return version.replace(/^v/, '')
  },

  version_equal: (a, b) => {
    return self.normalize_version(a) === self.normalize_version(b)
  },

  archive_name: (name) => {
    let formula = self.formulas[name]

    if (formula.format === '7z') {
      return `${name}.7z`
    } else if (formula.format === 'executable') {
      return `${name}${self.ext()}`
    } else {
      throw new Error(`Unknown formula format: ${formula.format}`)
    }
  },

  download_to_file: (url, file) => {
    console.log(`downloading ${url} to ${file}`)
    let req = needle.get(url)
    let sink = fstream.Writer({
      path: file,
      mode: 0o777
    })
    req.pipe(sink)

    return new Promise((resolve, reject) => {
      sink.on('close', resolve)
      req.on('error', reject)
    })
  },

  get_latest_version: (channel) => {
    return new Promise((resolve, reject) => {
      let url = `${channel}/LATEST`
      needle.get(url, (err, res) => {
        if (err || res.statusCode !== 200) {
          return reject(err || `status code: ${res.statusCode}`)
        }
        let version = res.body.toString('utf8').replace(/\s/g, '')
        resolve(version)
      })
    })
  },

  fetch: async (opts, name) => {
    let noop = () => null
    let {onstatus = noop} = opts

    let formula = self.formulas[name]
    if (!formula) throw new Error(`Unknown formula: ${name}`)

    let channel = `${self.root_url()}/${name}/${self.os()}-${self.arch()}`

    let download_version = async (version) => {
      let archive_name = self.archive_name(name)
      let archive_path = path.join(self.bin_path(), archive_name)
      let archive_url = `${channel}/${version}/${archive_name}`
      onstatus(`Downloading ${name} ${version}`, 'download')
      log(opts, `downloading ${name} ${version} from ${archive_url}`)

      await self.download_to_file(archive_url, archive_path)

      if (formula.format === 'executable') {
        log(opts, `executable formula, no extract step`)
      } else {
        log(opts, `${formula.format} formula, extracting`)
        onstatus(`Installing ${name}`, 'install')
        await extract.extract({ archive_path, dest_path: self.bin_path() })
      }
    }

    onstatus(`Making sure we have the latest of everything...`, 'stopwatch')
    let get_latest_version = partial(self.get_latest_version, channel)

    let check = formula.version_check

{'fraction_non_alphanumeric': 0.17446973616140715, 'fraction_numerical': 0.008535954474909468, 'mean_word_length': 2.436888888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781379', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2535, 'n_words': 612}