<commit_before>const fs = require('fs')
const rp = require('request-promise-native')

// Define GhostInspector class
class GhostInspector {
  constructor (apiKey) {
    this.host = 'https://api.ghostinspector.com'
    this.prefix = '/v1'
    this.apiKey = apiKey
  }

  buildRequestUrl (path) {
    return this.host + this.prefix + path
  }

  buildQueryString (params = {}) {
    let queryString = '?'
    // Add params
    for (let key in params) {
      // handle array params
      const val = params[key]
      if (val instanceof Array) {
        for (let item of val) {
          queryString += key + '[]=' + encodeURIComponent(item) + '&'
        }
      } else {
        queryString += key + '=' + encodeURIComponent(val) + '&'
      }
    }
    return queryString
  }

  deepConvertParamToString (param) {
    if (param instanceof Array) {
      return param.map(this.deepConvertParamToString)
    } else {
      return param.toString()
    }
  }

  buildFormData (params = {}) {
    // Copy over all fields and convert non-arrays to strings
    const formData = {}
    for (let key in params) {
      formData[key] = this.deepConvertParamToString(params[key])
    }
    return formData
  }

  getOverallResultOutcome (data) {
    if (data instanceof Array) {
      let passing = data.length ? true : null
      for (let entry of data) {
        passing = passing && entry.passing
      }
      return passing
    } else {
      return data.passing === undefined ? null : data.passing
    }
  }

  async request (method, path, params, callback) {
    // Sort out params and callback
    if (typeof params === 'function') {
      callback = params
      params = {}
    } else if (!params || typeof params !== 'object') {
      params = {}
    }
    // Add API key to params
    params.apiKey = this.apiKey
    // Setup initial request options
    const options = {
      method: method,
      uri: this.buildRequestUrl(path),
      headers: {
        'User-Agent': 'Ghost Inspector Node.js Bindings'
      },
      json: true,
      timeout: 3600000
    }
    // Customize request based on GET or POST
    if (method === 'POST') {
      // Add params as form data
      options.formData = this.buildFormData(params)
      // Check for special `dataFile` parameter (path to CSV file) and open read stream for it
      if (params.dataFile) {
        options.formData.dataFile = fs.createReadStream(params.dataFile.toString())
      }
    } else {
      // Add params as query string
      options.uri += this.buildQueryString(params)
    }
    // Send request to API
    let result
    try {
      result = await rp(options)
    } catch (err) {
      if (typeof callback === 'function') {
        callback(err)
        return
      }
      throw err
    }
    // Process response
    if (result.code === 'ERROR') {
      const err = new Error(result.message)
      if (typeof callback === 'function') {
        callback(err)
        return
      }
      throw err
    } else {
      if (typeof callback === 'function') {
        callback(null, result.data)
      }
      return result.data
    }
  }

  async download (path, dest, callback) {
    // Add API key to params
    const params = {
      apiKey: this.apiKey
    }
    const options = {
      method: 'GET',
      uri: this.buildRequestUrl(path) + this.buildQueryString(params),
      headers: {
        'User-Agent': 'Ghost Inspector Node.js Bindings'
      }
    }<commit_msg>Centralize user agent string<commit_after>const fs = require('fs')
const rp = require('request-promise-native')

// Define GhostInspector class
class GhostInspector {
  constructor (apiKey) {
    this.userAgent = 'Ghost Inspector Node.js Bindings'
    this.host = 'https://api.ghostinspector.com'
    this.prefix = '/v1'
    this.apiKey = apiKey
  }

  buildRequestUrl (path) {
    return this.host + this.prefix + path
  }

  buildQueryString (params = {}) {
    let queryString = '?'
    // Add params
    for (let key in params) {
      // handle array params
      const val = params[key]
      if (val instanceof Array) {
        for (let item of val) {
          queryString += key + '[]=' + encodeURIComponent(item) + '&'
        }
      } else {
        queryString += key + '=' + encodeURIComponent(val) + '&'
      }
    }
    return queryString
  }

  deepConvertParamToString (param) {
    if (param instanceof Array) {
      return param.map(this.deepConvertParamToString)
    } else {
      return param.toString()
    }
  }

  buildFormData (params = {}) {
    // Copy over all fields and convert non-arrays to strings
    const formData = {}
    for (let key in params) {
      formData[key] = this.deepConvertParamToString(params[key])
    }
    return formData
  }

  getOverallResultOutcome (data) {
    if (data instanceof Array) {
      let passing = data.length ? true : null
      for (let entry of data) {
        passing = passing && entry.passing
      }
      return passing
    } else {
      return data.passing === undefined ? null : data.passing
    }
  }

  async request (method, path, params, callback) {
    // Sort out params and callback
    if (typeof params === 'function') {
      callback = params
      params = {}
    } else if (!params || typeof params !== 'object') {
      params = {}
    }
    // Add API key to params
    params.apiKey = this.apiKey
    // Setup initial request options
    const options = {
      method: method,
      uri: this.buildRequestUrl(path),
      headers: {
        'User-Agent': this.userAgent
      },
      json: true,
      timeout: 3600000
    }
    // Customize request based on GET or POST
    if (method === 'POST') {
      // Add params as form data
      options.formData = this.buildFormData(params)
      // Check for special `dataFile` parameter (path to CSV file) and open read stream for it
      if (params.dataFile) {
        options.formData.dataFile = fs.createReadStream(params.dataFile.toString())
      }
    } else {
      // Add params as query string
      options.uri += this.buildQueryString(params)
    }
    // Send request to API
    let result
    try {
      result = await rp(options)
    } catch (err) {
      if (typeof callback === 'function') {
        callback(err)
        return
      }
      throw err
    }
    // Process response
    if (result.code === 'ERROR') {
      const err = new Error(result.message)
      if (typeof callback === 'function') {
        callback(err)
        return
      }
      throw err
    } else {
      if (typeof callback === 'function') {
        callback(null, result.data)
      }
      return result.data
    }
  }

  async download (path, dest, callback) {
    // Add API key to params
    const params = {
      apiKey: this.apiKey
    }
    const options = {
      method: 'GET',
      uri: this.buildRequestUrl(path) + this.buildQueryString(params),
      headers: {
        'User-Agent': this.userAgent
      }
    }
{'fraction_non_alphanumeric': 0.11486779367143476, 'fraction_numerical': 0.002311804652506863, 'mean_word_length': 2.112410071942446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1473290', 'n_tokens_mistral': 2057, 'n_tokens_neox': 2029, 'n_words': 580}