<commit_before>const AWS = require('aws-sdk')
var fs = require('fs')
const { execSync } = require('child_process')
const Issue = require('../issues').Issue
const zlib = require('zlib')
const s3 = new AWS.S3()
const isNode = typeof window === 'undefined'

/**
 * Remote Files
 *
 * Helper functions for checking for and using remote file paths
 *
 */

const remoteFiles = {
  // Initiaties access of a remote file from git-annex remote
  // Get remotes info the call to try successive remotes
  // Called by testFile
  getAnnexedFile: function(file, dir, limit, callback) {
    // Build config object
    const config = {
      file: file,
      dir: dir,
    }
    if (limit) config.limit = limit
    config.remotesInfo = this.getRemotesInfo(dir, file)

    // try all the special git-annex remotes, and exit if there is an issue (reading / fetching files)
    // if all remotes fail, throw issue code 97
    Promise.all(
      config.remotesInfo.map((remote, idx) => {
        return this.tryRemote(remote, config)
          .then(data => callback(null, null, data))
          .catch(err => {
            if (err.code) {
              return callback(err, null, null)
            }
            if (idx == config.remotesInfo.length) {
              return callback(
                new Issue({ code: 97, file: config.file }),
                null,
                null,
              )
            }
          })
      }),
    )
  },

  // Try to access file from a remote
  tryRemote: function(remote, config) {
    // Get current remote
    config.s3Params = this.getSingleRemoteInfo(config.dir, remote.remoteUuid)
    const dir = config.dir.endsWith('/') ? config.dir.slice(0, -1) : config.dir
    const datasetName = dir.split('/')[dir.split('/').length - 1]
    const key = datasetName + config.file.relativePath
    // Add additonal parameters
    config.s3Params['Key'] = key
    config.s3Params['VersionId'] = remote.versionId
    return this.accessRemoteFile(config)
  },

  // Download a remote file from its path
  accessRemoteFile: function(config) {
    if (config.limit) config.s3Params['Range'] = 'bytes=0-' + config.limit
    return new Promise((resolve, reject) => {
      this.constructAwsRequest(config)
        .then(buffer => {
          if (config.file.name.endsWith('.gz')) {
            this.extractGzipBuffer(buffer, config)
              .then(data => resolve(data))
              .catch(err => reject(err))
          } else {
            resolve(buffer)
          }
        })
        .catch(reject)
    })
  },

  constructAwsRequest: function(config) {
    const hasCreds = isNode
      ? Object.keys(process.env).indexOf('AWS_ACCESS_KEY_ID') > -1
      : false
    if (hasCreds) {
      return s3
        .getObject(config.s3Params)
        .promise()
        .then(data => data.Body)
    } else {
      // bucket + key url
      let url =
        s3.getSignedUrl('getObject', config.s3Params) +
        config.s3Params.Bucket +
        '/' +
        config.s3Params.Key

      // add version to url, if exists
      url = config.s3Params.VersionId
        ? url + '?VersionId=' + config.s3Params.VersionId
        : url

      // add range to url, if exists
      url = config.s3Params.Range
        ? url + '?Range=' + config.s3Params.Range
        : url
      return fetch(url).then(resp => resp.buffer())
    }
  },

  extractGzipBuffer: function(buffer, config) {
    return new Promise((resolve, reject) => {
      const decompressStream = zlib
        .createGunzip()
        .on('data', function(chunk) {
          resolve(chunk)
          decompressStream.pause()
        })
        .on('error', function() {
          return reject(new Issue({ code: 28, file: config.file }))
        })
      decompressStream.write(buffer)
    })
  },

  // Function for calling local git-annex
  callGitAnnex: function(cmd) {
    const stream = execSync(cmd, {
      shell: true,
    })
    return stream.toString()
  },

  // Ask git-annex for more information about a file
  getRemotesInfo: function(dir, file) {
    // Remove leading slash from  relativePath
    const relativePath = file.relativePath.startsWith('/')
      ? file.relativePath.substring(1)
      : file.relativePath
    const getRemoteCmd = `cd ${dir}
    git show git-annex:$(git-annex examinekey --json $(git-annex lookupkey ${relativePath}) | jq -r .hashdirlower)$(git-annex lookupkey ${relativePath}).log.rmet`
    const resp = this.callGitAnnex(getRemoteCmd)
    const remotesInfo = this.processRemoteMetadata(resp)
    return remotesInfo
  },

  // Get info from a given git-annex remote
  getSingleRemoteInfo: function(dir, uuid) {
    const infoCmd = `cd ${dir}
    git-annex info ${uuid}`
    const resp = this.callGitAnnex(infoCmd)
    return this.getRemoteBucket(resp)
  },

  // Obtain bucket field from git-annex info query
  getRemoteBucket: function(resp) {
    const params = {
      Bucket: null,
    }
    for (let line of resp.split('\n')) {
      if (line.includes('bucket: ')) {
        params.Bucket = line.split(': ')[1]
      }
    }
    return params
  },

  // Manipulate the response from git-annex lookupkey query
  processRemoteMetadata: function(resp) {
    const remotesInfo = []<commit_msg>fix some small bugs in remoteFiles<commit_after>const AWS = require('aws-sdk')
const fs = require('fs')
const { execSync } = require('child_process')
const Issue = require('../issues').Issue
const zlib = require('zlib')
const s3 = new AWS.S3()
const isNode = typeof window === 'undefined'

/**
 * Remote Files
 *
 * Helper functions for checking for and using remote file paths
 *
 */

const remoteFiles = {
  // Initiaties access of a remote file from git-annex remote
  // Get remotes info the call to try successive remotes
  // Called by testFile
  getAnnexedFile: function(file, dir, limit, callback) {
    // Build config object
    const config = {
      file: file,
      dir: dir,
    }
    if (limit) config.limit = limit
    config.remotesInfo = this.getRemotesInfo(dir, file)

    // try all the special git-annex remotes, and exit if there is an issue (reading / fetching files)
    // if all remotes fail, throw issue code 97
    Promise.all(
      config.remotesInfo.map((remote, idx) => {
        return this.tryRemote(remote, config)
          .then(data => callback(null, null, data))
          .catch(err => {
            if (err.code) {
              return callback(err, null, null)
            }
            if (idx == config.remotesInfo.length) {
              return callback(
                new Issue({ code: 97, file: config.file }),
                null,
                null,
              )
            }
          })
      }),
    )
  },

  // Try to access file from a remote
  tryRemote: function(remote, config) {
    // Get current remote
    config.s3Params = this.getSingleRemoteInfo(config.dir, remote.remoteUuid)
    const dir = config.dir.endsWith('/') ? config.dir.slice(0, -1) : config.dir
    const datasetName = dir.split('/')[dir.split('/').length - 1]
    const key = datasetName + config.file.relativePath
    // Add additonal parameters
    config.s3Params['Key'] = key
    config.s3Params['VersionId'] = remote.versionId
    return this.accessRemoteFile(config)
  },

  // Download a remote file from its path
  accessRemoteFile: function(config) {
    if (config.limit) config.s3Params['Range'] = 'bytes=0-' + config.limit
    return new Promise((resolve, reject) => {
      this.constructAwsRequest(config)
        .then(buffer => {
          if (config.file.name.endsWith('.gz')) {
            this.extractGzipBuffer(buffer, config)
              .then(data => resolve(data))
              .catch(err => reject(err))
          } else {
            resolve(buffer)
          }
        })
        .catch(reject)
    })
  },

  constructAwsRequest: function(config) {
    const hasCreds = isNode
      ? Object.keys(process.env).indexOf('AWS_ACCESS_KEY_ID') > -1
      : false
    if (hasCreds) {
      return s3
        .getObject(config.s3Params)
        .promise()
        .then(data => data.Body)
    } else {
      // bucket + key url
      let url =
        s3.getSignedUrl('getObject', config.s3Params) +
        config.s3Params.Bucket +
        '/' +
        config.s3Params.Key

      // add version to url, if exists
      url = config.s3Params.VersionId
        ? url + '?VersionId=' + config.s3Params.VersionId
        : url

      // add range to url, if exists
      url = config.s3Params.Range
        ? url + '?Range=' + config.s3Params.Range
        : url
      return fetch(url).then(resp => resp.buffer())
    }
  },

  extractGzipBuffer: function(buffer, config) {
    return new Promise((resolve, reject) => {
      try {
        const decompressStream = zlib
          .createGunzip()
          .on('data', function(chunk) {
            resolve(chunk)
            decompressStream.pause()
          })
          .on('error', function() {
            return reject(new Issue({ code: 28, file: config.file }))
          })
        decompressStream.write(buffer)
      } catch (e) {
        return reject(new Issue({ code: 28, file: config.file }))
      }
    })
  },

  // Function for calling local git-annex
  callGitAnnex: function(cmd) {
    const stream = execSync(cmd, {
      shell: true,
    })
    return stream.toString()
  },

  // Ask git-annex for more information about a file
  getRemotesInfo: function(dir, file) {
    // Remove leading slash from  relativePath
    const relativePath =
      file.relativePath && file.relativePath.startsWith('/')
        ? file.relativePath.substring(1)
        : file.relativePath
    const getRemoteCmd = `cd ${dir}
    git show git-annex:$(git-annex examinekey --json $(git-annex lookupkey ${relativePath}) | jq -r .hashdirlower)$(git-annex lookupkey ${relativePath}).log.rmet`
    const resp = this.callGitAnnex(getRemoteCmd)
    const remotesInfo = this.processRemoteMetadata(resp)
    return remotesInfo
  },

  // Get info from a given git-annex remote
  getSingleRemoteInfo: function(dir, uuid) {
    const infoCmd = `cd ${dir}
    git-annex info ${uuid}`
    const resp = this.callGitAnnex(infoCmd)
    return this.getRemoteBucket(resp)
  },

  // Obtain bucket field from git-annex info query
  getRemoteBucket: function(resp) {
    const params = {
      Bucket: null,
    }
    for (let line of resp.split('\n')) {
      if (line.includes('bucket: ')) {
        params.Bucket = line.split(': ')[1]
      }
    }
    return params
  },

  // Manipulate the response from git-annex lookupkey query
  processRemoteMetadata: function(resp) {
    const remotesInfo = []
{'fraction_non_alphanumeric': 0.12408140192198983, 'fraction_numerical': 0.005652911249293386, 'mean_word_length': 2.2894329098233652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7190950', 'n_tokens_mistral': 3228, 'n_tokens_neox': 3169, 'n_words': 816}