<commit_before>// http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Kinesis.html
const AWS = require('aws-sdk')

const kinesis = new AWS.Kinesis()

function listStreams () {
  kinesis.listStreams({}, (err, data) => {
    if (err) {
      console.error(err)
    } else {
      console.log(data)
    }
  })
}

function readShard (shardIterator) {
  const params = {
    ShardIterator: shardIterator,
    Limit: 100,
  }
  kinesis.getRecords(params, (err, data) => {
    if (err) console.log(err, err.stack)
    else {
      data.Records.forEach((x) => {
        console.log(x.Data)
        console.log(x.Data.toString())
      })
      readShard(data.NextShardIterator)
    }
  })
}

function getShardId (streamName) {
  return new Promise((resolve, reject) => {
    const params = {
      StreamName: streamName,
    }
    kinesis.describeStream(params, (err, data) => {
      if (err) {
        reject(err)
      } else {
        if (data.StreamDescription.Shards.length) {
          // For heavy duty cases, we would return all shard ids and spin up a
          // reader for each shards
          resolve(data.StreamDescription.Shards[0].ShardId)
        } else {
          reject('No shards!')
        }
      }
    })
  })
}

function getShardIterator (streamName, shardId) {
  return new Promise((resolve, reject) => {
    const params = {
      ShardId: shardId,
      ShardIteratorType: 'LATEST',
      StreamName: streamName,
    }
    kinesis.getShardIterator(params, (err, data) => {
      if (err) {
        reject(err)
      } else {
        resolve(data.ShardIterator)
      }
    })
  })
}

module.exports.listStreams = listStreams
module.exports.main = function (streamName) {
  getShardId(streamName)
  .then((shardId) => getShardIterator(streamName, shardId))
  .then((shardIterator) => readShard(shardIterator))
  .catch((err) => console.log(err, err.stack))
}<commit_msg>Refactor: list functions in lifecycle order<commit_after>// http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Kinesis.html
const AWS = require('aws-sdk')

const kinesis = new AWS.Kinesis()

function listStreams () {
  kinesis.listStreams({}, (err, data) => {
    if (err) {
      console.error(err)
    } else {
      console.log(data)
    }
  })
}

function getShardId (streamName) {
  return new Promise((resolve, reject) => {
    const params = {
      StreamName: streamName,
    }
    kinesis.describeStream(params, (err, data) => {
      if (err) {
        reject(err)
      } else {
        if (data.StreamDescription.Shards.length) {
          // For heavy duty cases, we would return all shard ids and spin up a
          // reader for each shards
          resolve(data.StreamDescription.Shards[0].ShardId)
        } else {
          reject('No shards!')
        }
      }
    })
  })
}

function getShardIterator (streamName, shardId) {
  return new Promise((resolve, reject) => {
    const params = {
      ShardId: shardId,
      ShardIteratorType: 'LATEST',
      StreamName: streamName,
    }
    kinesis.getShardIterator(params, (err, data) => {
      if (err) {
        reject(err)
      } else {
        resolve(data.ShardIterator)
      }
    })
  })
}

function readShard (shardIterator) {
  const params = {
    ShardIterator: shardIterator,
    Limit: 100,
  }
  kinesis.getRecords(params, (err, data) => {
    if (err) console.log(err, err.stack)
    else {
      data.Records.forEach((x) => {
        console.log(x.Data)
        console.log(x.Data.toString())
      })
      readShard(data.NextShardIterator)
    }
  })
}

// EXPORTS
//////////

module.exports.listStreams = listStreams

module.exports.main = function (streamName) {
  getShardId(streamName)
  .then((shardId) => getShardIterator(streamName, shardId))
  .then((shardIterator) => readShard(shardIterator))
  .catch((err) => console.log(err, err.stack))
}
{'fraction_non_alphanumeric': 0.1402185223725286, 'fraction_numerical': 0.002081165452653486, 'mean_word_length': 2.5568917668825164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2778808', 'n_tokens_mistral': 1311, 'n_tokens_neox': 1296, 'n_words': 246}