<commit_before>// A script to put items into DynamoDB.
// Usage:
//   AWS_REGION=us-west-2 TABLE_NAME=Something DYNAMODB_ENDPOINT=SomeEndpoint \
//     node scripts/upload-data-to-ddb.js ./fixtures.json
/* eslint import/no-extraneous-dependencies: 0 */
const AWS = require('aws-sdk')
const fs = require('fs')
const path = require('path')

// Expect one argument, the fixtures file.
const args = process.argv.slice(2)
const filePath = args[0]

// Update as needed.
const FIXTURES_FILE_PATH = path.resolve(filePath)
// eslint-disable-next-line prefer-destructuring
const TABLE_NAME = process.env.TABLE_NAME

AWS.config.update({
  region: process.env.AWS_REGION,
  endpoint: process.env.DYNAMODB_ENDPOINT,
  accessKeyId: process.env.AWS_ACCESS_KEY_ID,
  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
})

const docClient = new AWS.DynamoDB.DocumentClient()

// To throttle large batches of data.
const SLEEP_MS_BETWEEN_QUERY_BATCHES = 100
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

const loadItemsIntoTable = async (items, tableName) => {
  const BATCH_MAX_ITEMS = 25
  let itemsToLoad = items
  if (items.length > BATCH_MAX_ITEMS) {
    await loadItemsIntoTable(items.slice(BATCH_MAX_ITEMS - 1), tableName)
    console.log(`Items loaded: ${items.length}`)
    if (SLEEP_MS_BETWEEN_QUERY_BATCHES > 0) {
      await sleep(SLEEP_MS_BETWEEN_QUERY_BATCHES)
    }
    itemsToLoad = items.slice(0, BATCH_MAX_ITEMS - 1)
  }
  const params = {
    RequestItems: {
      [tableName]: itemsToLoad.map(item => ({
        PutRequest: {
          Item: item,
        },
      })),
    },
  }<commit_msg>Comment out access keys (use CLI defaults)<commit_after>// A script to put items into DynamoDB.
// Usage:
//   AWS_REGION=us-west-2 TABLE_NAME=Something DYNAMODB_ENDPOINT=SomeEndpoint \
//     node scripts/upload-data-to-ddb.js ./fixtures.json
/* eslint import/no-extraneous-dependencies: 0 */
const AWS = require('aws-sdk')
const fs = require('fs')
const path = require('path')

// Expect one argument, the fixtures file.
const args = process.argv.slice(2)
const filePath = args[0]

// Update as needed.
const FIXTURES_FILE_PATH = path.resolve(filePath)
// eslint-disable-next-line prefer-destructuring
const TABLE_NAME = process.env.TABLE_NAME

AWS.config.update({
  region: process.env.AWS_REGION,
  endpoint: process.env.DYNAMODB_ENDPOINT,
  // accessKeyId: process.env.AWS_ACCESS_KEY_ID,
  // secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
})

const docClient = new AWS.DynamoDB.DocumentClient()

// To throttle large batches of data.
const SLEEP_MS_BETWEEN_QUERY_BATCHES = 100
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

const loadItemsIntoTable = async (items, tableName) => {
  const BATCH_MAX_ITEMS = 25
  let itemsToLoad = items
  if (items.length > BATCH_MAX_ITEMS) {
    await loadItemsIntoTable(items.slice(BATCH_MAX_ITEMS - 1), tableName)
    console.log(`Items loaded: ${items.length}`)
    if (SLEEP_MS_BETWEEN_QUERY_BATCHES > 0) {
      await sleep(SLEEP_MS_BETWEEN_QUERY_BATCHES)
    }
    itemsToLoad = items.slice(0, BATCH_MAX_ITEMS - 1)
  }
  const params = {
    RequestItems: {
      [tableName]: itemsToLoad.map(item => ({
        PutRequest: {
          Item: item,
        },
      })),
    },
  }
{'fraction_non_alphanumeric': 0.11836115326251896, 'fraction_numerical': 0.007890743550834597, 'mean_word_length': 3.8257686676427527, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3195682', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1160, 'n_words': 240}