<commit_before>#!/usr/bin/env node -r babel/register
/**
 * Database scripts
 * Usage:
 *  ./database.js [action]
 * Available actions:
 *  create - create the NODE_ENV database
 *  drop - drop the NODE_ENV database
 *  prepare - create missing tables for NODE_ENV database
 *  reset - drop, create, and prepare the NODE_ENV database
 * Example:
 *  NODE_ENV=test ./database.js refresh
 */
import RethinkDB from 'rethinkdbdash'
import { getModelList } from '@server/models'
import { database, env } from '@server/config'
console.log(`Loaded database config for "${env}" environment`)
const MODEL_LIST = getModelList()

// Perform the action
executeAction(process.argv[2])
async function executeAction (action) {
  try {
    const r = RethinkDB(database)
    switch (action) {
      case 'drop':
        await dropDatabase(r)
        break
      case 'create':
        await createDatabase(r)
        break
      case 'prepare':
        await createTables(r)
        break
      case 'reset':
        await dropDatabase(r)
        await createDatabase(r)
        await createTables(r)
        break
      default:
        throw new Error(`Unknown action "${action}"`)
    }
    await r.getPoolMaster().drain()
  } catch (err) {
    console.error(`Error executing action: ${err}`)
    process.exit(1)
  }
}

// ACTIONS
async function dropDatabase (r) {
  const list = await r.dbList()
  if (list.includes(database.db)) {
    try {
      console.log(`Dropping "${database.db}" database`)
      await r.dbDrop(database.db)
      console.log('Database dropped')
    } catch (err) {
      console.error(`Error dropping "${database.db}" database`)
      throw err
    }
  } else {
    console.log(`Unable to drop "${database.db}", it doesn't exist`)
  }
}

async function createDatabase (r) {
  const list = await r.dbList()
  if (!list.includes(database.db)) {
    try {
      console.log(`Creating "${database.db}" database`)
      await r.dbCreate(database.db)
      console.log('Database created')
    } catch (err) {
      console.error(`Error creating "${database.db}" database`)
      throw err
    }
  } else {
    console.log(`Unable to create "${database.db}", already exists`)
  }
}

async function createTables (r) {
  console.log(`Creating tables for "${database.db}" database`)
  const tableList = await r.tableList()
  await Promise.all(MODEL_LIST.map(async (model) => {
    const { TABLE, INDEXES, TYPE } = model

    // Create table if missing
    if (tableList.indexOf(TABLE) === -1) {
      console.log(`Creating table "${TABLE}" for ${TYPE} model`)
      await r.tableCreate(TABLE)
    }

    // Create indexes if missing
    const indexList = await r.table(TABLE).indexList()
    await Promise.all(INDEXES.map(async (INDEX) => {
      if (indexList.indexOf(INDEX) === -1) {
        console.log(`Creating index "${INDEX}" for ${TABLE} table`)
        await r.table(TABLE).indexCreate(INDEX)
      }
    }))
  }))
  console.log(`Created tables and indexes`)
}
<commit_msg>Update database script to use new models<commit_after>#!/usr/bin/env node -r babel/register
/**
 * Database scripts
 * Usage:
 *  ./database.js [action]
 * Available actions:
 *  create - create the NODE_ENV database
 *  drop - drop the NODE_ENV database
 *  prepare - create missing tables for NODE_ENV database
 *  reset - drop, create, and prepare the NODE_ENV database
 * Example:
 *  NODE_ENV=test ./database.js refresh
 */
import RethinkDB from 'rethinkdbdash'
import { database, env } from '@server/config'
import { instantiateModels } from '@server/models'
console.log(`Loaded database config for "${env}" environment`)

// Perform the action
executeAction(process.argv[2])
async function executeAction (action) {
  try {
    const r = RethinkDB(database)
    switch (action) {
      case 'drop':
        await dropDatabase(r)
        break
      case 'create':
        await createDatabase(r)
        break
      case 'prepare':
        await createTables(r)
        break
      case 'reset':
        await dropDatabase(r)
        await createDatabase(r)
        await createTables(r)
        break
      default:
        throw new Error(`Unknown action "${action}"`)
    }
    await r.getPoolMaster().drain()
  } catch (err) {
    console.error(`Error executing action: ${err}`)
    process.exit(1)
  }
}

// ACTIONS
async function dropDatabase (r) {
  const list = await r.dbList()
  if (list.includes(database.db)) {
    try {
      console.log(`Dropping "${database.db}" database`)
      await r.dbDrop(database.db)
      console.log('Database dropped')
    } catch (err) {
      console.error(`Error dropping "${database.db}" database`)
      throw err
    }
  } else {
    console.log(`Unable to drop "${database.db}", it doesn't exist`)
  }
}

async function createDatabase (r) {
  const list = await r.dbList()
  if (!list.includes(database.db)) {
    try {
      console.log(`Creating "${database.db}" database`)
      await r.dbCreate(database.db)
      console.log('Database created')
    } catch (err) {
      console.error(`Error creating "${database.db}" database`)
      throw err
    }
  } else {
    console.log(`Unable to create "${database.db}", already exists`)
  }
}

async function createTables (r) {
  console.log(`Creating tables for "${database.db}" database`)
  const instances = instantiateModels(r)
  const instanceList = Object.values(instances)
  await Promise.all(instanceList.map(model => model.createTable()))
  await Promise.all(instanceList.map(model => model.createIndexes()))
  console.log(`Created tables and indexes`)
}

{'fraction_non_alphanumeric': 0.1327449567723343, 'fraction_numerical': 0.001080691642651297, 'mean_word_length': 3.023913043478261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4689857', 'n_tokens_mistral': 1665, 'n_tokens_neox': 1662, 'n_words': 456}