<commit_before>import childProcess from 'child_process'
import fs from 'fs'
import https from 'https'
import mongoose from 'mongoose'
import schedule from 'node-schedule'
import winston from 'winston'
import { version } from '../../package.json'

// Holds the last commit that was synchronized
let lastCommit  =  ''

let db = {}

db.update = (collection) => {
  let url = `https://raw.githubusercontent.com/cobalt-uoft/datasets/master/${collection}.json`
  https.get(url, res => {
    let filePath = `.cobalt_data/${collection}.json`
    let stream = fs.createWriteStream(filePath, {'flags': 'w'})

    res.on('data', chunk => {
      stream.write(chunk)
    })

    res.on('end', () => {
      stream.end()

      mongoose.connection.db.collection(collection).remove(function(e, count) {
        if (e) {
          return winston.warn(`Could not import ${collection} to MongoDB.`, e)
        }

        let shell = childProcess.spawn('mongoimport', [
          '-d', 'cobalt',
          '-c', collection,
          '--file', filePath
        ])

        shell.on('close', code => {
          if (code == 0) {
            winston.info(`Synced ${collection}.`)

            // TODO clean this up
            if (collection === 'athletics') {
              let cmd = 'mongo cobalt --eval "db.athletics.find().forEach(doc=>{doc.date=new Date(doc.date);doc.events.forEach((_,i)=>{doc.events[i].start_time=new Date(doc.events[i].start_time);doc.events[i].end_time=new Date(doc.events[i].end_time)});db.athletics.save(doc)});"'
              childProcess.exec(cmd, error => {
                if (!error) {
                  winston.info(`Updated dates for ${collection}.`)
                } else {
                  winston.warn(`Could not update date values for ${collection}.`)
                }
              })
            }
          } else {
            winston.warn(`Could not import ${collection} to MongoDB. \
              The 'mongoexport' command left us with exit code ${code}.`)
          }<commit_msg>Update post import date conversion<commit_after>import childProcess from 'child_process'
import fs from 'fs'
import https from 'https'
import mongoose from 'mongoose'
import schedule from 'node-schedule'
import winston from 'winston'
import { version } from '../../package.json'

// Holds the last commit that was synchronized
let lastCommit  =  ''

let db = {}

db.update = (collection) => {
  let url = `https://raw.githubusercontent.com/cobalt-uoft/datasets/master/${collection}.json`
  https.get(url, res => {
    let filePath = `.cobalt_data/${collection}.json`
    let stream = fs.createWriteStream(filePath, {'flags': 'w'})

    res.on('data', chunk => {
      stream.write(chunk)
    })

    res.on('end', () => {
      stream.end()

      mongoose.connection.db.collection(collection).remove(function(e, count) {
        if (e) {
          return winston.warn(`Could not import ${collection} to MongoDB.`, e)
        }

        let shell = childProcess.spawn('mongoimport', [
          '-d', 'cobalt',
          '-c', collection,
          '--file', filePath
        ])

        shell.on('close', code => {
          if (code == 0) {
            winston.info(`Synced ${collection}.`)

            // TODO clean this up
            if (['athletics', 'exams'].indexOf(collection) > -1) {
              let cmd = `db.${collection}.find().forEach(doc=>{doc.date=new Date(doc.date);db.${collection}.save(doc)});`

              childProcess.exec(`mongo cobalt --eval "${cmd}"`, error => {
                if (!error) {
                  winston.info(`Updated dates for ${collection}.`)
                } else {
                  winston.warn(`Could not update date values for ${collection}.`)
                }
              })
            }
          } else {
            winston.warn(`Could not import ${collection} to MongoDB. \
              The 'mongoexport' command left us with exit code ${code}.`)
          }
{'fraction_non_alphanumeric': 0.15441363520732637, 'fraction_numerical': 0.0007631645891630628, 'mean_word_length': 2.1889699918897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5647970', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1145, 'n_words': 273}