<commit_before>@run = ->
  
  fs = require 'fs'
  util = (require '../cakeutils').include()

  mongo = require 'mongodb'
  mongoConf = require('../.dbconf').mongodb

  pg = require 'pg'
  pgurl = require('../.dbconf').postgres


  runImportTasks = (importTasks) ->
    for taskName, importFunc of importTasks
      console.log ">>>>>>>>>> Importing '#{taskName}' >>>>>>>>>>"
      importFunc()


  runImportTasks

    aiddata : ->    
      tempDir = "data/temp"
      tempFile = tempDir + "/_aiddata.json" 
      util.mkdir tempDir

      omitNullValueFields = true

      pgclient = new pg.Client(pgurl)
      # disconnect client when all queries are finished        
      #pgclient.on('drain', pgclient.end.bind(pgclient))
      pgclient.connect()

      limit = 10

      cntQuery = pgclient.query(
        if limit? 
          "SELECT #{limit} AS count"
        else
          "SELECT COUNT(*) AS count FROM aiddata2"
      )

      cntQuery.on 'row', (row) ->
        
        totalRecordsNum = row.count
        console.log "totalRecordsNum: " + totalRecordsNum

        numProcessed = 0


        jsonFile = fs.createWriteStream(tempFile)
        jsonFile.write "["

        firstRow = true
        query = pgclient.query "SELECT * FROM aiddata2 #{if limit? then 'LIMIT '+limit}"
        query.on 'row', (row) ->

          #row._id = row.aiddata_id
          #delete row.aiddata_id

          if omitNullValueFields
            for k,v of row
              unless v?
                delete row[k]

          #console.log "Upserted #{numUpserted} of #{totalRecordsNum}, last _id: #{row._id}"
          numProcessed++

          if numProcessed < totalRecordsNum
            jsonFile.write JSON.stringify(row) + ","
          else
            jsonFile.write JSON.stringify(row) + "]"
            console.log "Read in: #{numProcessed}"
            util.run "/usr/bin/mongoimport -d aiddata -c aiddata --upsert --upsertFields aiddata_id data/temp/_aiddata.json",
              ->
                fs.unlink tempFile 
                console.log "Done"



        query.on 'end', -> pgclient.end()
<commit_msg>username/password supplied to mongoimport<commit_after>@run = ->
  
  fs = require 'fs'
  util = (require '../cakeutils').include()

  #mongo = require 'mongodb'
  mongoConf = require('../.dbconf').mongodb

  pg = require 'pg'
  pgurl = require('../.dbconf').postgres


  runImportTasks = (importTasks) ->
    for taskName, importFunc of importTasks
      console.log ">>>>>>>>>> Importing '#{taskName}' >>>>>>>>>>"
      importFunc()


  runImportTasks

    aiddata : ->    
      tempDir = "data/temp"
      tempFile = tempDir + "/_aiddata.json" 
      util.mkdir tempDir

      omitNullValueFields = true

      pgclient = new pg.Client(pgurl)
      # disconnect client when all queries are finished        
      #pgclient.on('drain', pgclient.end.bind(pgclient))
      pgclient.connect()

      limit = 10

      cntQuery = pgclient.query(
        if limit? 
          "SELECT #{limit} AS count"
        else
          "SELECT COUNT(*) AS count FROM aiddata2"
      )

      cntQuery.on 'row', (row) ->
        
        totalRecordsNum = row.count
        console.log "totalRecordsNum: " + totalRecordsNum

        numProcessed = 0


        jsonFile = fs.createWriteStream(tempFile)
        jsonFile.write "["

        firstRow = true
        query = pgclient.query "SELECT * FROM aiddata2 #{if limit? then 'LIMIT '+limit}"
        query.on 'row', (row) ->

          #row._id = row.aiddata_id
          #delete row.aiddata_id

          if omitNullValueFields
            for k,v of row
              unless v?
                delete row[k]

          #console.log "Upserted #{numUpserted} of #{totalRecordsNum}, last _id: #{row._id}"
          numProcessed++

          if numProcessed < totalRecordsNum
            jsonFile.write JSON.stringify(row) + ","
          else
            jsonFile.write JSON.stringify(row) + "]"
            console.log "Read in: #{numProcessed}"
            util.run "/usr/bin/mongoimport " +
                        "-d aiddata -c aiddata --upsert --upsertFields aiddata_id " +
                        "-u #{mongoConf.user} -p #{mongoConf.password} "+ 
                        "#{tempFile}",
              ->
                fs.unlink tempFile 
                console.log "Done"



        query.on 'end', -> pgclient.end()

{'fraction_non_alphanumeric': 0.12402923709456373, 'fraction_numerical': 0.0022841480127912287, 'mean_word_length': 1.832470892626132, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4090288', 'n_tokens_mistral': 1348, 'n_tokens_neox': 1281, 'n_words': 305}