<commit_before>var pkg = require("./package.json")
var MongoClient = require("mongodb").MongoClient
var params = require("commander")

params
  .version(pkg.version)
  .description(pkg.description)
  .option("-H, --host <address>", "DataBase host (127.0.0.1)", "127.0.0.1")
  .option("-P, --port <number>", "DataBase port (27017)", "27017")
  .option("-d, --db <name>", "DataBase name (logs)", "logs")
  .option("-c, --collection <name>", "DataBase collection name (logs)", "logs")
  .option("-u, --user <username>", "DataBase username (root)", "root")
  .option("-p, --pass <password>", "DataBase username password ()", "")
  .parse(process.argv)

var mongoOptions = "mongodb://"
  + (params.user && params.pass)
    ? "[" + params.user + ":" + params.pass + "]@"
    : ""
  + params.host
  + ":" + params.port
  + "/" + params.db

MongoClient.connect(mongoOptions, onConnection)

function onConnection(e, db) {

  if (e) {
    return handleError(e)
  }

  var collection = db.collection(params.collection)
  
  process.stdin.resume()
  process.stdin.setEncoding("utf8")
  process.stdin.on("data", function stdin(data) {

    var document = null

    try {
      document = JSON.parse(data)
    } catch (e) {
      document = {
        msg: data
      }
    }

    collection.insertOne(document, function insertOne(e) {
      if (e) {
        return handleError(e)
      }
    })

    process.stdout.write(JSON.stringify(document) + "\n")

  })
}

function handleError(e) {
  if (e instanceof Error) {
    console.error(e)
  } else {
    console.error(new Error(e))
  }
  return
}<commit_msg>Add process SIGINT handler<commit_after>var pkg = require("./package.json")
var MongoClient = require("mongodb").MongoClient
var params = require("commander")

params
  .version(pkg.version)
  .description(pkg.description)
  .option("-H, --host <address>", "DataBase host (127.0.0.1)", "127.0.0.1")
  .option("-P, --port <number>", "DataBase port (27017)", "27017")
  .option("-d, --db <name>", "DataBase name (logs)", "logs")
  .option("-c, --collection <name>", "DataBase collection name (logs)", "logs")
  .option("-u, --user <username>", "DataBase username (root)", "root")
  .option("-p, --pass <password>", "DataBase username password ()", "")
  .parse(process.argv)

var mongoOptions = "mongodb://"
  + (params.user && params.pass
      ? "[" + params.user + ":" + params.pass + "]@"
      : "")
  + params.host
  + ":" + params.port
  + "/" + params.db

MongoClient.connect(mongoOptions, onConnection)

function onConnection(e, db) {

  if (e) {
    return handleError(e)
  }

  var collection = db.collection(params.collection)
  
  process.on("SIGINT", function() {
    db.close(function() {
      process.exit()
    })
  })

  process.stdin.resume()
  process.stdin.setEncoding("utf8")
  process.stdin.on("data", function stdin(data) {

    var document = null

    try {
      document = JSON.parse(data)
    } catch (e) {
      document = {
        msg: data
      }
    }

    collection.insertOne(document, function insertOne(e) {
      if (e) {
        return handleError(e)
      }
    })

    process.stdout.write(JSON.stringify(document) + "\n")

  })
}

function handleError(e) {
  if (e instanceof Error) {
    console.error(e)
  } else {
    console.error(new Error(e))
  }
}
{'fraction_non_alphanumeric': 0.1884498480243161, 'fraction_numerical': 0.01398176291793313, 'mean_word_length': 3.057953144266338, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3149626', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1126, 'n_words': 232}