<commit_before>var fs = require('fs');
var natural = require('natural');
var async = require('async');

var licenseDir = './data/license/';
var nonLicenseDir = './data/not-license/';
var testDir = './data/test/';

var minWordLength = 5;
var maxWordCount = 25;

var classifier = new natural.BayesClassifier();
var tokenizer = new natural.WordTokenizer();

// using async.series to maintain the order of execution

async.series([
  //adding the licenses to the classifier
  function(callback) {
    console.log('Processing training data...');
    fs.readdir(licenseDir, function(err, files) {
      if(err) throw err;
      var count = 0;
      files.forEach(function(file){
        count++;
        fs.readFile(licenseDir + file, 'utf-8', function(err,data){
          if (err) throw err;
          console.log("classifying: " + licenseDir + file);
          var frequentWords = getFrequency(tokenizer.tokenize(data), minWordLength, maxWordCount);
          //console.log(frequentWords);
          classifier.addDocument(frequentWords,'license');

          //finished reading last license file,
          //signal we are can move on to non-licenses
          if(--count === 0)
            callback();
        });
      });
    });
  },
  //adding the non-licenses to the classifier
  function(callback) {
    fs.readdir(nonLicenseDir, function(err, files) {
      if(err) throw err;
      var count = 0;
      files.forEach(function(file){
        count++;
        fs.readFile(nonLicenseDir + file, 'utf-8', function(err,data){
          if (err) throw err;

          console.log("classifying: " + nonLicenseDir + file);
          var frequentWords = getFrequency(tokenizer.tokenize(data), minWordLength, maxWordCount);
          //console.log(frequentWords);
          classifier.addDocument(frequentWords,'not license');

          if(--count === 0) { //finished reading last file, so now we can train
            classifier.train();
            callback();
          }
        });
      });
    });
  },
  // read our test data files and see if our classifier worked.
  function(callback) {
    console.log('\n\nTesting classifier:');
    fs.readdir(testDir, function(err, files) {
      if(err) throw err;
      var count = 0;
      files.forEach(function(file){
        count++;
        fs.readFile(testDir + file, 'utf-8', function(err,data){
          if (err) throw err;


          var frequentWords = getFrequency(tokenizer.tokenize(data), minWordLength, maxWordCount);

          console.log(testDir + file + ' classified as: ' + classifier.classify(frequentWords));
          //console.log(frequentWords);

          if(--count === 0)
            callback();
        });
      });
    });
  }
]);








// Returns the list of the most common words in the array
// That are greater or equal in length to minLength
// and appear moretimes than cutoff
// Inspired by http://codereview.stackexchange.com/a/63979
function getFrequency(words, minLength, cutoff) {
  //var cleanString = string.replace(/[\.,-\/#!$%\^&\*;:{}=\-_`~()]/g,""),
  //words = cleanString.split(' '),
  var frequencies = {};
  var frequency;

  for(var i=0; i < words.length; i++) {

    var word = words[i].toLowerCase();

    if(word.length < minLength)
      continue;

    frequencies[word] = frequencies[word] || 0;
    frequencies[word]++;
  }

  words = Object.keys(frequencies);
  return words.sort(function (a,b) { return frequencies[b] -frequencies[a];}).slice(0,cutoff);
}
<commit_msg>refactoring main.js to remove duplicate code<commit_after>var fs = require('fs');
var natural = require('natural');
var async = require('async');

var licenseDir = './data/license/';
var nonLicenseDir = './data/not-license/';
var testDir = './data/test/';

var minWordLength = 5;
var maxWordCount = 25;

var classifier = new natural.BayesClassifier();
var tokenizer = new natural.WordTokenizer();

// using async.series to maintain the order of execution

async.series([
  //adding the licenses to the classifier
  function(callback) {
    console.log('Processing training data...');
    classifyDirectory(licenseDir, 'license', callback);
  },

  //adding the non-licenses to the classifier
  function(callback) {
    classifyDirectory(nonLicenseDir, 'not license', callback);
  },

  //train on the data
  function(callback) {
    classifier.train();
    callback();
  },
  // read our test data files and see if our classifier worked.
  function(callback) {
    console.log('\nTesting classifier:');
    classifyDirectory(testDir, 'test', callback);
  }
]);

//iterate over files in a directory dir and classify them with classification
function classifyDirectory(dir, classification, callback) {
  fs.readdir(dir, function(err, files) {
    if(err) throw err;
    var count = 0;
    files.forEach(function(file){
      count++;
      fs.readFile(dir + file, 'utf-8', function(err,data){
        if (err) throw err;

        var frequentWords = getFrequency(tokenizer.tokenize(data), minWordLength, maxWordCount);
        //console.log(frequentWords);

        //if test, output results
        if(classification === 'test') {
          console.log(testDir + file + ' classified as: ' + classifier.classify(frequentWords));

        } else { //otherwise, add document.
          console.log("classifying: " + licenseDir + file);
          classifier.addDocument(frequentWords, classification);
        }

        //finished reading last license file,
        //signal we are can move on to non-licenses
        if(--count === 0)
          callback();
      });
    });
  });
}

// Returns the list of the most common words in the array
// That are greater or equal in length to minLength
// and appear moretimes than cutoff
// Inspired by http://codereview.stackexchange.com/a/63979
function getFrequency(words, minLength, cutoff) {
  //var cleanString = string.replace(/[\.,-\/#!$%\^&\*;:{}=\-_`~()]/g,""),
  //words = cleanString.split(' '),
  var frequencies = {};
  var frequency;

  for(var i=0; i < words.length; i++) {

    var word = words[i].toLowerCase();

    if(word.length < minLength)
      continue;

    frequencies[word] = frequencies[word] || 0;
    frequencies[word]++;
  }

  words = Object.keys(frequencies);
  return words.sort(function (a,b) { return frequencies[b] -frequencies[a];}).slice(0,cutoff);
}

{'fraction_non_alphanumeric': 0.14550641940085593, 'fraction_numerical': 0.005389126644476145, 'mean_word_length': 2.892658852560148, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2777421', 'n_tokens_mistral': 1928, 'n_tokens_neox': 1836, 'n_words': 492}