<commit_before>var path = require('path');

var Torrent = require(__dirname+'/models/Torrent.js');

var extToIgnore = [
      '.url', '.txt', '.ico', '.srt', '.gif', '.log', 
      '.nfo', '.cbr', '.ass', '.lnk', '.rtf', '.bc!', 
      '.bmp', '.m3u', '.mht', '.cue', '.sfv', '.diz',
      '.azw3', '.odt', '.chm', '.md5', '.idx', '.sub',
      '.ini', '.html', '.ssa', '.lit', '.xml', '.clpi',
      '.bup', '.ifo', '.htm', '.info', '.css', '.php', 
      '.js', '.jar', '.json', '.sha', '.docx', '.csv'
    ];
var extToCateg = {
  'Images' : ['.png', '.jpeg', '.jpg'],    
  'Program' : ['.exe'],
  'ISO'   : ['.rar', '.iso', '.zip', '.dmg', '.tgz', '.gz', '.chd', '.7z', '.cab', '.apk', '.cdr', '.wbfs', '.dat', '.rar', '.lzma', '.mds', '.gho', '.ima', '.nrg'],  
  'Book' : ['.epub', '.pdf', '.cbz', '.cbr', '.cb7', '.cba', '.cbt', '.djvu', '.fb2', '.mobi'],  
  'Music' : ['.flac', '.mp3', '.m4p', '.m4r', '.m4a', '.m4b', '.ape', '.wma', '.ogg'],  
  'Video' : ['.mp4', '.mkv', '.3gp', '.flv', '.f4v', '.avi', '.rm', '.rmvb', '.wmv', '.mov', '.mpg', '.mpeg', '.ts', '.m2ts', '.m4v', '.asf', '.vob', '.divx'],
};

console.logCopy = console.log.bind(console);

console.log = function(data) {
   if(arguments.length) {
      var timestamp = '[' + new Date().toUTCString() + ']';
      this.logCopy(timestamp, arguments);
   }
};

var filter = { 'category' : /Unknown/ };

var stream = Torrent.find(filter).sort({'lastmod': -1}).limit(100).stream();
stream.on('data', function(torrent){
  console.log("Treating "+torrent._id+" categorization");
  var self = this;
  if(typeof torrent.files !== "undefined") {
    var files = torrent.files;
    var exts = [];
    files.forEach(function(file){
      var ext = path.extname(file).toLowerCase();
      if( extToIgnore.indexOf(ext) < 0 ) {
        if(ext.length < 6) {
          exts.push( ext ); 
        }
      }
    });
    exts.sort();
    exts = exts.filter( function(value, index, self){
      return self.indexOf(value) === index;
    });
    exts = exts.filter( function(value){
      return value.length !== 0
    });

    var category = 'Unknown';
    for(var k in extToCateg) {
      var v = extToCateg[k];
      //if too much extensions we can't know
      if(exts.length<5) {
        //for each uniq extension
        exts.forEach(function(e){
          //check if in category (asc)
          if(v.indexOf(e)>=0){
            category = k;
          }
        });
      }
    }
    torrent.category=category;
    torrent.save(function(err){
      if(err) {console.log(err); process.exit(1);}
      console.log("Categorized as "+category+" !");
    })
  }
})

stream.on('error', function(err) {
  console.log("Error : "+err); process.exit(1);
});

stream.on('close', function(){
  process.exit();
});
<commit_msg>Updated categorize script to use Other by default<commit_after>var path = require('path');

var Torrent = require(__dirname+'/models/Torrent.js');

var extToIgnore = [
      '.url', '.txt', '.ico', '.srt', '.gif', '.log', 
      '.nfo', '.cbr', '.ass', '.lnk', '.rtf', '.bc!', 
      '.bmp', '.m3u', '.mht', '.cue', '.sfv', '.diz',
      '.azw3', '.odt', '.chm', '.md5', '.idx', '.sub',
      '.ini', '.html', '.ssa', '.lit', '.xml', '.clpi',
      '.bup', '.ifo', '.htm', '.info', '.css', '.php', 
      '.js', '.jar', '.json', '.sha', '.docx', '.csv'
    ];
var extToCateg = {
  'Images' : ['.png', '.jpeg', '.jpg'],    
  'Program' : ['.exe'],
  'ISO'   : ['.rar', '.iso', '.zip', '.dmg', '.tgz', '.gz', '.chd', '.7z', '.cab', '.apk', '.cdr', '.wbfs', '.dat', '.rar', '.lzma', '.mds', '.gho', '.ima', '.nrg'],  
  'Book' : ['.epub', '.pdf', '.cbz', '.cbr', '.cb7', '.cba', '.cbt', '.djvu', '.fb2', '.mobi'],  
  'Music' : ['.flac', '.mp3', '.m4p', '.m4r', '.m4a', '.m4b', '.ape', '.wma', '.ogg'],  
  'Video' : ['.mp4', '.mkv', '.3gp', '.flv', '.f4v', '.avi', '.rm', '.rmvb', '.wmv', '.mov', '.mpg', '.mpeg', '.ts', '.m2ts', '.m4v', '.asf', '.vob', '.divx'],
};

console.logCopy = console.log.bind(console);

console.log = function(data) {
   if(arguments.length) {
      var timestamp = '[' + new Date().toUTCString() + ']';
      this.logCopy(timestamp, arguments);
   }
};

var filter = { 'category' : /Unknown/ };

var stream = Torrent.find(filter).sort({'imported': -1}).limit(100).stream();
stream.on('data', function(torrent){
  var self = this;
  self.pause();
  console.log("Treating "+torrent._id+" categorization");
  if(typeof torrent.files !== "undefined") {
    var files = torrent.files;
    var exts = [];
    files.forEach(function(file){
      var ext = path.extname(file).toLowerCase();
      if( extToIgnore.indexOf(ext) < 0 ) {
        if(ext.length < 6) {
          exts.push( ext ); 
        }
      }
    });
    exts.sort();
    exts = exts.filter( function(value, index, self){
      return self.indexOf(value) === index;
    });
    exts = exts.filter( function(value){
      return value.length !== 0
    });

    var category = 'Other';
    for(var k in extToCateg) {
      var v = extToCateg[k];
      //if too much extensions we can't know
      if(exts.length<5) {
        //for each uniq extension
        exts.forEach(function(e){
          //check if in category (asc)
          if(v.indexOf(e)>=0){
            category = k;
          }
        });
      }
    }
    torrent.category=category;
    torrent.save(function(err){
      if(err) {console.log(err); process.exit(1);}
      console.log("Categorized as "+category+" !");
      self.resume();
    })
  }
})

stream.on('error', function(err) {
  console.log("Error : "+err); process.exit(1);
});

stream.on('close', function(){
  process.exit();
});

{'fraction_non_alphanumeric': 0.271626913492346, 'fraction_numerical': 0.009611961552153792, 'mean_word_length': 2.5563291139240505, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1298419', 'n_tokens_mistral': 2129, 'n_tokens_neox': 2091, 'n_words': 461}