<commit_before>var fs = require('fs'),
	path = require('path'),
	_ = require('lodash');
	ptn = require('parse-torrent-name'),
	replaceExt = require('replace-ext'),
	junk = require('junk');

var SUPPORTED_FILETYPES = new RegExp("(avi|mkv|mpeg|mov|mp4|m4v|wmv)$","g");  //Pipe seperated

var walk = function(dir, done) {
	var results = [];
	fs.readdir(dir, function(err, list) {
		if (err)
			return done(err);
		var i = 0;
		(function next() {
			var file = list[i++];
			if (!file)
				return done(null, results);
			file = dir + '/' + file;
			fs.stat(file, function(err, stat) {
				if (stat && stat.isDirectory()) {
					walk(file, function(err, res) {
						results = results.concat(res);
						next();
					});
				} else {
					var ext = file.split(".");
					ext = ext[ext.length - 1];
					if (ext.match(SUPPORTED_FILETYPES)) {
						results.push(file);
					}
					next();
				}
			});
		})();
	});
};
var FileParser = function(file){
	var name = replaceExt(file, '');
	parsed = ptn(name);
	//console.log(info);
	return ptn(name);
};
var FolderParser = function(dir, callback){
	walk(dir, function(err, files){
		if(err){
			return callback(err);
		}
		var list;
		files = files.filter(junk.not);
		list = _.each(files, function(file, index, list){
			var parsedFile = FileParser(path.basename(file));
			parsedFile.file = path.basename(file);
			if(file !== parsedFile && typeof parsedFile == 'object' && parsedFile.title){
				list[index] = parsedFile;
			}else{
				list[index] = null;
			}
			return file;
		});
		//console.log(list)
		var shows = [];
		_.each(list,  function(item){
			if(item && item.title && item.season){
				// HACK: All DW episodes after 2005 are separated
				if(item.title == 'Doctor Who' && item.year == '2005'){
					item.title = 'Doctor Who 2005';
				}
				// START HACK: Hack around the various parse-torrent-name bugs
				// Trim whitespaces
				var tmp = item.title.trim();
				// Is there an - or _ at the end? Then there probably was a parsing error
				if(tmp[tmp.length - 1] == '-' 
					|| tmp[tmp.length - 1] == '_'){
					// Remove the offending - or _
					tmp = tmp.slice(0, tmp.length - 1);
				}
				// Use whitespace instead of - and _ to separate words
				tmp = tmp.replace(/_.*_/, ' ');
				tmp = tmp.replace(/-.*-/, ' ');
				// Trim once more
				tmp = tmp.trim();
				// Use tmp variable as show title from now on
				if(tmp){
					item.title = tmp;
				}
				// END HACK
				if(!shows[item.title]){<commit_msg>[REMOVE OR IMPROVE] Hack for The Office U.S<commit_after>var fs = require('fs'),
	path = require('path'),
	_ = require('lodash');
	ptn = require('parse-torrent-name'),
	replaceExt = require('replace-ext'),
	junk = require('junk');

var SUPPORTED_FILETYPES = new RegExp("(avi|mkv|mpeg|mov|mp4|m4v|wmv)$","g");  //Pipe seperated

var walk = function(dir, done) {
	var results = [];
	fs.readdir(dir, function(err, list) {
		if (err)
			return done(err);
		var i = 0;
		(function next() {
			var file = list[i++];
			if (!file)
				return done(null, results);
			file = dir + '/' + file;
			fs.stat(file, function(err, stat) {
				if (stat && stat.isDirectory()) {
					walk(file, function(err, res) {
						results = results.concat(res);
						next();
					});
				} else {
					var ext = file.split(".");
					ext = ext[ext.length - 1];
					if (ext.match(SUPPORTED_FILETYPES)) {
						results.push(file);
					}
					next();
				}
			});
		})();
	});
};
var FileParser = function(file){
	var name = replaceExt(file, '');
	parsed = ptn(name);
	//console.log(info);
	return ptn(name);
};
var FolderParser = function(dir, callback){
	walk(dir, function(err, files){
		if(err){
			return callback(err);
		}
		var list;
		files = files.filter(junk.not);
		list = _.each(files, function(file, index, list){
			var parsedFile = FileParser(path.basename(file));
			parsedFile.file = path.basename(file);
			if(file !== parsedFile && typeof parsedFile == 'object' && parsedFile.title){
				list[index] = parsedFile;
			}else{
				list[index] = null;
			}
			return file;
		});
		//console.log(list)
		var shows = [];
		_.each(list,  function(item){
			if(item && item.title && item.season){
				// HACK: All DW episodes after 2005 are separated
				if(item.title == 'Doctor Who' && item.year == '2005'){
					item.title = 'Doctor Who 2005';
				}
				// HACK: The Office
				if(item.title == 'The Office'){
					item.title = 'The Office US';
				}
				// START HACK: Hack around the various parse-torrent-name bugs
				// Trim whitespaces
				var tmp = item.title.trim();
				// Is there an - or _ at the end? Then there probably was a parsing error
				if(tmp[tmp.length - 1] == '-' 
					|| tmp[tmp.length - 1] == '_'){
					// Remove the offending - or _
					tmp = tmp.slice(0, tmp.length - 1);
				}
				// Use whitespace instead of - and _ to separate words
				tmp = tmp.replace(/_.*_/, ' ');
				tmp = tmp.replace(/-.*-/, ' ');
				// Trim once more
				tmp = tmp.trim();
				// Use tmp variable as show title from now on
				if(tmp){
					item.title = tmp;
				}
				// END HACK
				if(!shows[item.title]){
{'fraction_non_alphanumeric': 0.1775940145697972, 'fraction_numerical': 0.007875566056310297, 'mean_word_length': 2.7298091042584436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357954', 'n_tokens_mistral': 2194, 'n_tokens_neox': 1778, 'n_words': 444}