<commit_before>var fs = require('graceful-fs');
var url = require('url');
var config = require('./config'); // get our config file
var mime = require('mime');
var sizeOf = require('image-size');
var http = require('http');
var walkSync = require('walk-sync');
var async = require('async');

var Tools = {
	fullUrl: function(req) {
	  return url.format({
	    protocol: req.protocol,
	    hostname: req.hostname,
	    port: req.app.settings.port,
	    pathname: req.originalUrl
	  });
	},

/*	getImagesFromDir: function(dir) {
		console.log(dir)
		var _this = this;  
		var results = [];
		var list = fs.readdirSync(dir)
			list.forEach(function(file) {
			file = dir + '/' + file
			console.log(file)
			var stat = fs.statSync(file)
			console.log(stat.isDirectory())
			if (stat && stat.isDirectory()) results = results.concat(Tools.getImages(file))
			else {
				if(mime.lookup(file) === "image/tiff"){
					results.push({
						file: file,
						dir : dir
					});
				} 
			}
		})
		return results
	},
*/
	getImages: function(req, res, next){
		req.images = walkSync(req.body.directory, { globs: ['Y00*/*.tif'] });
		var result = [];
		async.each(req.images, function(path) {
			var dir = path.substring(0, path.lastIndexOf("/"))
			var dirParts = dir.split("/");

			console.log(dir);
			console.log(dirParts);
			result.push({
				dir: dir,
				file: path
			})
		},
		function(err){
			req.images = result;
			next();
		});
		/*req.images = Tools.getImagesFromDir(req.body.directory).sort();*/
		
	},

	fetchUrlContent: function(url, callback) {
		var bodyarr = [];
		http.get(url, function(res) {
			res.on('data', function(chunk){
				bodyarr.push(chunk);
			});
			res.on('end', function(){
				callback(null, bodyarr.join(''));
			});<commit_msg>removed some dev code<commit_after>var fs = require('graceful-fs');
var url = require('url');
var config = require('./config'); // get our config file
var mime = require('mime');
var sizeOf = require('image-size');
var http = require('http');
var walkSync = require('walk-sync');
var async = require('async');

var Tools = {
	fullUrl: function(req) {
	  return url.format({
	    protocol: req.protocol,
	    hostname: req.hostname,
	    port: req.app.settings.port,
	    pathname: req.originalUrl
	  });
	},

/*	getImagesFromDir: function(dir) {
		console.log(dir)
		var _this = this;  
		var results = [];
		var list = fs.readdirSync(dir)
			list.forEach(function(file) {
			file = dir + '/' + file
			console.log(file)
			var stat = fs.statSync(file)
			console.log(stat.isDirectory())
			if (stat && stat.isDirectory()) results = results.concat(Tools.getImages(file))
			else {
				if(mime.lookup(file) === "image/tiff"){
					results.push({
						file: file,
						dir : dir
					});
				} 
			}
		})
		return results
	},
*/
	getImages: function(req, res, next){
		req.images = walkSync(req.body.directory, { globs: ['**/*.tif'] });
		var result = [];
		async.each(req.images, function(path) {
			var dir = path.substring(0, path.lastIndexOf("/"))
			var dirParts = dir.split("/");
			result.push({
				dir: dir,
				file: path
			})
		},
		function(err){
			req.images = result;
			next();
		});
		/*req.images = Tools.getImagesFromDir(req.body.directory).sort();*/
		
	},

	fetchUrlContent: function(url, callback) {
		var bodyarr = [];
		http.get(url, function(res) {
			res.on('data', function(chunk){
				bodyarr.push(chunk);
			});
			res.on('end', function(){
				callback(null, bodyarr.join(''));
			});
{'fraction_non_alphanumeric': 0.18461981566820276, 'fraction_numerical': 0.001152073732718894, 'mean_word_length': 3.2199270959902795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '29733', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1188, 'n_words': 247}