<commit_before>var http = require('http');
var fs = require('fs');
var cheerio = require('cheerio');
var _bekazon = require('./bekazon');

var bekazon = new _bekazon();

module.exports = function(){
	var bekazonUrl = 'http://bekazon.com/image/cache/data/digital_comic/';
	
	/**
	downloader function
	*/
	
	var artists = [];
	this.downloadArtist = function(){
		console.log('downloading '+bekazonUrl);
		http.get(bekazonUrl, function(resp){
			var datas = [];
			
			resp.on('data', function(data){
				datas.push(data);
			}).on('end', function(){
				// list of artists
				var stringResponse = Buffer.concat(datas).toString();
				var $ = cheerio.load(stringResponse);
				
				var a = $('a').not(0); // remove up anchor
				if( a.length > 0 ){
					a.each(function(i,e){
						if( !/^coverthumb/ig.test( e.attribs.href ) && e.attribs.href != 'imagecachedata' ){
							var artistName = e.attribs.href.replace( /[^a-z0-9]/ig, '' );
							artists.push( artistName );
						}
					});
					_loopArtist();
				}
			});
		});
	}
	
	function _loopArtist(){
		if( artists.length > 0 ){
			bekazon.saveArtist(artists[0], function(err, result){
				_downloadChapter(artists[0]);
			});
		}else{
			console.log('done');
			process.exit();
		}
	}
	
	var chapters = [];
	function _downloadChapter( anArtist ){
		var existingChapter = [];
		bekazon.getChapter(anArtist, null, function(err, savedChapters){
			console.log(err);
			if( savedChapters && savedChapters.length > 0 ){
				existingChapter = savedChapters.map(function(value, index, currentArr){
					console.log(value.chapter);
					return value.chapter;
				});
			}
			
			console.log('downloading '+bekazonUrl+anArtist+'/');
			http.get( bekazonUrl+anArtist+'/', function(resp){
				var _data = [];
				resp.on('data', function(data){
					_data.push(data);
				}).on('end', function(){
					var stringResponse = Buffer.concat(_data);
					var $ = cheerio.load(stringResponse);
					var url = $('a');
					<commit_msg>disable exiting app after download<commit_after>var http = require('http');
var fs = require('fs');
var cheerio = require('cheerio');
var _bekazon = require('./bekazon');

var bekazon = new _bekazon();

module.exports = function(){
	var bekazonUrl = 'http://bekazon.com/image/cache/data/digital_comic/';
	
	/**
	downloader function
	*/
	
	var artists = [];
	this.downloadArtist = function(){
		console.log('downloading '+bekazonUrl);
		http.get(bekazonUrl, function(resp){
			var datas = [];
			
			resp.on('data', function(data){
				datas.push(data);
			}).on('end', function(){
				// list of artists
				var stringResponse = Buffer.concat(datas).toString();
				var $ = cheerio.load(stringResponse);
				
				var a = $('a').not(0); // remove up anchor
				if( a.length > 0 ){
					a.each(function(i,e){
						if( !/^coverthumb/ig.test( e.attribs.href ) && e.attribs.href != 'imagecachedata' ){
							var artistName = e.attribs.href.replace( /[^a-z0-9]/ig, '' );
							artists.push( artistName );
						}
					});
					_loopArtist();
				}
			});
		});
	}
	
	function _loopArtist(){
		if( artists.length > 0 ){
			bekazon.saveArtist(artists[0], function(err, result){
				_downloadChapter(artists[0]);
			});
		}else{
			console.log('done');
			//process.exit();
		}
	}
	
	var chapters = [];
	function _downloadChapter( anArtist ){
		var existingChapter = [];
		bekazon.getChapter(anArtist, null, function(err, savedChapters){
			console.log(err);
			if( savedChapters && savedChapters.length > 0 ){
				existingChapter = savedChapters.map(function(value, index, currentArr){
					console.log(value.chapter);
					return value.chapter;
				});
			}
			
			console.log('downloading '+bekazonUrl+anArtist+'/');
			http.get( bekazonUrl+anArtist+'/', function(resp){
				var _data = [];
				resp.on('data', function(data){
					_data.push(data);
				}).on('end', function(){
					var stringResponse = Buffer.concat(_data);
					var $ = cheerio.load(stringResponse);
					var url = $('a');
					
{'fraction_non_alphanumeric': 0.17897655659188302, 'fraction_numerical': 0.00403327451474666, 'mean_word_length': 3.2620837808807734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5153221', 'n_tokens_mistral': 1695, 'n_tokens_neox': 1340, 'n_words': 250}