<commit_before><commit_msg>add app.js to startup the crawler and do some indexing.<commit_after>import crawler from './lib/crawler';
import Indexer from './lib/indexer';

import Opencc from 'opencc';
const opencc = new Opencc('tw2s.json');

let comicIndexer = new Indexer('./newdb');

let initLinks = [
	'http://www.comicvip.com/comic/all.html',
	'http://mh.99770.cc/comiclist/0',
	'http://www.99comic.com/lists/',
	'http://www.dmeden.com/comic/'
];

// init crawlers
let crawlers = initLinks.map(initLink => {
	return new crawler(initLink, 1000);
});


let index = (err, comicList) => {
	if(err) console.log(err);
	else {
		let url = comicList[0].url;
		
		// @fixme
		// not a good pattern
		if(url.indexOf('comicvip') !== -1)
			comicIndexer.add('comicvip', comicList);

		else if(url.indexOf('99770') !== -1)
			comicIndexer.add('99770', comicList);
		
		else if(url.indexOf('99comic') !== -1)
			comicIndexer.add('99comic', comicList);

		else if(url.indexOf('dmeden') !== -1)
			comicIndexer.add('dmeden', comicList);

		else
			console.log('can not find the proper indexer');
	}
};

let main = () => {
	for(crawler of crawlers) {
		crawler.start(index);
	}
};

main();
{'fraction_non_alphanumeric': 0.18808510638297873, 'fraction_numerical': 0.02723404255319149, 'mean_word_length': 3.879668049792531, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6661526', 'n_tokens_mistral': 496, 'n_tokens_neox': 448, 'n_words': 92}