<commit_before>var MPGScraper,
	Xray = require('x-ray'),
	bunyan = require('bunyan');

MPGScraper = (function(){
	// Scrappers init
	var X = Xray();
	X.delay(500, 1000);

	// log method init
	var log = bunyan.createLogger({
		name: 'scraperlogger',
		streams: [
			{
				level: 'trace',
				stream: process.stdout
			}
		]
	});

	// Module configuration
	var FIXTURES_PAGE = "FIXTURES_PAGE";
	var FIXTURE_STATISTICS_PAGE = "FIXTURE_STATISTICS_PAGE";
	var TEAM_PAGE = "TEAM_PAGE";
	var BLOG_PAGE = "BLOG_PAGE";
	var BLOG_TEAM_PAGE = "BLOG_TEAM_PAGE";
	var BLOG_PLAYER_PAGE = "BLOG_PLAYER_PAGE";
	var PAGE_TYPE = [FIXTURES_PAGE, FIXTURE_STATISTICS_PAGE, TEAM_PAGE, BLOG_PAGE, BLOG_TEAM_PAGE, BLOG_PLAYER_PAGE];

	var TYPE_MAPPING = {
		FIXTURES_PAGE: "getResultTableFromFixtureUrl",
		FIXTURE_STATISTICS_PAGE: "getStatisticsFromFixture",
		TEAM_PAGE: "getListTeams",
		BLOG_TEAM_PAGE: "getPlayersFromTeam",
		BLOG_PLAYER_PAGE: "getPlayerStatistics",
	}

	// Module implementation
	var that = {

		// call functions
		call: function(page_type, page_url, successCallback, errorCallback, localX, localLog){
			var isValidPageType = PAGE_TYPE.some(function(type, index, array){
				return page_type === type
			})

			if (isValidPageType){
				var method = TYPE_MAPPING[page_type];
				if (!localX) log.debug('localXray not provided, using global Xray');
				if (!localLog) log.debug('localLog not provided, using global log');
				if (localX) X = localX;
				if (localLog) log = localLog;

				log.info("Scraping "+page_type+ " URL = " + page_url);

				that[method](page_url, successCallback, errorCallback, X, log);
			}else{
				log.warn('Unrecognized page_type \''+page_type+'\'.')
			}
		},

		// Methods
		getResultTableFromFixtureUrl: function(fixturesUrl, successCallback, errorCallback, X, log){
			X(fixturesUrl, "#tabres > table > tbody", {
				'fixturesResults': X('tr:not(:first-child)',  [{
				homeTeam:  'td.equipeDom b',
				awayTeam:  'td.equipeExt b',
				score: 'td.score',
				scoreDetailUrl: 'td.score a@href'
			}])})(function(err, results){
				if(!err){
					results.fixturesResults.forEach(function(fixtureResult, index, fixturesResults){
						var score = fixtureResult.score.split(' - ');
						if (score.length>1){
							fixtureResult.played = 1;
							fixtureResult.homeScore = parseInt(score[0]);
							fixtureResult.awayScore = parseInt(score[1]);
						}else{
							fixtureResult.played = 0;
						}
						delete fixtureResult.score;
						
					})
					if (successCallback) successCallback(results);
					
				}else{
					if(errorCallback) errorCallback(err);
				}
			});
		},

		getStatisticsFromFixture: function(fixtureStatisticsUrl, successCallback, errorCallback, localX, localLog){
			X(fixtureStatisticsUrl, {
				script: "#content > script:nth-child(3)",
				players: X('#content .joueur', [{
						playerId:"@id", 
						note: ".note p"
					}]),
				})(function(err, statistics){
					log.info(err);
					var script = statistics.script;
					if(!err){
						var regex = /var stat = (.*);/g;
						matchs = script.match(regex);
						if (matchs){
							var result = JSON.parse(matchs[0].split(" = ")[1].split(";")[0]);
							for(index in statistics.players){
								var playerId = statistics.players[index].playerId;
								var note = statistics.players[index].note;
								result[playerId].note = note;
							}
							if (successCallback) successCallback(result);
						}
					}else{
						if(errorCallback) errorCallback(err);
					}
			});
		},

		getListTeams: function(blogURL, successCallback, errorCallback, localX, localLog){
			X(blogURL, "#menu-item-1247 > ul li", [{
				teamName: 'a',
				teaURL: 'a@href'
			}])(function(err, teams){
				if(!err){
					if (successCallback) successCallback(teams);
				}else{
					if(errorCallback) errorCallback(err);
				}
			})
<commit_msg>Adding home and away information<commit_after>var MPGScraper,
	Xray = require('x-ray'),
	bunyan = require('bunyan');

MPGScraper = (function(){
	// Scrappers init
	var X = Xray();
	X.delay(500, 1000);

	// log method init
	var log = bunyan.createLogger({
		name: 'scraperlogger',
		streams: [
			{
				level: 'trace',
				stream: process.stdout
			}
		]
	});

	// Module configuration
	var FIXTURES_PAGE = "FIXTURES_PAGE";
	var FIXTURE_STATISTICS_PAGE = "FIXTURE_STATISTICS_PAGE";
	var TEAM_PAGE = "TEAM_PAGE";
	var BLOG_PAGE = "BLOG_PAGE";
	var BLOG_TEAM_PAGE = "BLOG_TEAM_PAGE";
	var BLOG_PLAYER_PAGE = "BLOG_PLAYER_PAGE";
	var PAGE_TYPE = [FIXTURES_PAGE, FIXTURE_STATISTICS_PAGE, TEAM_PAGE, BLOG_PAGE, BLOG_TEAM_PAGE, BLOG_PLAYER_PAGE];

	var TYPE_MAPPING = {
		FIXTURES_PAGE: "getResultTableFromFixtureUrl",
		FIXTURE_STATISTICS_PAGE: "getStatisticsFromFixture",
		TEAM_PAGE: "getListTeams",
		BLOG_TEAM_PAGE: "getPlayersFromTeam",
		BLOG_PLAYER_PAGE: "getPlayerStatistics",
	}

	// Module implementation
	var that = {

		// call functions
		call: function(page_type, page_url, successCallback, errorCallback, localX, localLog){
			var isValidPageType = PAGE_TYPE.some(function(type, index, array){
				return page_type === type
			})

			if (isValidPageType){
				var method = TYPE_MAPPING[page_type];
				if (!localX) log.debug('localXray not provided, using global Xray');
				if (!localLog) log.debug('localLog not provided, using global log');
				if (localX) X = localX;
				if (localLog) log = localLog;

				log.info("Scraping "+page_type+ " URL = " + page_url);

				that[method](page_url, successCallback, errorCallback, X, log);
			}else{
				log.warn('Unrecognized page_type \''+page_type+'\'.')
			}
		},

		// Methods
		getResultTableFromFixtureUrl: function(fixturesUrl, successCallback, errorCallback, X, log){
			X(fixturesUrl, "#tabres > table > tbody", {
				'fixturesResults': X('tr:not(:first-child)',  [{
				homeTeam:  'td.equipeDom b',
				awayTeam:  'td.equipeExt b',
				score: 'td.score',
				scoreDetailUrl: 'td.score a@href'
			}])})(function(err, results){
				if(!err){
					results.fixturesResults.forEach(function(fixtureResult, index, fixturesResults){
						var score = fixtureResult.score.split(' - ');
						if (score.length>1){
							fixtureResult.played = 1;
							fixtureResult.homeScore = parseInt(score[0]);
							fixtureResult.awayScore = parseInt(score[1]);
						}else{
							fixtureResult.played = 0;
						}
						delete fixtureResult.score;
						
					})
					if (successCallback) successCallback(results);
					
				}else{
					if(errorCallback) errorCallback(err);
				}
			});
		},

		getStatisticsFromFixture: function(fixtureStatisticsUrl, successCallback, errorCallback, localX, localLog){
			X(fixtureStatisticsUrl, {
				script: "#content > script:nth-child(3)",
				homePlayers: X('#content .joueur.home', [{
						playerId:"@id", 
						note: ".note p"
					}]),
				awayPlayers: X('#content .joueur.away', [{
						playerId:"@id", 
						note: ".note p"
					}]),
				})(function(err, statistics){
					if(!err){
						var script = statistics.script;
						for(var i = 0; i<statistics.homePlayers.length; i++){
							statistics.homePlayers[i].is_home = "1";

						}
						for(var i = 0; i<statistics.awayPlayers.length; i++){
							statistics.awayPlayers[i].is_home = "0";
						}
						statistics.players = statistics.awayPlayers.concat(statistics.homePlayers);
						
						var regex = /var stat = (.*);/g;
						matchs = script.match(regex);
						if (matchs){
							var result = JSON.parse(matchs[0].split(" = ")[1].split(";")[0]);
							for(index in statistics.players){
								var playerId = statistics.players[index].playerId;
								var note = statistics.players[index].note;
								var is_home = statistics.players[index].is_home;
								result[playerId].note = note;
								result[playerId].is_home = is_home;
							}
							if (successCallback) successCallback(result);
						}
					}else{
						if(errorCallback) errorCallback(err);
					}
			});
		},

		getListTeams: function(blogURL, successCallback, errorCallback, localX, localLog){
			X(blogURL, "#menu-item-1247 > ul li", [{
				teamName: 'a',
				teaURL: 'a@href'
			}])(function(err, teams){
				if(!err){
					if (successCallback) successCallback(teams);
				}else{
					if(errorCallback) errorCallback(err);
				}
			})

{'fraction_non_alphanumeric': 0.14336175395858708, 'fraction_numerical': 0.00535931790499391, 'mean_word_length': 3.464926590538336, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4080175', 'n_tokens_mistral': 3578, 'n_tokens_neox': 2797, 'n_words': 551}