<commit_before>var eventId = 'rp14';

var fs = require('fs');
var path = require('path');

var allTracks = {
	'Business & Innovation':  { id:'business-innovation', label_de:'Business & Innovation',  label_en:'Business & Innovation', color:[194.0, 56.0, 24.0, 1.0] },
	'Science & Technology':   { id:'science-technology',  label_de:'Wissenschaft & Technik', label_en:'Science & Technology' , color:[0.0, 0.0, 0.0, 1.0] },
	'Politics & Society':     { id:'politics-society',    label_de:'Politik & Gesellschaft', label_en:'Politics & Society'   , color:[111.0, 79.0, 132.0, 1.0] },
	'Research & Education':   { id:'research-education',  label_de:'Forschung & Bildung',    label_en:'Research & Education' , color:[0.0, 0.0, 0.0, 1.0] },
	'Culture':                { id:'culture',             label_de:'Kultur',                 label_en:'Culture'              , color:[193.0, 117.0, 28.0, 1.0] },
	'Media':                  { id:'media',               label_de:'Medien',                 label_en:'Media'                , color:[78.0, 144.0, 178.0, 1.0] },
	're:publica':             { id:'republica',           label_de:'re:publica',             label_en:'re:publica'           , color:[0.0, 0.0, 0.0, 1.0] },
	're:campaign':            { id:'recampaign',          label_de:'re:campaign',            label_en:'re:campaign'          , color:[0.0, 0.0, 0.0, 1.0] },
	'Other':                  { id:'other',               label_de:'Other',                  label_en:'Other'                , color:[101.0, 156.0, 45.0, 1.0] }
}

var allFormats = {
	'Diskussion': { id:'discussion', label_de:'Diskussion', label_en:'Discussion' },
	'Vortrag':    { id:'talk',       label_de:'Vortrag',    label_en:'Talk'       },
	'Workshop':   { id:'workshop',   label_de:'Workshop',   label_en:'Workshop'   },
	'Aktion':     { id:'action',     label_de:'Aktion',     label_en:'Action'     }
}

var allLevels = {
	'Beginner':         { id:'beginner',     label_de:'Anfänger',         label_en:'Beginner'     },
	'Fortgeschrittene': { id:'intermediate', label_de:'Fortgeschrittene', label_en:'Intermediate' },
	'Experten':         { id:'advanced',     label_de:'Experten',         label_en:'Advanced'     }
};

var allLanguages = {
	'Englisch': { id:'en', label_de:'Englisch', label_en:'English' },
	'Deutsch':  { id:'de', label_de:'Deutsch',  label_en:'German'  }
};

var allDays = {
	'06.05.2014': { 'id':'rp14-day-1', 'label_de':'6. Mai', 'label_en':'6. May', 'date':'2014-05-06' },
	'07.05.2014': { 'id':'rp14-day-2', 'label_de':'7. Mai', 'label_en':'7. May', 'date':'2014-05-07' },
	'08.05.2014': { 'id':'rp14-day-3', 'label_de':'8. Mai', 'label_en':'8. May', 'date':'2014-05-08' },
};

// we now supply a order preference with the location
var locationOrderPreference = [
		'rp14-location-2594', // stage 1
		'rp14-location-2595', // stage 2
		'rp14-location-2596', // stage 3
		'rp14-location-2597', // stage 4
		'rp14-location-2598', // stage 5
		'rp14-location-2599', // stage 6
		'rp14-location-2600', // stage A
		'rp14-location-2601', // stage B
		'rp14-location-2602', // stage C
		'rp14-location-2603', // stage D
		'rp14-location-2604', // stage E
		'rp14-location-2693', // stage J
		'rp14-location-2692', // stage T
		'rp14-location-2708', // store
		'rp14-location-2710', // GIG lounge
		'rp14-location-2709', // GIG makerspace
		'rp14-location-2711', // MIKZ
		'rp14-location-2712', // new thinking
		'rp14-location-2713', // republica
];

var eventURLPrefix = "https://14.re-publica.de/";



exports.scrape = function (callback) {
	require('../lib/json_requester').get(
		{
			urls: {
				sessions: 'http://re-publica.de/event/1/sessions/json',
				speakers: 'http://re-publica.de/event/1/speakers/json',
				rooms:    'http://re-publica.de/event/1/rooms.json'
			}
		},
		function (result) {
			var data = [];

			var sessionList  = result.sessions.items;
			var speakerList  = result.speakers.items;
			var locationList = toArray(result.rooms.rooms      );

			var locationMap = {};
			var speakerMap = {};

			speakerList.forEach(function (speaker) {
				// skip potential invalid speakers, those happen.
				if (speaker.uid == "" || speaker.label == "") return;

				var entry = {
					'id': 'rp14-speaker-'+speaker.uid,
					'name': speaker.label,<commit_msg>remove invalid speakers via trim<commit_after>var eventId = 'rp14';

var fs = require('fs');
var path = require('path');

var allTracks = {
	'Business & Innovation':  { id:'business-innovation', label_de:'Business & Innovation',  label_en:'Business & Innovation', color:[194.0, 56.0, 24.0, 1.0] },
	'Science & Technology':   { id:'science-technology',  label_de:'Wissenschaft & Technik', label_en:'Science & Technology' , color:[0.0, 0.0, 0.0, 1.0] },
	'Politics & Society':     { id:'politics-society',    label_de:'Politik & Gesellschaft', label_en:'Politics & Society'   , color:[111.0, 79.0, 132.0, 1.0] },
	'Research & Education':   { id:'research-education',  label_de:'Forschung & Bildung',    label_en:'Research & Education' , color:[0.0, 0.0, 0.0, 1.0] },
	'Culture':                { id:'culture',             label_de:'Kultur',                 label_en:'Culture'              , color:[193.0, 117.0, 28.0, 1.0] },
	'Media':                  { id:'media',               label_de:'Medien',                 label_en:'Media'                , color:[78.0, 144.0, 178.0, 1.0] },
	're:publica':             { id:'republica',           label_de:'re:publica',             label_en:'re:publica'           , color:[0.0, 0.0, 0.0, 1.0] },
	're:campaign':            { id:'recampaign',          label_de:'re:campaign',            label_en:'re:campaign'          , color:[0.0, 0.0, 0.0, 1.0] },
	'Other':                  { id:'other',               label_de:'Other',                  label_en:'Other'                , color:[101.0, 156.0, 45.0, 1.0] }
}

var allFormats = {
	'Diskussion': { id:'discussion', label_de:'Diskussion', label_en:'Discussion' },
	'Vortrag':    { id:'talk',       label_de:'Vortrag',    label_en:'Talk'       },
	'Workshop':   { id:'workshop',   label_de:'Workshop',   label_en:'Workshop'   },
	'Aktion':     { id:'action',     label_de:'Aktion',     label_en:'Action'     }
}

var allLevels = {
	'Beginner':         { id:'beginner',     label_de:'Anfänger',         label_en:'Beginner'     },
	'Fortgeschrittene': { id:'intermediate', label_de:'Fortgeschrittene', label_en:'Intermediate' },
	'Experten':         { id:'advanced',     label_de:'Experten',         label_en:'Advanced'     }
};

var allLanguages = {
	'Englisch': { id:'en', label_de:'Englisch', label_en:'English' },
	'Deutsch':  { id:'de', label_de:'Deutsch',  label_en:'German'  }
};

var allDays = {
	'06.05.2014': { 'id':'rp14-day-1', 'label_de':'6. Mai', 'label_en':'6. May', 'date':'2014-05-06' },
	'07.05.2014': { 'id':'rp14-day-2', 'label_de':'7. Mai', 'label_en':'7. May', 'date':'2014-05-07' },
	'08.05.2014': { 'id':'rp14-day-3', 'label_de':'8. Mai', 'label_en':'8. May', 'date':'2014-05-08' },
};

// we now supply a order preference with the location
var locationOrderPreference = [
		'rp14-location-2594', // stage 1
		'rp14-location-2595', // stage 2
		'rp14-location-2596', // stage 3
		'rp14-location-2597', // stage 4
		'rp14-location-2598', // stage 5
		'rp14-location-2599', // stage 6
		'rp14-location-2600', // stage A
		'rp14-location-2601', // stage B
		'rp14-location-2602', // stage C
		'rp14-location-2603', // stage D
		'rp14-location-2604', // stage E
		'rp14-location-2693', // stage J
		'rp14-location-2692', // stage T
		'rp14-location-2708', // store
		'rp14-location-2710', // GIG lounge
		'rp14-location-2709', // GIG makerspace
		'rp14-location-2711', // MIKZ
		'rp14-location-2712', // new thinking
		'rp14-location-2713', // republica
];

var eventURLPrefix = "https://14.re-publica.de/";



exports.scrape = function (callback) {
	require('../lib/json_requester').get(
		{
			urls: {
				sessions: 'http://re-publica.de/event/1/sessions/json',
				speakers: 'http://re-publica.de/event/1/speakers/json',
				rooms:    'http://re-publica.de/event/1/rooms.json'
			}
		},
		function (result) {
			var data = [];

			var sessionList  = result.sessions.items;
			var speakerList  = result.speakers.items;
			var locationList = toArray(result.rooms.rooms      );

			var locationMap = {};
			var speakerMap = {};

			speakerList.forEach(function (speaker) {
				// skip potential invalid speakers, those happen.
				if (speaker.uid == "" || speaker.label.trim() == "") return;

				var entry = {
					'id': 'rp14-speaker-'+speaker.uid,
					'name': speaker.label,
{'fraction_non_alphanumeric': 0.1993678295481152, 'fraction_numerical': 0.06743151486771248, 'mean_word_length': 2.7800884955752214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '626364', 'n_tokens_mistral': 3646, 'n_tokens_neox': 3085, 'n_words': 551}