<commit_before>/* get node modules */
var fs = require('fs');
var path = require('path');
var eventId = "32c3";

/* get npm modules */
var scrapyard = require('scrapyard');
var moment = require('moment');
var ent = require('ent');
var cheerio = require('cheerio');
var sanitizeHtml = require('sanitize-html');
var parseCSV = require('csv-parse');
var async = require('async');
var md5 = require('MD5');
var ical = require('ical');
var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
var icalendar = require('icalendar');

var log = require(path.resolve(__dirname, '../../api/lib/log.js'));
var json_requester = require('../lib/json_requester');

var additional_schedule_url = "http://data.conference.bits.io/data/32c3/voc/workshops.schedule.json";
var sendezentrum_schedule_url = "https://frab.das-sendezentrum.de/de/32c3/public/schedule.json";
var sendezentrum_speaker_url = "https://frab.das-sendezentrum.de/de/32c3/public/speakers.json";
var schedule_url = "http://data.conference.bits.io/data/32c3/schedule.json"; //"https://events.ccc.de/congress/2015/Fahrplan/schedule.json";
var speakers_url = "http://data.conference.bits.io/data/32c3/speakers-frap.json" // "https://events.ccc.de/congress/2015/Fahrplan/speakers.json";
var voc_streams_api_url = "https://streaming.media.ccc.de/streams/v1.json";
var poi_titles_url = "https://github.com/NoMoKeTo/c3nav/raw/master/src/projects/32c3/titles.json";
// var pois = "https://raw.githubusercontent.com/NoMoKeTo/c3nav/master/src/projects/32c3/pois.json";
var poi_graph_url = "https://raw.githubusercontent.com/NoMoKeTo/c3nav/master/src/projects/32c3/graph.json";


 //"https://gist.githubusercontent.com/MaZderMind/d5737ab867ade7888cb4/raw/bb02a27ca758e1ca3de96b1bf3f811541436ab9d/streams-v1.json" 
// later at https://streaming.media.ccc.de/streams/v1.json

// for debugging we can just pretend rp14 was today
var originalStartDate = new Date(Date.UTC(2015, 11, 27, 10, 0, 0, 0));
var fakeDate = originalStartDate; // new Date(Date.UTC(2015, 11, 23, 16, 0, 0, 0));<commit_msg>revert schedule URLs<commit_after>/* get node modules */
var fs = require('fs');
var path = require('path');
var eventId = "32c3";

/* get npm modules */
var scrapyard = require('scrapyard');
var moment = require('moment');
var ent = require('ent');
var cheerio = require('cheerio');
var sanitizeHtml = require('sanitize-html');
var parseCSV = require('csv-parse');
var async = require('async');
var md5 = require('MD5');
var ical = require('ical');
var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
var icalendar = require('icalendar');

var log = require(path.resolve(__dirname, '../../api/lib/log.js'));
var json_requester = require('../lib/json_requester');

var additional_schedule_url = "http://data.conference.bits.io/data/32c3/voc/workshops.schedule.json";
var sendezentrum_schedule_url = "https://frab.das-sendezentrum.de/de/32c3/public/schedule.json";
var sendezentrum_speaker_url = "https://frab.das-sendezentrum.de/de/32c3/public/speakers.json";
var schedule_url = "https://events.ccc.de/congress/2015/Fahrplan/schedule.json"; 
//"http://data.conference.bits.io/data/32c3/schedule.json"; //"https://events.ccc.de/congress/2015/Fahrplan/schedule.json";
var speakers_url = "https://events.ccc.de/congress/2015/Fahrplan/speakers.json";
//"http://data.conference.bits.io/data/32c3/speakers-frap.json" // "https://events.ccc.de/congress/2015/Fahrplan/speakers.json";
var voc_streams_api_url = "https://streaming.media.ccc.de/streams/v1.json";
var poi_titles_url = "https://github.com/NoMoKeTo/c3nav/raw/master/src/projects/32c3/titles.json";
// var pois = "https://raw.githubusercontent.com/NoMoKeTo/c3nav/master/src/projects/32c3/pois.json";
var poi_graph_url = "https://raw.githubusercontent.com/NoMoKeTo/c3nav/master/src/projects/32c3/graph.json";


 //"https://gist.githubusercontent.com/MaZderMind/d5737ab867ade7888cb4/raw/bb02a27ca758e1ca3de96b1bf3f811541436ab9d/streams-v1.json" 
// later at https://streaming.media.ccc.de/streams/v1.json

// for debugging we can just pretend rp14 was today
var originalStartDate = new Date(Date.UTC(2015, 11, 27, 10, 0, 0, 0));
var fakeDate = originalStartDate; // new Date(Date.UTC(2015, 11, 23, 16, 0, 0, 0));
{'fraction_non_alphanumeric': 0.18682858477347034, 'fraction_numerical': 0.051377860812704346, 'mean_word_length': 5.06657223796034, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 22, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2451504', 'n_tokens_mistral': 1729, 'n_tokens_neox': 1595, 'n_words': 263}