<commit_before>'use strict';

var util = require('util');
var sdk = require('./nitroSdk.js');
var ibl = require('./iblApi/ibl.js');

const ibl_key = '';

var programmes = [];

function clone(obj) {
	return JSON.parse(JSON.stringify(obj));
}

function dumpProgrammes() {

	var index = 100000;

	var entries = [];
	for (let programme of programmes) {
		var parent = clone(programme);
		delete parent.initial_children;
		for (let child of programme.initial_children) {
			child.parent = parent;
			entries.push(child);
		}
	}

	console.log('#index|type|name|pid|available|expires|episode|seriesnum|episodenum|versions|duration|desc|channel|categories|thumbnail|timeadded|guidance|web');

	for (let entry of entries) {
		if (entry.type == 'episode') {

			var versions = '';
			var duration = 0;
			for (let version of entry.versions) {
				versions = versions + (versions ? ',' : '') + version.kind;
				if (duration == 0) duration = version.duration.value;
			}

			var s = ''+(index++)+'|';
			s += 'tv|';
			s += entry.title + '|';
			s += entry.id + '|';
			s += entry.versions[0].availability.start + '|';
			s += entry.versions[0].availability.end||'' + '|';
			s += '1|';
			s += (entry.parent_position||1) + '|';
			s += versions +'|';
			s += sdk.iso8601durationToSeconds(duration) + '|';
			s += (entry.synopses.small ? entry.synopses.small : (entry.synopses.medium ?
				entry.synopses.medium : (entry.synopses.large ? entry.synopses.large : ''))) + '|';
			s += entry.master_brand.id + '|';
			s += entry.categories.join(',')+'|';
			s += entry.images.standard + '|';
			s += new Date().toISOString() + '|';
			s += entry.guidance + '|';
			s += 'http://bbc.co.uk/programmes/'+entry.id;

			console.log(s);

		}
	}
}

function getProgrammes(chid,page) {
	var query = sdk.newQuery();
   	query.add(ibl.commonLangEn);
   	query.add(ibl.commonPage,page);
	query.add(ibl.commonInitialChildCount,4);
	console.error(chid+' page: '+page);
   	sdk.make_request(ibl.host,ibl.getChannelsProgrammes(chid),ibl_key,query,{},function(obj){
		programmes = programmes.concat(obj.channel_programmes.elements);<commit_msg>gip_ibl; add missing field<commit_after>'use strict';

var util = require('util');
var sdk = require('./nitroSdk.js');
var ibl = require('./iblApi/ibl.js');

const ibl_key = '';

var programmes = [];

function clone(obj) {
	return JSON.parse(JSON.stringify(obj));
}

function dumpProgrammes() {

	var index = 100000;

	var entries = [];
	for (let programme of programmes) {
		var parent = clone(programme);
		delete parent.initial_children;
		for (let child of programme.initial_children) {
			child.parent = parent;
			entries.push(child);
		}
	}

	console.log('#index|type|name|pid|available|expires|episode|seriesnum|episodenum|versions|duration|desc|channel|categories|thumbnail|timeadded|guidance|web');

	for (let entry of entries) {
		if (entry.type == 'episode') {

			var versions = '';
			var duration = 0;
			for (let version of entry.versions) {
				versions = versions + (versions ? ',' : '') + version.kind;
				if (duration == 0) duration = version.duration.value;
			}

			var s = ''+(index++)+'|';
			s += 'tv|';
			s += entry.title + '|';
			s += entry.id + '|';
			s += entry.versions[0].availability.start + '|';
			s += (entry.versions[0].availability.end||'') + '|';
			s += (entry.subtitle||'') + '|';
			s += '1|';
			s += (entry.parent_position||1) + '|';
			s += versions +'|';
			s += sdk.iso8601durationToSeconds(duration) + '|';
			s += (entry.synopses.small ? entry.synopses.small : (entry.synopses.medium ?
				entry.synopses.medium : (entry.synopses.large ? entry.synopses.large : ''))) + '|';
			s += entry.master_brand.id + '|';
			s += entry.categories.join(',')+'|';
			s += entry.images.standard + '|';
			s += new Date().toISOString() + '|';
			s += entry.guidance + '|';
			s += 'http://bbc.co.uk/programmes/'+entry.id;

			console.log(s);

		}
	}
}

function getProgrammes(chid,page) {
	var query = sdk.newQuery();
   	query.add(ibl.commonLangEn);
   	query.add(ibl.commonPage,page);
	query.add(ibl.commonInitialChildCount,4);
	console.error(chid+' page: '+page);
   	sdk.make_request(ibl.host,ibl.getChannelsProgrammes(chid),ibl_key,query,{},function(obj){
		programmes = programmes.concat(obj.channel_programmes.elements);
{'fraction_non_alphanumeric': 0.1961655365910685, 'fraction_numerical': 0.00794949731119944, 'mean_word_length': 3.9918319719953326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2161383', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1530, 'n_words': 260}