<commit_before>
const request = require('../lib/request');
const parse = require('csv-parse');
const http = require('http');
const Promise = require('bluebird');
const _ = require('lodash');
const fs = require('fs');
const QueryDimension = require('@bengler/imdi-dataset').QueryDimension;

const groupsUrl = "https://docs.google.com/a/bengler.no/spreadsheets/d/1Wm0yF_Rs6VLW9dS_nZgbJrt2ymXEiaFVKBvhUae6lRs"
const outFile = "./data/groups.json"

const csvSuffix = "/export?format=csv"

const importGroups = function(url) {
	url += csvSuffix;
  return request.get(url).then(response => {
    return new Promise((resolve, reject) => {
      parse(response.body, {comment: '#', columns: true}, (err, output) => {
        if (err) {
          return reject(err);
        }
        resolve(output);
      });
		})
	});
}

function trim(s) {
	return s ? s.trim() : s;
}

const mungeLine = function(line) {
	line = _.omit(line, (e) => e == '');

	if (line.dimensions) {
		line.dimensions = line.dimensions.split(';').map(trim).filter(Boolean).map(QueryDimension.parse);
	}

	if (line.time) {
		const parsedTime = line.time.split(',').map(trim).filter(Boolean);
		if (parsedTime.length === 1 && !(/^\d+$/.test(parsedTime))) {
			line.time = parsedTime[0];
		}
		else {
			line.time = parsedTime;
		}
	}
	return line
}

const parseGroups = function(lines) {
	let currentGroup = {};
	let groups = []
	lines.forEach( (line) => {
		line = mungeLine(line);
		if (line.groupKind !== undefined && currentGroup.kind != line.groupKind) {
			if (currentGroup.groupKind !== undefined) {
				groups.push(currentGroup);
			}
			currentGroup = {
				groupKind: line.groupKind,
				title: line.title,
				items: []
			};
		} else if (Object.keys(line).length > 0 && line.groupKind != "#") { // # in groupKind is skip notation
			currentGroup.items.push(line);
		}
	})
	return groups;
}

console.log("Reading components");
importGroups(groupsUrl).then( (result) => {
	const groupDict = parseGroups(result);
	fs.writeFile(outFile, JSON.stringify(groupDict, null, 2), function(err) {
    if (err) {
      throw err;
    }
    console.log("Done!")
  });
}).catch( (error) => {
	console.log("Could not fetch: " + error.stack)
});
<commit_msg>Supports SOLOing a single chart for debugging ease<commit_after>
const request = require('../lib/request');
const parse = require('csv-parse');
const http = require('http');
const Promise = require('bluebird');
const _ = require('lodash');
const fs = require('fs');
const QueryDimension = require('@bengler/imdi-dataset').QueryDimension;

const groupsUrl = "https://docs.google.com/a/bengler.no/spreadsheets/d/1Wm0yF_Rs6VLW9dS_nZgbJrt2ymXEiaFVKBvhUae6lRs"
const outFile = "./data/groups.json"

const csvSuffix = "/export?format=csv"

// Mute all channels except debug
const SOLO = true;

const importGroups = function(url) {
	url += csvSuffix;
  return request.get(url).then(response => {
    return new Promise((resolve, reject) => {
      parse(response.body, {comment: '#', columns: true}, (err, output) => {
        if (err) {
          return reject(err);
        }
        resolve(output);
      });
		})
	});
}

function trim(s) {
	return s ? s.trim() : s;
}

const mungeLine = function(line) {
	line = _.omit(line, (e) => e == '');

	if (line.dimensions) {
		line.dimensions = line.dimensions.split(';').map(trim).filter(Boolean).map(QueryDimension.parse);
	}

	if (line.time) {
		const parsedTime = line.time.split(',').map(trim).filter(Boolean);
		if (parsedTime.length === 1 && !(/^\d+$/.test(parsedTime))) {
			line.time = parsedTime[0];
		}
		else {
			line.time = parsedTime;
		}
	}
	return line
}

const parseGroups = function(lines) {
	let currentGroup = {};
	let groups = []
	lines.forEach( (line) => {
		line = mungeLine(line);
		if (line.groupKind !== undefined && currentGroup.kind != line.groupKind) {
			if (currentGroup.groupKind !== undefined) {
				groups.push(currentGroup);
			}
			currentGroup = {
				groupKind: line.groupKind,
				title: line.title,
				items: []
			};
		} else if (Object.keys(line).length > 0 && line.groupKind != "#") { // # in groupKind is skip notation

			if (SOLO && line.debug) {
				currentGroup.items.push(line);
			}
		}
	})
	return groups;
}

console.log("Reading components");
importGroups(groupsUrl).then( (result) => {
	const groupDict = parseGroups(result);
	fs.writeFile(outFile, JSON.stringify(groupDict, null, 2), function(err) {
    if (err) {
      throw err;
    }
    console.log("Done!")
  });
}).catch( (error) => {
	console.log("Could not fetch: " + error.stack)
});

{'fraction_non_alphanumeric': 0.17856358444981332, 'fraction_numerical': 0.004392708104546453, 'mean_word_length': 3.665983606557377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1312731', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1567, 'n_words': 312}