<commit_before>#!/usr/bin/env node
'use strict';

var fs = require('fs');
var path = require('path');
var request = require('request');
var iconv = require('iconv-lite');

var POSTAL_CODE_URL = 'http://www.bring.no/hele-bring/forside/_attachment/159761';
var input = process.argv[2];

function convert(data) {
	var dir = 'converted';
	var ret = {};
	var ret2 = {};
	var csv = iconv.decode(data, 'latin1').trim();

	csv.split('\r\n').forEach(function(row) {
		var columns = row.split('\t');
		ret[columns[0]] = columns.slice(1);
		ret2[columns[0]] = columns[1];
	});

	fs.writeFileSync(path.join(dir, 'postal-codes.tsv'), csv.replace(/\r\n/g, '\n'), 'utf8');
	fs.writeFileSync(path.join(dir, 'postal-codes.csv'), csv.replace(/\r\n/g, '\n').replace(/\t/g, ','), 'utf8');
	fs.writeFileSync(path.join(dir, 'postal-codes.json'), JSON.stringify(ret), 'utf8');
	fs.writeFileSync(path.join(dir, 'postal-codes-simple.json'), JSON.stringify(ret2), 'utf8');
	fs.writeFileSync(path.join(dir, 'postal-codes-simple.js'), 'module.exports=' + JSON.stringify(ret2), 'utf8');
}

if (input) {
	convert(fs.readFileSync(input));
} else {
	request(POSTAL_CODE_URL, {encoding: null}, function(err, res, body) {
		if (err || res.statusCode !== 200) {
			return console.log(err);
		}
		convert(body);
	});
}
<commit_msg>Simplify the output write<commit_after>#!/usr/bin/env node
'use strict';

var fs = require('fs');
var path = require('path');
var request = require('request');
var iconv = require('iconv-lite');

var POSTAL_CODE_URL = 'http://www.bring.no/hele-bring/forside/_attachment/159761';
var OUTPUT_DIR = 'converted';
var input = process.argv[2];


function write(filename, data) {
	fs.writeFileSync(path.join(OUTPUT_DIR, filename), data, 'utf8');
}

function convert(data) {
	var ret = {};
	var ret2 = {};
	var csv = iconv.decode(data, 'latin1').trim();

	csv.split('\r\n').forEach(function(row) {
		var columns = row.split('\t');
		ret[columns[0]] = columns.slice(1);
		ret2[columns[0]] = columns[1];
	});

	write('postal-codes.tsv', csv.replace(/\r\n/g, '\n'));
	write('postal-codes.csv', csv.replace(/\r\n/g, '\n').replace(/\t/g, ','));
	write('postal-codes.json', JSON.stringify(ret));
	write('postal-codes-simple.json', JSON.stringify(ret2));
	write('postal-codes-simple.js', 'module.exports=' + JSON.stringify(ret2));
}

if (input) {
	convert(fs.readFileSync(input));
} else {
	request(POSTAL_CODE_URL, {encoding: null}, function(err, res, body) {
		if (err || res.statusCode !== 200) {
			return console.log(err);
		}
		convert(body);
	});
}

{'fraction_non_alphanumeric': 0.2220031545741325, 'fraction_numerical': 0.017350157728706624, 'mean_word_length': 4.886310904872389, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753243', 'n_tokens_mistral': 1014, 'n_tokens_neox': 959, 'n_words': 166}