<commit_before>#!/usr/bin/node
"use strict";
var fs = require("fs"), zlib = require("zlib"), http = require("https");
function download(gid, writeStream, cb){
	http.get({
		host:"docs.google.com",
		path:"/spreadsheets/d/15G8h9S0Ph7UfzaZMyn35LpJY8gTgmTtru6K2QmdTUmw/export?format=csv&id=15G8h9S0Ph7UfzaZMyn35LpJY8gTgmTtru6K2QmdTUmw&gid="+gid,
		headers: { "GData-Version": "3.0", "Accept-Encoding:gzip" },
	}, function(res){
		if (res.headers["Content-Encoding"] == "gzip"){
			res = res.pipe(zlib.createGunzip());
		}
		res.pipe(writeStream);
		res.on("end", function(){
			cb(null);
		});
	}).on("error", function(err){
		cb(err);
	});
}
var dbgid = [
	["pillar", "0"],
	["weapon", "1863409466"],
	["shield", "457582620"],
	["permanent", "420516648"],
	["spell", "1605384839"],
	["creature", "1045918250"],
	["active", "657211460"],
];
dbgid.forEach(function(pair){
	if (process.argv.length == 2 || process.argv.some(function(x) { return x.indexOf(pair[0]) == 0; })){
		download(pair[1], fs.createWriteStream(pair[0]+".csv"), function(err){
			if (err){
				console.log("Failed to download " + pair[0], err.message);
			}else console.log(pair[0]);
		});
	}
});<commit_msg>Fix headers<commit_after>#!/usr/bin/node
"use strict";
var fs = require("fs"), zlib = require("zlib"), http = require("https");
function download(gid, writeStream, cb){
	http.get({
		host:"docs.google.com",
		path:"/spreadsheets/d/15G8h9S0Ph7UfzaZMyn35LpJY8gTgmTtru6K2QmdTUmw/export?format=csv&id=15G8h9S0Ph7UfzaZMyn35LpJY8gTgmTtru6K2QmdTUmw&gid="+gid,
		headers: { "GData-Version": "3.0", "Accept-Encoding":"gzip" },
	}, function(res){
		if (res.headers["content-encoding"] == "gzip"){
			res = res.pipe(zlib.createGunzip());
		}
		res.pipe(writeStream);
		res.on("end", function(){
			cb(null);
		});
	}).on("error", function(err){
		cb(err);
	});
}
var dbgid = [
	["pillar", "0"],
	["weapon", "1863409466"],
	["shield", "457582620"],
	["permanent", "420516648"],
	["spell", "1605384839"],
	["creature", "1045918250"],
	["active", "657211460"],
];
dbgid.forEach(function(pair){
	if (process.argv.length == 2 || process.argv.some(function(x) { return x.indexOf(pair[0]) == 0; })){
		download(pair[1], fs.createWriteStream(pair[0]+".csv"), function(err){
			if (err){
				console.log("Failed to download " + pair[0], err.message);
			}else console.log(pair[0]);
		});
	}
});
{'fraction_non_alphanumeric': 0.23168654173764908, 'fraction_numerical': 0.075809199318569, 'mean_word_length': 5.635593220338983, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2673596', 'n_tokens_mistral': 1122, 'n_tokens_neox': 979, 'n_words': 141}