<commit_before>const os = require("os");
const fs = require("fs");
const path = require("path");
const GitHubApi = require("github");
const restify = require("restify");

const languages = fs.readFileSync("languages.txt").toString().split("\n");
const swears = fs.readFileSync("swears.txt").toString().split("\n");
const credentials = require("./credentials.json");
let contentQueue = require("./contentQueue.json");
let github = new GitHubApi();
github.authenticate({
	type: "basic",
	username: credentials.username,
	password: <PASSWORD>
});
const memoryTimeout = 4000; // ms to wait for memory to be freed
const retryTimeout = 2000;

let memoryAvailable = () => os.freemem() <= os.totalmem() * .80;

let swearSearch = new RegExp(swears.join("|"), "gi");
let server = restify.createStringClient({url: "http://localhost:8080"});

server.get(`/setRemaining/${contentQueue.length}`, () => {});

function readFile(file) {
	if (!memoryAvailable()) {
		setTimeout(readFile, memoryTimeout, file);
		return;
	}
	github.repos.getContent({
		owner: file.owner,
		repo: file.repo,
		path: file.path
	})
	.then(result => {
		const content = new Buffer(result.data.content, "base64").toString("utf8");
		let numSwears = content.match(swearSearch).length;
		server.get(`/report/${file.language}/${numSwears}`, () => {});
		if (!contentQueue.length) {
			console.log("DONE");
			return;
		}
		else {
			let index = Math.floor(Math.random() * contentQueue.length);
			console.log(`Reading index ${index}`);
			readFile(contentQueue[index]);
			contentQueue.splice(index, 1);
		}
	})
	.catch(err => {<commit_msg>Quick fix to allow C# to work<commit_after>const os = require("os");
const fs = require("fs");
const path = require("path");
const GitHubApi = require("github");
const restify = require("restify");

const languages = fs.readFileSync("languages.txt").toString().split("\n");
const swears = fs.readFileSync("swears.txt").toString().split("\n");
const credentials = require("./credentials.json");
let contentQueue = require("./contentQueue.json");
let github = new GitHubApi();
github.authenticate({
	type: "basic",
	username: credentials.username,
	password: <PASSWORD>
});
const memoryTimeout = 4000; // ms to wait for memory to be freed
const retryTimeout = 2000;

let memoryAvailable = () => os.freemem() <= os.totalmem() * .80;

let swearSearch = new RegExp(swears.join("|"), "gi");
let server = restify.createStringClient({url: "http://localhost:8080"});

server.get(`/setRemaining/${contentQueue.length}`, () => {});

function readFile(file) {
	if (!memoryAvailable()) {
		setTimeout(readFile, memoryTimeout, file);
		return;
	}
	github.repos.getContent({
		owner: file.owner,
		repo: file.repo,
		path: file.path
	})
	.then(result => {
		const content = new Buffer(result.data.content, "base64").toString("utf8");
		let numSwears = content.match(swearSearch).length;
		if (file.language == "C#") file.language = "C%23";
		server.get(`/report/${file.language}/${numSwears}`, () => {});
		if (!contentQueue.length) {
			console.log("DONE");
			return;
		}
		else {
			let index = Math.floor(Math.random() * contentQueue.length);
			console.log(`Reading index ${index}`);
			readFile(contentQueue[index]);
			contentQueue.splice(index, 1);
		}
	})
	.catch(err => {
{'fraction_non_alphanumeric': 0.18970272755133313, 'fraction_numerical': 0.011645724793135151, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4970122', 'n_tokens_mistral': 1133, 'n_tokens_neox': 1046, 'n_words': 227}