<commit_before>"use strict";
const request = require('request');
const urljoin = require('urljoin');
const _ = require('lodash');

// TODO disable logging of text/event-stream

const ignoreResponseHeaders = ['set-cookie', 'content-length'];
const ignoreHeaders = ['host', 'cookie'];
const heartbeat = ':heartbeat signal';

function makeRequestOptions(req, url, serverPort) {
	const result = Object.assign(
		{ url, headers: {} },
		_.pick(req, 'method', 'query', 'timeout', 'maxRedirects')
	);

	_.each(req.headers, (val, key) => {
		if (_.contains(ignoreHeaders, key) === false) {
			result.headers[key] = val;
		}
	});

	const port = `${(req.secure ? 443 : serverPort)}`;

	if (req.hostname) {
		result.headers['x-forwarded-host'] = `${req.hostname}:${port}`;
	}
	if (req.ip) {
		result.headers['x-forwarded-for'] = req.ip;
	}

	result.headers['x-forwarded-proto'] = req.secure ? 'https' : 'http';
	result.headers['x-forwarded-port'] = port;

	return result;
}

module.exports = function makeProxy(options) {
	let base = options.base;
	if (!base && options.port) {
		base = `http://localhost:${options.port}`;
	}
	return (req, res) => {
		const url = urljoin(base, req.url);
		// console.log('request:', JSON.stringify(
		// 	_.pick(req, 'hostname', 'port', 'method', 'url', 'headers'),
		// 	null, 2
		// ));
		const requestOptions = makeRequestOptions(req, url, options.serverPort);
		requestOptions.host = req.host;
		const apiRequest = req.pipe(request(requestOptions));

		apiRequest.on('error', err => {
			console.log('error: ', err);
			res.end();
		});

		apiRequest.on('end', () => {
			res.end();
		});

		function streamData() {
			apiRequest.on('data', data => {
				const s = data.toString().trim();
				if (s) {
					if (s.indexOf(heartbeat) < 0) {
						console.log('%s', s);
					}
					res.write(`${s}\n\n`);
				} else {
					res.write(`${heartbeat}\n\n`);
				}
				res.flush();
			});
		}

		apiRequest.on('response', response => {
			const headers = response.headers;
			// console.log('response headers:', JSON.stringify(headers, null, 2));
			_.each(headers, (val, key) => {
				if (_.contains(ignoreResponseHeaders, key) === false) {
					res.set(key, val);
				}
			});
			const isEventStream = headers['content-type'] === 'text/event-stream';
			if (isEventStream) {
				streamData();
			} else {
				apiRequest.pipe(res);
			}
		});
	};
};
<commit_msg>lint code<commit_after>'use strict'; // eslint-disable-line
const request = require('request');
const urljoin = require('urljoin');
const _ = require('lodash');

// TODO disable logging of text/event-stream

const ignoreResponseHeaders = ['set-cookie', 'content-length'];
const ignoreHeaders = ['host', 'cookie'];
const heartbeat = ':heartbeat signal';

function makeRequestOptions(req, url, serverPort) {
	const result = Object.assign(
		{ url, headers: {} },
		_.pick(req, 'method', 'query', 'timeout', 'maxRedirects')
	);

	_.each(req.headers, (val, key) => {
		if (_.contains(ignoreHeaders, key) === false) {
			result.headers[key] = val;
		}
	});

	const port = `${(req.secure ? 443 : serverPort)}`;

	if (req.hostname) {
		result.headers['x-forwarded-host'] = `${req.hostname}:${port}`;
	}
	if (req.ip) {
		result.headers['x-forwarded-for'] = req.ip;
	}

	result.headers['x-forwarded-proto'] = req.secure ? 'https' : 'http';
	result.headers['x-forwarded-port'] = port;

	return result;
}

module.exports = function makeProxy(options) {
	let base = options.base;
	if (!base && options.port) {
		base = `http://localhost:${options.port}`;
	}
	return (req, res) => {
		const url = urljoin(base, req.url);
		// console.log('request:', JSON.stringify(
		// 	_.pick(req, 'hostname', 'port', 'method', 'url', 'headers'),
		// 	null, 2
		// ));
		const requestOptions = makeRequestOptions(req, url, options.serverPort);
		requestOptions.host = req.host;
		const apiRequest = req.pipe(request(requestOptions));

		apiRequest.on('error', err => {
			console.log('error: ', err);
			res.end();
		});

		apiRequest.on('end', () => {
			res.end();
		});

		function streamData() {
			apiRequest.on('data', data => {
				const s = data.toString().trim();
				if (s) {
					if (s.indexOf(heartbeat) < 0) {
						console.log('%s', s);
					}
					res.write(`${s}\n\n`);
				} else {
					res.write(`${heartbeat}\n\n`);
				}
				res.flush();
			});
		}

		apiRequest.on('response', response => {
			const headers = response.headers;
			// console.log('response headers:', JSON.stringify(headers, null, 2));
			_.each(headers, (val, key) => {
				if (_.contains(ignoreResponseHeaders, key) === false) {
					res.set(key, val);
				}
			});
			const isEventStream = headers['content-type'] === 'text/event-stream';
			if (isEventStream) {
				streamData();
			} else {
				apiRequest.pipe(res);
			}
		});
	};
};

{'fraction_non_alphanumeric': 0.20489232699142798, 'fraction_numerical': 0.0025088856366297305, 'mean_word_length': 3.5303030303030303, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6655625', 'n_tokens_mistral': 1875, 'n_tokens_neox': 1650, 'n_words': 326}