<commit_before>var request = require('superagent'),
	mkdirp = require('mkdirp'),
	fs = require('fs'),
	path = require('path'),
	async = require('async'),
	knox = require('knox'),
	hashFile = require('hash_file'),
	crypto = require('crypto');

var Page = require('./Page'),
	Site = require('./Site'),
	File = require('./File'),
	fsUtil = require('./fs-util'),
	listBucket = require('./listBucket');

Site.prototype.syncS3 = function(callback) {
	if (!this.settings.s3key || !this.settings.s3secret || !this.settings.s3bucket)
		return new Error('S3 settings not found.');
	var site = this,
		rootUrl = this.root.url,
		s3List = {};

	var client = knox.createClient({
		key: this.settings.s3key,
		secret: this.settings.s3secret,
		bucket: this.settings.s3bucket,
		region: this.settings.s3region || 'us-standard',
		// We have to set secure to false, when the bucket name has .'s in it
		// because Amazon doesn't like it
		secure: this.settings.s3bucket.indexOf('.') == -1
	});

	var util = {
		escapeRegex: function(str) {
			return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
		},

		getPageHtml: function(page, callback) {
			request
				.get(page.href())
				.buffer()
				.end(function(err, res) {
					if (err) return callback(err);
					return callback(null, res.text);
				});
		},<commit_msg>Improve s3 syncing warning.<commit_after>var request = require('superagent'),
	mkdirp = require('mkdirp'),
	fs = require('fs'),
	path = require('path'),
	async = require('async'),
	knox = require('knox'),
	hashFile = require('hash_file'),
	crypto = require('crypto');

var Page = require('./Page'),
	Site = require('./Site'),
	File = require('./File'),
	fsUtil = require('./fs-util'),
	listBucket = require('./listBucket');

Site.prototype.syncS3 = function(callback) {
	if (!this.settings.s3key || !this.settings.s3secret || !this.settings.s3bucket)
		return new Error('S3 configuration not found. Missing one or more of s3key, s3secret and s3bucket settings in site-directory/settings.md');
	var site = this,
		rootUrl = this.root.url,
		s3List = {};

	var client = knox.createClient({
		key: this.settings.s3key,
		secret: this.settings.s3secret,
		bucket: this.settings.s3bucket,
		region: this.settings.s3region || 'us-standard',
		// We have to set secure to false, when the bucket name has .'s in it
		// because Amazon doesn't like it
		secure: this.settings.s3bucket.indexOf('.') == -1
	});

	var util = {
		escapeRegex: function(str) {
			return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
		},

		getPageHtml: function(page, callback) {
			request
				.get(page.href())
				.buffer()
				.end(function(err, res) {
					if (err) return callback(err);
					return callback(null, res.text);
				});
		},
{'fraction_non_alphanumeric': 0.19642857142857142, 'fraction_numerical': 0.01020408163265306, 'mean_word_length': 3.575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2307737', 'n_tokens_mistral': 1013, 'n_tokens_neox': 921, 'n_words': 219}