<commit_before>"use strict";
var request = require("request");
var S3 = require('aws-sdk/clients/s3');
var s3 = new S3({
    apiVersion: '2006-03-01',
    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY    
});

let getUrl = (key) => `https://s3.amazonaws.com/${process.env.AWS_BUCKET_NAME}/${key}`;

let immutable = (key, url) => {
    var bucket = process.env.AWS_BUCKET_NAME;
    var cachedUrl = getUrl(key);
    return new Promise((resolve, reject) => {
        s3.headObject({
            Bucket: bucket,
            Key: key,
        }, function(err, data) {
            if (data) {
                console.log(`s3: Found ${key}.`)
                console.log(`s3: Available at ${cachedUrl}.`)
                resolve(cachedUrl);
            } else {
                console.log(`request: Fetch ${url}.`)
                request(url, function (error, response, body) {
                    if (error) {
                        reject(error);
                    } else if (response.statusCode != 200) {
                        var err = new Error(`${ response.statusCode } ${ response.statusMessage }`);
                        try { err.data = JSON.parse(body); } catch (e) {}
                        reject(err);
                    } else {
                        console.log(`request: Found ${url}.`)
                        console.log(`s3: Attempting to cache ${url} in ${key}.`)
                        s3.putObject({
                            Bucket:       bucket,
                            Key:          key,
                            Body:         body,
                            ContentType:  "text/html;charset=utf-8",
                            CacheControl: "max-age=315569000, immutable"
                        }, function(err, data) {
                            if (data) {
                                console.log(`s3: Succesfully cached ${url} in ${key}.`)
                                console.log(`s3: Available at ${cachedUrl}.`)
                                resolve(cachedUrl);
                            } else {
                                console.log(`s3: Failed to cache ${url} in ${key}.`)
                                reject(err);
                            }
                        });
                    }
                });
            }          
        });
    });
};

let mutable = (key, url) => {
    var bucket = process.env.AWS_BUCKET_NAME;
    var cachedUrl = getUrl(key);
    console.log(`request: Fetch ${url}.`)
    return new Promise((resolve, reject) => {
        request(url, function (error, response, body) {
            if (error) {
                reject(error);
            } else if (response.statusCode != 200) {
                reject(new Error(`${ response.statusCode } ${ response.statusMessage }`));
            } else {
                console.log(`request: Found ${url}.`)
                console.log(`s3: Attempting to cache ${url} in ${key}.`)
                s3.putObject({
                    Bucket:       bucket,
                    Key:          key,
                    Body:         body,
                    ContentType:  "text/html;charset=utf-8"
                }, function(err, data) {
                    if (data) {<commit_msg>Improve error logging further.<commit_after>"use strict";
var request = require("request");
var S3 = require('aws-sdk/clients/s3');
var s3 = new S3({
    apiVersion: '2006-03-01',
    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY    
});

let getUrl = (key) => `https://s3.amazonaws.com/${process.env.AWS_BUCKET_NAME}/${key}`;

let immutable = (key, url) => {
    var bucket = process.env.AWS_BUCKET_NAME;
    var cachedUrl = getUrl(key);
    return new Promise((resolve, reject) => {
        s3.headObject({
            Bucket: bucket,
            Key: key,
        }, function(err, data) {
            if (data) {
                console.log(`s3: Found ${key}.`)
                console.log(`s3: Available at ${cachedUrl}.`)
                resolve(cachedUrl);
            } else {
                console.log(`request: Fetch ${url}.`)
                request(url, function (error, response, body) {
                    if (error) {
                        error.data = { request_url: url };
                        reject(error);
                    } else if (response.statusCode != 200) {
                        var err = new Error(`${ response.statusCode } ${ response.statusMessage }`);
                        try { err.data = JSON.parse(body); } catch (e) { err.data = {}; }
                        error.data.request_url = url;
                        reject(err);
                    } else {
                        console.log(`request: Found ${url}.`)
                        console.log(`s3: Attempting to cache ${url} in ${key}.`)
                        s3.putObject({
                            Bucket:       bucket,
                            Key:          key,
                            Body:         body,
                            ContentType:  "text/html;charset=utf-8",
                            CacheControl: "max-age=315569000, immutable"
                        }, function(err, data) {
                            if (data) {
                                console.log(`s3: Succesfully cached ${url} in ${key}.`)
                                console.log(`s3: Available at ${cachedUrl}.`)
                                resolve(cachedUrl);
                            } else {
                                console.log(`s3: Failed to cache ${url} in ${key}.`)
                                reject(err);
                            }
                        });
                    }
                });
            }          
        });
    });
};

let mutable = (key, url) => {
    var bucket = process.env.AWS_BUCKET_NAME;
    var cachedUrl = getUrl(key);
    console.log(`request: Fetch ${url}.`)
    return new Promise((resolve, reject) => {
        request(url, function (error, response, body) {
            if (error) {
                error.data = { request_url: url };
                reject(error);
            } else if (response.statusCode != 200) {
                var err = new Error(`${ response.statusCode } ${ response.statusMessage }`);
                try { err.data = JSON.parse(body); } catch (e) { err.data = {}; }
                error.data.request_url = url;
                reject(err);
            } else {
                console.log(`request: Found ${url}.`)
                console.log(`s3: Attempting to cache ${url} in ${key}.`)
                s3.putObject({
                    Bucket:       bucket,
                    Key:          key,
                    Body:         body,
                    ContentType:  "text/html;charset=utf-8"
                }, function(err, data) {
                    if (data) {
{'fraction_non_alphanumeric': 0.12766878245816504, 'fraction_numerical': 0.01154068090017311, 'mean_word_length': 1.165209244222361, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3248912', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1777, 'n_words': 393}