<commit_before>#!/usr/bin/env phantomjs

var page = require('webpage').create(),
    system = require('system'),
    fs = require('fs'),
    setCookie = require('./setcookie'),
    logger = require('./logger').setMethod(system.stderr.writeLine),
    log = logger(),
    url = require('./url'),

    customHeaders,

    GET_UNMODIFIED = false,
    WAIT = 0,

    blacklist = {
        set: function(str) {
            // try to parse the string as JSON and assign the
            // blacklist key to the hosts array
            try {
                this.hosts = JSON.parse(str).blacklist;

                log('DEBUG', 'Blacklist loaded with ' + this.hosts.length +
                                ' host' + (this.hosts.length == 1 ? '' : 's'));
            } catch (ex) {
                log('WARN', 'Unable to parse the blacklist: ' + ex);
            }
        },
        has: function(host) {
            if (!this.hosts || !this.hosts.length)
                return false;

            return this.hosts.some(function(h) {
                var re = new RegExp(h.replace(/([\-\.])/g, '\\$1') + '$', 'i');

                return re.test(host);
            });
        }
    },

    response = {
        setCookie: function(name, data) {
            log('DEBUG', 'Setting cookie ' + name);
            this.cookies = this.cookies || {};

            if (this.cookies[name]) {
                this.cookies[name].value = data.value;
            } else {
                this.cookies[name] = data;
            }

            this.checkReady();
        },
        setHeader: function(name, value) {
            log('DEBUG', 'Setting HTTP header ' + name);
            this.headers = this.headers || {};

            this.headers[name] = value;

            this.checkReady();
        },
        setBody: function(value) {
            log('DEBUG', 'Setting body content');<commit_msg>Add getHeader function; make setHeader case-insensitive<commit_after>#!/usr/bin/env phantomjs

var page = require('webpage').create(),
    system = require('system'),
    fs = require('fs'),
    setCookie = require('./setcookie'),
    logger = require('./logger').setMethod(system.stderr.writeLine),
    log = logger(),
    url = require('./url'),

    customHeaders,

    GET_UNMODIFIED = false,
    WAIT = 0,

    blacklist = {
        set: function(str) {
            // try to parse the string as JSON and assign the
            // blacklist key to the hosts array
            try {
                this.hosts = JSON.parse(str).blacklist;

                log('DEBUG', 'Blacklist loaded with ' + this.hosts.length +
                                ' host' + (this.hosts.length == 1 ? '' : 's'));
            } catch (ex) {
                log('WARN', 'Unable to parse the blacklist: ' + ex);
            }
        },
        has: function(host) {
            if (!this.hosts || !this.hosts.length)
                return false;

            return this.hosts.some(function(h) {
                var re = new RegExp(h.replace(/([\-\.])/g, '\\$1') + '$', 'i');

                return re.test(host);
            });
        }
    },

    response = {
        setCookie: function(name, data) {
            log('DEBUG', 'Setting cookie ' + name);
            this.cookies = this.cookies || {};

            if (this.cookies[name]) {
                this.cookies[name].value = data.value;
            } else {
                this.cookies[name] = data;
            }

            this.checkReady();
        },
        setHeader: function(name, value) {
            log('DEBUG', 'Setting HTTP header ' + name);
            this.headers = this.headers || {};

            var key = this.getHeader(name).name || name;

            this.headers[key] = value;

            this.checkReady();
        },
        getHeader: function(name) {
            for (var h in this.headers) {
                if (name.toLowerCase() == h.toLowerCase())
                    return {
                        name: h,
                        value: this.headers[h]
                    };
            }

            return {};
        },
        setBody: function(value) {
            log('DEBUG', 'Setting body content');
{'fraction_non_alphanumeric': 0.14668581000239292, 'fraction_numerical': 0.0014357501794687725, 'mean_word_length': 1.4401634559252774, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5684417', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1117, 'n_words': 256}