<commit_before>/*jslint indent: 4, node: true */
/*global require: false, process: false */

var http = require('http'),
    path = require('path'),
    fs = require('fs'),
    url = require('url'),
    htmlparser = require('htmlparser2'), // npm install htmlparser2 -g
    async = require('async'), //npm install async -g
    downloadFile = function (uri, filename, callback) {
        "use strict";

        var downloadError = function (err) {
                callback(err.message, {'url': uri});
            },
            downloadStart = function (res) {

                // Don't create the file until we get the response.
                var file = fs.createWriteStream(filename);

                res.on('data', function (chunk) {

                    file.write(chunk, 'binary');
                });
                res.on('end', function () {

                    file.end();
                    callback(null, { 'url': uri, 'filename': filename });
                });
            },
            directoryReady = function (err) {
                var req = {};

                if (err && err.code !== 'EEXIST') {
                    callback(err.code, {'url': uri});
                } else {
                    req = http.get(uri, downloadStart);
                    req.on('error', downloadError);
                }
            },
            directory = path.dirname(filename);

        fs.mkdir(directory, directoryReady);
    },
    getHTML = function (uri, callback) {
        "use strict";

        var req = http.get(uri, function (res) {

            var output = '';

            res.setEncoding('utf8');

            res.on('data', function (chunk) {

                output += chunk;
            });

            res.on('end', function () {

                callback(null, { 'status': res.statusCode, 'text': output });
            });
        });
        req.on('error', function (err) {
            callback(err.message, null);
        });
    },
    parseHtml = function (sourceUrl, callback) {
        "use strict";

        var results = [];

        getHTML(sourceUrl, function (err, response) {

            var whenOpenTag = function (name, attribs) {
                    if (name === 'a') {
                        if (attribs.href) {
                            results.push(url.resolve(sourceUrl, attribs.href));
                        }
                    } else if (name === 'img') {
                        if (attribs.src) {
                            results.push(url.resolve(sourceUrl, attribs.src));
                        }
                    } else if (name === 'iframe') {
                        if (attribs.src) {
                            results.push(url.resolve(sourceUrl, attribs.src));
                        }
                    }
                },
                parser = new htmlparser.Parser({
                    onopentag: whenOpenTag,
                });

            if (err) {
                callback(err, { 'sourceUrl': sourceUrl });
            } else if (response.status === 200) {
                parser.write(response.text);
                parser.end();
                callback(null, { 'sourceUrl': sourceUrl, 'urlsFound': results });
            } else {
                callback(response.status, { 'sourceUrl': sourceUrl });
            }
        });
    },<commit_msg>Added two additional places to find urls when parsing.<commit_after>/*jslint indent: 4, node: true */
/*global require: false, process: false */

var http = require('http'),
    path = require('path'),
    fs = require('fs'),
    url = require('url'),
    htmlparser = require('htmlparser2'), // npm install htmlparser2 -g
    async = require('async'), //npm install async -g
    downloadFile = function (uri, filename, callback) {
        "use strict";

        var downloadError = function (err) {
                callback(err.message, {'url': uri});
            },
            downloadStart = function (res) {

                // Don't create the file until we get the response.
                var file = fs.createWriteStream(filename);

                res.on('data', function (chunk) {

                    file.write(chunk, 'binary');
                });
                res.on('end', function () {

                    file.end();
                    callback(null, { 'url': uri, 'filename': filename });
                });
            },
            directoryReady = function (err) {
                var req = {};

                if (err && err.code !== 'EEXIST') {
                    callback(err.code, {'url': uri});
                } else {
                    req = http.get(uri, downloadStart);
                    req.on('error', downloadError);
                }
            },
            directory = path.dirname(filename);

        fs.mkdir(directory, directoryReady);
    },
    getHTML = function (uri, callback) {
        "use strict";

        var req = http.get(uri, function (res) {

            var output = '';

            res.setEncoding('utf8');

            res.on('data', function (chunk) {

                output += chunk;
            });

            res.on('end', function () {

                callback(null, { 'status': res.statusCode, 'text': output });
            });
        });
        req.on('error', function (err) {
            callback(err.message, null);
        });
    },
    parseHtml = function (sourceUrl, callback) {
        "use strict";

        var results = [];

        getHTML(sourceUrl, function (err, response) {

            var whenOpenTag = function (name, attribs) {
                    if (name === 'a') {
                        if (attribs.href) {
                            results.push(url.resolve(sourceUrl, attribs.href));
                        }
                    } else if (name === 'img') {
                        if (attribs.src) {
                            results.push(url.resolve(sourceUrl, attribs.src));
                        }
                        if (attribs["data-src"]) {
                            results.push(url.resolve(sourceUrl, attribs["data-src"]));
                        }
                    } else if (name === 'iframe') {
                        if (attribs.src) {
                            results.push(url.resolve(sourceUrl, attribs.src));
                        }
                    } else if (name === 'meta') {
                        if (attribs.content) {
                            results.push(url.resolve(sourceUrl, attribs.content));
                        }
                    }
                },
                parser = new htmlparser.Parser({
                    onopentag: whenOpenTag,
                });

            if (err) {
                callback(err, { 'sourceUrl': sourceUrl });
            } else if (response.status === 200) {
                parser.write(response.text);
                parser.end();
                callback(null, { 'sourceUrl': sourceUrl, 'urlsFound': results });
            } else {
                callback(response.status, { 'sourceUrl': sourceUrl });
            }
        });
    },
{'fraction_non_alphanumeric': 0.13359583743495992, 'fraction_numerical': 0.0019687807621994093, 'mean_word_length': 1.2204183577895722, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2163669', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1720, 'n_words': 402}