<commit_before>// Dependencies
var fs = require('fs');
var url = require('url');
var http = require('http');
var jsdom = require("jsdom");
var exec = require('child_process').exec;

var thread = process.argv[2];
var host = url.parse(thread);
var folderName = host.pathname.split('/').pop();
var mkdir = 'mkdir -p ' + 'downloads/' + folderName;
var chmod = 'chmod -R 777 downloads/' + folderName;

var mkdirExec = exec(mkdir, function (err, stdout, stderr) {
    if (err) {
        throw err;
    } else {
        console.log('folder created');
    }
});
var chmodExec = exec(chmod, function (err, stdout, stderr) {
    if (err) {
        throw err;
    }
});

var links = [];

jsdom.env(
    thread,
    ["http://code.jquery.com/jquery.js"],
    function (err, window) {
        images = window.$(".postContainer .file .fileThumb");
        for (i = 0; i < images.length; i++) {
            links.push(window.$(images[i]).attr('href'));
        }
       for (i = 0; i < links.length;i++) {
           href = 'http://'+links[i].replace(/\/\//, '');
           splitted = href.split("/");
           filename = splitted[splitted.length - 1];

           var request = http.get(href, function(res){
               var imagedata = '';
               res.setEncoding('binary');

               res.on('data', function(chunk){
                   imagedata += chunk
               });

               res.on('end', function(){
                   fs.writeFile(filename, imagedata, 'binary', function(err){
                       if (err) {
                           throw err
                       }
                       console.log(filename + ' saved.')
                   });
               })
           })
       }
    }<commit_msg>use http-request because...<commit_after>// Dependencies
var url = require('url');
var http = require('http-request');
var jsdom = require("jsdom");
var exec = require('child_process').exec;

var thread = process.argv[2];
var host = url.parse(thread);
var folderName = host.pathname.split('/').pop();
var mkdir = 'mkdir -p ' + 'downloads/' + folderName;
var chmod = 'chmod -R 777 downloads/' + folderName;

var mkdirExec = exec(mkdir, function (err, stdout, stderr) {
    if (err) {
        throw err;
    } else {
        console.log('folder created');
    }
});
var chmodExec = exec(chmod, function (err, stdout, stderr) {
    if (err) {
        throw err;
    }
});

var links = [];

jsdom.env(
    thread,
    ["http://code.jquery.com/jquery.js"],
    function (err, window) {
        images = window.$(".postContainer .file .fileThumb");
        for (i = 0; i < images.length; i++) {
            links.push(window.$(images[i]).attr('href'));
        }
       for (i = 0; i < links.length;i++) {
           href = 'http://'+links[i].replace(/\/\//, '');
           splitted = href.split("/");
           filename = splitted[splitted.length - 1];
           fullPath = 'downloads/' + folderName + '/' + filename;

           options = {url: href};

           http.get(options, fullPath, function (error, result) {
               if (error) {
                   console.error(error);
               } else {
                   console.log('File downloaded at: ' + result.file);
               }
           });
       }
    }
{'fraction_non_alphanumeric': 0.17091132249156182, 'fraction_numerical': 0.004295796256520405, 'mean_word_length': 1.86215978928885, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '990065', 'n_tokens_mistral': 1005, 'n_tokens_neox': 967, 'n_words': 232}