<commit_before>var http = require('http');
var https = require('https');
var path = require('path');
var fs = require('fs');
var gitHubWebhookHandler = require('github-webhook-handler');
var Sftp = require('./sftp');
var settings = {};

settings.hooks = [{name: '/gitHubPuller',
    localPath: '/var/www/gitHubPuller',
    sftpPath: '/home/nick',
    sftp: {
        username: 'nick',
        password: '<PASSWORD>',
        host: '192.168.0.12',
        }
    }];

for (var i = 0; i < settings.hooks.length; i++) {
    var localPath = settings.hooks[i].localPath, 
        handler = gitHubWebhookHandler({ path: settings.hooks[i].name, secret: 'lorcanvida' });
    handler.on('push', function (event) {
        var added = event.payload.head_commit.added,
            removed = event.payload.head_commit.removed,
            modified = event.payload.head_commit.modified,
            //remotePath = event.payload.head_commit.url,
            remotePath =  'https://raw.githubusercontent.com/' + event.payload.repository.full_name + '/master',
            sftpClient = new Sftp.Client(settings.sftp),
            j;
        sftpClient.on('error', function(err) {
            errorHandler(err, 'sftpCluent');
        });
        for (j = 0; j < modified.length; j++) {
            var mod = modified[j], body = '';
           // remotePath = 'https://raw.githubusercontent.com/seethespark/gitHubPuller/master';
            https.get(remotePath + '/' + mod, function(res) {
               // console.log(res.body);
               res.on('data', function(chunk) { body += chunk; });
               res.on('end', function() {
               //console.log(body);
                    fs.writeFile(path.join(localPath, mod), body, function(err) {
                        if (err) { errorHandler(err, 'push2'); return; }
                        /// for testing this is inside the local write
                        sftpClient.write ({
                            content: new Buffer(body),
                            destination: path.join(sftpPath, mod)
                        }, function() {
                            if (err) { errorHandler(err, 'push2'); return; }
                        });
                    });<commit_msg>missing object ref<commit_after>var http = require('http');
var https = require('https');
var path = require('path');
var fs = require('fs');
var gitHubWebhookHandler = require('github-webhook-handler');
var Sftp = require('./sftp');
var settings = {};

settings.hooks = [{name: '/gitHubPuller',
    localPath: '/var/www/gitHubPuller',
    sftpPath: '/home/nick',
    sftp: {
        username: 'nick',
        password: '<PASSWORD>',
        host: '192.168.0.12',
        }
    }];

for (var i = 0; i < settings.hooks.length; i++) {
    var localPath = settings.hooks[i].localPath, 
        handler = gitHubWebhookHandler({ path: settings.hooks[i].name, secret: 'lorcanvida' });
    handler.on('push', function (event) {
        var added = event.payload.head_commit.added,
            removed = event.payload.head_commit.removed,
            modified = event.payload.head_commit.modified,
            //remotePath = event.payload.head_commit.url,
            remotePath =  'https://raw.githubusercontent.com/' + event.payload.repository.full_name + '/master',
            sftpClient = new Sftp.Client(settings.hooks[i].sftp),
            j;
        sftpClient.on('error', function(err) {
            errorHandler(err, 'sftpCluent');
        });
        for (j = 0; j < modified.length; j++) {
            var mod = modified[j], body = '';
           // remotePath = 'https://raw.githubusercontent.com/seethespark/gitHubPuller/master';
            https.get(remotePath + '/' + mod, function(res) {
               // console.log(res.body);
               res.on('data', function(chunk) { body += chunk; });
               res.on('end', function() {
               //console.log(body);
                    fs.writeFile(path.join(localPath, mod), body, function(err) {
                        if (err) { errorHandler(err, 'push2'); return; }
                        /// for testing this is inside the local write
                        sftpClient.write ({
                            content: new Buffer(body),
                            destination: path.join(settings.hooks[i].sftpPath, mod)
                        }, function() {
                            if (err) { errorHandler(err, 'push2'); return; }
                        });
                    });
{'fraction_non_alphanumeric': 0.14988864142538977, 'fraction_numerical': 0.005790645879732739, 'mean_word_length': 1.8641581632653061, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7398249', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1258, 'n_words': 256}