<commit_before>var GitHubApi = require("github").GitHubApi
  , lconfig = require('../../Common/node/lconfig')
  , request = require('request')
  , github = new GitHubApi()
  , async = require('async')
  , nfs = require('node-fs')
  , fs = require('fs')
  , lockerUrl
  , auth
  , viewers = []
  ;

lconfig.load('../../Config/config.json');

exports.sync = function(processInfo, cb) {
    auth = processInfo.auth;
    auth.headers = {"Authorization":"token "+auth.accessToken, "Connection":"keep-alive"};
    var cached = {};
    if (processInfo.config && processInfo.config.cached)
        cached = processInfo.config.cached;
    lockerUrl = processInfo.lockerUrl;
    exports.syncRepos(cached, function(err, repos) {
        if (err) console.error(err);
        var responseObj = {data : {}, config: { cached: cached }};
        responseObj.data.repo = repos;
        responseObj.data.view = viewers;
        console.error(viewers);
        cb(err, responseObj);
    });
};

exports.syncRepos = function(cached, callback) {
    github.getRepoApi().getUserRepos(auth.username, function(err, repos) {
        if(err || !repos || !repos.length) return callback(err, []);
        // process each one to get richer data
        async.forEach(repos, function(repo, cb){
            repo.id = getIDFromUrl(repo.url);
            // nothing changed
            var ckey = repo.pushed_at + repo.watchers;
            if(cached[repo.id] == ckey) return cb();
            cached[repo.id] = ckey;
            // get the watchers, is nice
            console.error("checking "+repo.id);
            github.getRepoApi().getRepoWatchers(auth.username, repo.id.substring(repo.id.indexOf('/') + 1), function(err, watchers){
                repo.watchers = watchers;
                // get a snapshot of the full repo, is nice too
                request.get({uri:"https://api.github.com/repos/" + repo.id + "/git/trees/HEAD?recursive=1", headers:auth.headers, json:true}, function(err, resp, tree) {
                    if(err || !tree || !tree.tree) return cb(err);
                    repo.tree = tree.tree;
                    // see if there's any viewers!
                    var viewer = false;
                    for(var i=0; i < repo.tree.length; i++) if(/^\w+\.app$/.test(repo.tree[i].path)) viewer = repo.tree[i].path;
                    if(!viewer) return cb();
                    console.error("found a viewer! "+repo.id);
                    syncRepo(repo, function(err) {
                        // mangle the manifest so it's a unique handle
                        var manifest = repo.id+"/"+viewer;
                        var js;
                        try {
                            js = JSON.parse(fs.readFileSync(manifest));
                            if(!js || js.static != "true") throw new Error("invalid manifest");
                            js.handle = repo.id.replace("/", "-");
                            js.author = auth.username;
                            fs.writeFileSync(manifest, JSON.stringify(js));
                        } catch (err) {
                            // bail, no viewer for you
                            console.error("failed: "+err);
                            return cb();
                        }
                        viewers.push({id:repo.id, manifest:manifest, at:repo.pushed_at, viewer:js.viewer});
                    });
                });
            });<commit_msg>re-added callback for github repos sync<commit_after>var GitHubApi = require("github").GitHubApi
  , lconfig = require('../../Common/node/lconfig')
  , request = require('request')
  , github = new GitHubApi()
  , async = require('async')
  , nfs = require('node-fs')
  , fs = require('fs')
  , lockerUrl
  , auth
  , viewers = []
  ;

lconfig.load('../../Config/config.json');

exports.sync = function(processInfo, cb) {
    auth = processInfo.auth;
    auth.headers = {"Authorization":"token "+auth.accessToken, "Connection":"keep-alive"};
    var cached = {};
    if (processInfo.config && processInfo.config.cached)
        cached = processInfo.config.cached;
    lockerUrl = processInfo.lockerUrl;
    exports.syncRepos(cached, function(err, repos) {
        if (err) console.error(err);
        var responseObj = {data : {}, config: { cached: cached }};
        responseObj.data.repo = repos;
        responseObj.data.view = viewers;
        console.error(viewers);
        cb(err, responseObj);
    });
};

exports.syncRepos = function(cached, callback) {
    github.getRepoApi().getUserRepos(auth.username, function(err, repos) {
        if(err || !repos || !repos.length) return callback(err, []);
        // process each one to get richer data
        async.forEach(repos, function(repo, cb) {
            repo.id = getIDFromUrl(repo.url);
            // nothing changed
            var ckey = repo.pushed_at + repo.watchers;
            if(cached[repo.id] == ckey) return cb();
            cached[repo.id] = ckey;
            // get the watchers, is nice
            console.error("checking "+repo.id);
            github.getRepoApi().getRepoWatchers(auth.username, repo.id.substring(repo.id.indexOf('/') + 1), function(err, watchers){
                repo.watchers = watchers;
                // get a snapshot of the full repo, is nice too
                request.get({uri:"https://api.github.com/repos/" + repo.id + "/git/trees/HEAD?recursive=1", headers:auth.headers, json:true}, function(err, resp, tree) {
                    if(err || !tree || !tree.tree) return cb(err);
                    repo.tree = tree.tree;
                    // see if there's any viewers!
                    var viewer = false;
                    for(var i=0; i < repo.tree.length; i++) if(/^\w+\.app$/.test(repo.tree[i].path)) viewer = repo.tree[i].path;
                    if(!viewer) return cb();
                    console.error("found a viewer! "+repo.id);
                    syncRepo(repo, function(err) {
                        // mangle the manifest so it's a unique handle
                        var manifest = repo.id+"/"+viewer;
                        var js;
                        try {
                            js = JSON.parse(fs.readFileSync(manifest));
                            if(!js || js.static != "true") throw new Error("invalid manifest");
                            js.handle = repo.id.replace("/", "-");
                            js.author = auth.username;
                            fs.writeFileSync(manifest, JSON.stringify(js));
                        } catch (err) {
                            // bail, no viewer for you
                            console.error("failed: "+err);
                            return cb();
                        }
                        viewers.push({id:repo.id, manifest:manifest, at:repo.pushed_at, viewer:js.viewer});
                        request.get({url:lockerUrl+'/map/upsert?manifest=Me/github/'+manifest}, function(err, resp) {
                            cb();
                        });
                    });
                });
            });
{'fraction_non_alphanumeric': 0.14273588909322393, 'fraction_numerical': 0.0008487763474324516, 'mean_word_length': 1.6588943211733735, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243053', 'n_tokens_mistral': 1997, 'n_tokens_neox': 1902, 'n_words': 409}