<commit_before>var Promise = require('bluebird'),
    fs = require('fs'),
    path = require('path'),
    cheerio = require('cheerio'),
     _ = require('lodash')

var glob = Promise.promisify(require("glob").glob)

var apps = {}

apps.init = function() {

    return glob(__dirname + '/../../fixtures/ui-xml/*.xml')
        .then(function(files) {
            apps.files = files
            return apps
        })
}

// get an app by its id, with fields specified in 'projection'
apps.get = function(id, projection) {

    var projection = projection || {}

    var file = __dirname + '/../../fixtures/ui-xml/' + id + '.xml'
    var xml

    var app = {
        id: id
    }

    //var name = path.basename(file, '.xml')
    var toks = id.split('-')
    //app.packageName = toks[0]
    //app.version = toks[1]

    if (projection.dom) {
        xml = fs.readFileSync(file, 'utf8')
        app.dom = cheerio.load(xml, {
            recognizeSelfClosing: true,
            xmlMode: true
        })
    }

    if (projection.xml) {
        app.xml = xml || fs.readFileSync(file, 'utf8')
    }

    if (projection.perm) {
        app.perm = unpickle('./datasets/apac/perm/' + id + '.json')
    }

    return Promise.resolve(app)
}



function unpickle(src){
    var contents = fs.readFileSync(src, 'utf8')
    var data = []
    var JSONPickle = function(o){
        data.push(o)
    }
    eval(contents)

    var gs = _.groupBy(data, '$type')
    return {
        appData: gs['edu.colorado.permgrep.appData'][0],
        perms: gs['edu.colorado.permgrep.onePerm']
    }
}

module.exports = apps<commit_msg>fix dataset path and add manifest projection<commit_after>var Promise = require('bluebird');
var fs = require('fs');
var path = require('path');
var cheerio = require('cheerio');
var _ = require('lodash');
var config = require("config");
var DATASET_PATH = path.resolve(__dirname, '../../', 'config', config.get('dataset.path'));

var glob = Promise.promisify(require("glob").glob);

var apps = {};

apps.init = function () {

    return glob(path.join(DATASET_PATH, 'ui', '*.xml'))
        .then(function (files) {
            apps.files = files;
            return apps;
        })
};

// get an app by its id, with fields specified in 'projection'
apps.get = function (id, projection) {

    var projection = projection || {};

    var uiFile = path.join(DATASET_PATH, 'ui', id + '.xml');
    var manifestFile = path.join(DATASET_PATH, 'manifest', id + '.xml');
    var xml;

    var app = {
        id: id
    };

    var idVal = id.split('-');
    app.packageName = idVal[0];
    app.version = idVal[1];

    if (projection.dom) {
        xml = fs.readFileSync(uiFile, 'utf8');
        app.dom = cheerio.load(xml, {
            recognizeSelfClosing: true,
            xmlMode: true
        });
    }

    if (projection.xml) {
        app.xml = xml || fs.readFileSync(uiFile, 'utf8');
    }

    if (projection.manifest) {
        var manifest = fs.readFileSync(manifestFile, 'utf8');
        app.manifest = cheerio.load(manifest, {
            recognizeSelfClosing: true,
            xmlMode: true
        })
    }

    if (projection.listings) {
        console.log('listings projection');
    }

    if (projection.perm) {
        if (fs.existsSync(path.join(DATASET_PATH, 'perm', id + '.json'))) {
            app.perm = unpickle(path.join(DATASET_PATH, 'perm', id + '.json'));
        }
        else {
            app.perm = '    '
        }
        return Promise.resolve(app);
    }

    return app
};


function unpickle(src) {
    var contents = fs.readFileSync(src, 'utf8');
    var data = [];
    var JSONPickle = function (o) {
        data.push(o)
    };
    eval(contents);

    var gs = _.groupBy(data, '$type');
    return {
        appData: gs['edu.colorado.permgrep.appData'][0],
        perms: gs['edu.colorado.permgrep.onePerm']
    }
}

module.exports = apps;
{'fraction_non_alphanumeric': 0.16075102880658437, 'fraction_numerical': 0.0033436213991769547, 'mean_word_length': 2.2462437395659434, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 23}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4067393', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1281, 'n_words': 277}