<commit_before>'use strict';
const gulp = require('gulp'),
    glob = require('glob'),
    path = require('path'),
    exec = require('child_process').exec,
    Promise = require('bluebird'),
    execAsync = Promise.promisify(exec),
    config = require('config'),
    mkdirp = require('mkdirp'),
    log = require('../lib/logger'),
    tagNameExtractor = require('../lib/index/tag-name-extractor'),
    suffixExtractor = require('../lib/index/suffix-extractor'),
    DATASET_PATH = path.resolve(__dirname + '/../', 'config',
    config.get('dataset.path'));

gulp.task('extract:archives', (done) => {
    const untarListings = 'tar xvjf ' + path.join(DATASET_PATH, 'listing',
            'listing.tar.bz2') + ' -C ' + path.join(DATASET_PATH, 'listing'),
        untarUI = 'tar xvjf ' + path.join(DATASET_PATH, 'ui', 'ui-xml.tar.bz2') +
                  ' -C ' + path.join(DATASET_PATH, 'ui'),
        untarManifest = 'tar xvjf ' + path.join(DATASET_PATH, 'manifest',
                        'manifest.tar.bz2') + ' -C ' + path.join(DATASET_PATH,
            'manifest'),
        untarCode = 'tar xvjf ' + path.join(DATASET_PATH, 'code',
                    'smali-invoked-methods.tar.bz2') + ' -C ' +
                     path.join(DATASET_PATH, 'code');

    log.info('Extracting listing details json files...');
    execAsync(untarListings)
        .then(() => {
            log.info('Extracting UI xml files...');
            return execAsync(untarUI);
        })
        .then(() => {
            log.info('Extracting Manifest xml files...');
            return execAsync(untarManifest);
        })
        .then(() => {
            log.info('Extracting code text files...');
            return execAsync(untarCode);
        })
        .then(() => {
            log.info('All archives have been extracted.');
            done();
        })
        .error((error) => {
            log.error(error);
            done(error);
        });
});

gulp.task('extract:ui-tag', (callback) => {
    // create a directory that contains all extracted files.
    const dir = path.resolve(__dirname + '/../', 'config',
                             config.get('indexes.extractUITagDir'));
    mkdirp.sync(dir, { mode: '2775' });
    // Extract tag names and attributes.
    glob(path.join(DATASET_PATH, 'ui', '*.xml'), (err, files) => {
        tagNameExtractor(files, dir, '-ui-tag', (e) => {
            callback(e);
        });
    });
});

gulp.task('extract:ui-suffix', (callback) => {
    // create a directory that contains all extracted files.
    const dir = path.resolve(__dirname + '/../', 'config',
                             config.get('indexes.extractUISuffixDir'));
    mkdirp.sync(dir, { mode: '2775' });
    // Extract tag names and attributes.
    glob(path.join(DATASET_PATH, 'ui', '*.xml'), (err, files) => {
        suffixExtractor(files, dir, (e) => {
            callback(e);
        });
    });
});

gulp.task('extract:manifest', (callback) => {
    // create a directory that contains all extracted files.
    const dir = path.resolve(__dirname + '/../', 'config',
        config.get('indexes.extractManifestDir'));
    mkdirp.sync(dir, { mode: '2775' });
    // Extract tag names and attributes.
    glob(path.join(DATASET_PATH, 'manifest', '*.xml'), (err, files) => {
        tagNameExtractor(files, dir, '-manifest-tag', (e) => {
            callback(e);
        });
    });
});<commit_msg>refactor and use the new dataset paths<commit_after>'use strict';
const gulp = require('gulp'),
    path = require('path'),
    _ = require('lodash'),
    exec = require('child_process').exec,
    Promise = require('bluebird'),
    execAsync = Promise.promisify(exec),
    fs = Promise.promisifyAll(require('fs')),
    config = require('config'),
    mkdirp = require('mkdirp'),
    log = require('../lib/logger'),
    tagNameExtractor = require('../lib/index/tag-name-extractor'),
    suffixExtractor = require('../lib/index/suffix-extractor'),
    CONFIG_PATH = path.resolve(__dirname + '/../', 'config');

function getUntarCommand(obj) {
    // If the target directory does not exist, create it
    mkdirp.sync(path.resolve(CONFIG_PATH, obj.target), { mode: '2775' });
    // return the extract tar command
    return 'tar xvjf ' + path.resolve(CONFIG_PATH, obj.source) +
           ' -C ' + path.resolve(CONFIG_PATH, obj.target);
}

gulp.task('extract:archives', () => {
    const listing = _.map(config.get('dataset.listing'), getUntarCommand),
        code = _.map(config.get('dataset.code'), getUntarCommand),
        manifest = _.map(config.get('dataset.manifest'), getUntarCommand),
        ui = _.map(config.get('dataset.ui'), getUntarCommand),
        commands = listing.concat(code).concat(manifest).concat(ui);
    return Promise.map(commands, (cmd) => {
        log.info(cmd);
        return execAsync(cmd);
    }).then(() => {
        log.info('All archives have been extracted.');
    }).error((error) => {
        log.error(error);
        return Promise.reject(error);
    });
});

gulp.task('extract:ui-tag', () => {
    const uiConfigs = _.map(config.dataset.ui, (ui) => {
        return { source: ui.target, target: ui.indexes.extractUITagDir };
    });
    return Promise.map(uiConfigs, (uiConfig) => {
        mkdirp.sync(path.resolve(CONFIG_PATH, uiConfig.target), { mode: '2775' });
        return fs.readdirAsync(path.resolve(CONFIG_PATH, uiConfig.source))
            .then((files) => {
                return _.filter(files, (f) => {
                    return path.extname(f) === '.xml';
                });
            })
            .then((xmlFileNames) => {
                const xmlFileNamePaths = _.map(xmlFileNames, (file) => {
                    return path.resolve(CONFIG_PATH, uiConfig.source, file);
                });
                return tagNameExtractor(xmlFileNamePaths,
                  path.resolve(CONFIG_PATH, uiConfig.target), '-ui-tag');
            })
            .then(() => {
                log.info('Extracted ui tag names for all XML files at ' +
                          path.resolve(CONFIG_PATH, uiConfig.source));
            })
            .catch((e) => {
                log.error(e);
            });
    }).then(() => {
        return Promise.resolve();
    }).catch((e) => {
        log.error(e);
        return Promise.reject(e);
    });
});

gulp.task('extract:ui-suffix', () => {
    const uiConfigs = _.map(config.dataset.ui, (ui) => {
        return { source: ui.target, target: ui.indexes.extractUISuffixDir };
    });
    return Promise.map(uiConfigs, (uiConfig) => {
        mkdirp.sync(path.resolve(CONFIG_PATH, uiConfig.target), { mode: '2775' });
        return fs.readdirAsync(path.resolve(CONFIG_PATH, uiConfig.source))
            .then((files) => {
                return _.filter(files, (f) => {
                    return path.extname(f) === '.xml';
                });
            })
            .then((xmlFileNames) => {
                const xmlFileNamePaths = _.map(xmlFileNames, (file) => {
                    return path.resolve(CONFIG_PATH, uiConfig.source, file);
                });
                return suffixExtractor(xmlFileNamePaths,
                  path.resolve(CONFIG_PATH, uiConfig.target));
            })
            .then(() => {
                log.info('Extracted ui suffix for all XML files at ' +
                          path.resolve(CONFIG_PATH, uiConfig.source));
            })
            .catch((e) => {
                log.error(e);
            });
    }).then(() => {
        return Promise.resolve();
    }).catch((e) => {
        log.error(e);
        return Promise.reject(e);
    });
});

gulp.task('extract:manifest', () => {
    const manifestConfigs = _.map(config.dataset.manifest, (manifest) => {
        return { source: manifest.target, target: manifest.indexes.extractManifestDir };
    });
    return Promise.map(manifestConfigs, (manifestConfig) => {
        mkdirp.sync(path.resolve(CONFIG_PATH, manifestConfig.target), { mode: '2775' });
        return fs.readdirAsync(path.resolve(CONFIG_PATH, manifestConfig.source))
            .then((files) => {
                return _.filter(files, (f) => {
                    return path.extname(f) === '.xml';
                });
            })
            .then((xmlFileNames) => {
                const xmlFileNamePaths = _.map(xmlFileNames, (file) => {
                    return path.resolve(CONFIG_PATH, manifestConfig.source, file);
                });
                return tagNameExtractor(xmlFileNamePaths,
                  path.resolve(CONFIG_PATH, manifestConfig.target), '-manifest-tag');
            })
            .then(() => {
                log.info('Extracted manifest tag names for all XML files at ' +
                          path.resolve(CONFIG_PATH, manifestConfig.source));
            })
            .catch((e) => {
                log.error(e);
            });
    }).then(() => {
        return Promise.resolve();
    }).catch((e) => {
        log.error(e);
        return Promise.reject(e);
    });
});
{'fraction_non_alphanumeric': 0.16877216312056736, 'fraction_numerical': 0.0035460992907801418, 'mean_word_length': 2.0686841210472626, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 21}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4767592', 'n_tokens_mistral': 2702, 'n_tokens_neox': 2622, 'n_words': 482}