<commit_before>var gulp = require('gulp'),
    rename = require('gulp-rename'),
    clean = require('gulp-clean'),
    ts = require('gulp-typescript'),
    sass = require('gulp-sass'),
    sourcemaps = require('gulp-sourcemaps'),
    jsontransform = require('gulp-json-transform'),
    concatjson = require('gulp-concat-json'),
    run = require('run-sequence'),
    exec = require('gulp-exec'),
    tsConfig = require('./tsconfig'),
    bibtex2json = require('./citation/bibtex2json'),
    fs = require('fs');

var paths = {
    src: 'app',
    dev: 'www',
    json: 'comparison-elements-json/',
    data: 'app/components/comparison/data/'
};

var files = {
    index: './index.html',
    systemjs: [
        './systemjs.config.js',
    ],
    typescripts: [
        './app/**/*.ts'
    ],
    scripts: [
        './app/**/*.js',
        './app/**/*.js.map',
    ],
    templates: [
        './app/**/*.html'
    ],
    css: [
        './app/**/*.css'
    ],
    data: [
        './app/components/comparison/data/*.json',
        './comparison-configuration/*',
        './citation/output/*'
    ],
    markdown: [
        './comparison-elements/*.md'
    ],
    json: [
        './comparison-elements-json/*.json'
    ],
    node_modules_scripts: [
        './node_modules/es6-shim/**/es6-shim.min.js',
        './node_modules/zone.js/dist/zone.js',
        './node_modules/reflect-metadata/Reflect.js',
        './node_modules/systemjs/dist/system.src.js',
        './node_modules/rxjs/**/*.js',
        './node_modules/angular2-in-memory-web-api/**/*.js',
        './node_modules/@angular/**/*.js',
        './node_modules/select2/**/*.js',
        './node_modules/jquery/**/*.js',
        './node_modules/file-saver/**/*js',
        './node_modules/showdown/**/*.js',
        './node_modules/npm-polymer-elements/**/*',
         "./node_modules/@vaadin/angular2-polymer/**/*",
         "./node_modules/angular2-select/**/*"
    ]
}

var destfiles = {
    index: './www',
    app: './www/app'
}

// BUILD / UPDATE standard files ---------------------------------<
gulp.task('build-standard', ['ts'], function () {})

gulp.task('update-standard', function () {
    gulp.watch(files.typescripts, ['ts']);
})

gulp.task('ts', function () {
    var tsProject = ts.createProject('tsconfig.json');
    return tsProject.src()
        .pipe(sourcemaps.init()) 
        .pipe(ts(tsConfig.compilerOptions))
        .pipe(sourcemaps.write('.'))
        .pipe(gulp.dest(paths.src));     
});

// END BUILD standard files -------------------------------------->

// BUILD / UPDATE data files -------------------------------------<
gulp.task('build-data', function (callback) {
    run('markdown', 'json', callback);
})

gulp.task('update-data', function () {
    gulp.watch(files.markdown, ['build-data']);
})

gulp.task('markdown', function(callback){
    var options = {
      continueOnError: false,
      pipeStdout: true
    }
    return gulp.src(files.markdown)
    .pipe(exec("./gradlew -q -b ./app/java/md-to-json/build.gradle run -PappArgs=\"[$/\n<%= file.contents.toString() %>\n/$, 1, true]\"", options))
    .pipe(rename({extname: ".json"}))
    .pipe(gulp.dest(paths.json));
});

gulp.task('json', function(){
    return gulp.src(files.json)
        .pipe(concatjson("data.json"))
        .pipe(jsontransform(function(data){
            return data;
        }, 2))
        .pipe(gulp.dest(paths.data))
})

gulp.task('citation', function(){
    var fileContent = JSON.parse(fs.readFileSync("./citation/config.json", "utf8"));
    bibtex2json.parse('./citation/' + fileContent.bibtex_file, 'utf8', './citation/' + fileContent.bibtex_style, './citation/output');
})
// ---------------------------------------------------------------><commit_msg>publish: gulp fix (add citation task to build-data task)<commit_after>var gulp = require('gulp'),
    rename = require('gulp-rename'),
    clean = require('gulp-clean'),
    ts = require('gulp-typescript'),
    sass = require('gulp-sass'),
    sourcemaps = require('gulp-sourcemaps'),
    jsontransform = require('gulp-json-transform'),
    concatjson = require('gulp-concat-json'),
    run = require('run-sequence'),
    exec = require('gulp-exec'),
    tsConfig = require('./tsconfig'),
    bibtex2json = require('./citation/bibtex2json'),
    fs = require('fs');

var paths = {
    src: 'app',
    dev: 'www',
    json: 'comparison-elements-json/',
    data: 'app/components/comparison/data/'
};

var files = {
    index: './index.html',
    systemjs: [
        './systemjs.config.js',
    ],
    typescripts: [
        './app/**/*.ts'
    ],
    scripts: [
        './app/**/*.js',
        './app/**/*.js.map',
    ],
    templates: [
        './app/**/*.html'
    ],
    css: [
        './app/**/*.css'
    ],
    data: [
        './app/components/comparison/data/*.json',
        './comparison-configuration/*',
        './citation/output/*'
    ],
    markdown: [
        './comparison-elements/*.md'
    ],
    json: [
        './comparison-elements-json/*.json'
    ],
    node_modules_scripts: [
        './node_modules/es6-shim/**/es6-shim.min.js',
        './node_modules/zone.js/dist/zone.js',
        './node_modules/reflect-metadata/Reflect.js',
        './node_modules/systemjs/dist/system.src.js',
        './node_modules/rxjs/**/*.js',
        './node_modules/angular2-in-memory-web-api/**/*.js',
        './node_modules/@angular/**/*.js',
        './node_modules/select2/**/*.js',
        './node_modules/jquery/**/*.js',
        './node_modules/file-saver/**/*js',
        './node_modules/showdown/**/*.js',
        './node_modules/npm-polymer-elements/**/*',
         "./node_modules/@vaadin/angular2-polymer/**/*",
         "./node_modules/angular2-select/**/*"
    ]
}

var destfiles = {
    index: './www',
    app: './www/app'
}

// BUILD / UPDATE standard files ---------------------------------<
gulp.task('build-standard', ['ts'], function () {})

gulp.task('update-standard', function () {
    gulp.watch(files.typescripts, ['ts']);
})

gulp.task('ts', function () {
    var tsProject = ts.createProject('tsconfig.json');
    return tsProject.src()
        .pipe(sourcemaps.init()) 
        .pipe(ts(tsConfig.compilerOptions))
        .pipe(sourcemaps.write('.'))
        .pipe(gulp.dest(paths.src));     
});

// END BUILD standard files -------------------------------------->

// BUILD / UPDATE data files -------------------------------------<
gulp.task('build-data', function (callback) {
    run('markdown', 'json', 'citation', callback);
})

gulp.task('update-data', function () {
    gulp.watch(files.markdown, ['build-data']);
})

gulp.task('markdown', function(callback){
    var options = {
      continueOnError: false,
      pipeStdout: true
    }
    return gulp.src(files.markdown)
    .pipe(exec("./gradlew -q -b ./app/java/md-to-json/build.gradle run -PappArgs=\"[$/\n<%= file.contents.toString() %>\n/$, 1, true]\"", options))
    .pipe(rename({extname: ".json"}))
    .pipe(gulp.dest(paths.json));
});

gulp.task('json', function(){
    return gulp.src(files.json)
        .pipe(concatjson("data.json"))
        .pipe(jsontransform(function(data){
            return data;
        }, 2))
        .pipe(gulp.dest(paths.data))
})

gulp.task('citation', function(){
    var fileContent = JSON.parse(fs.readFileSync("./citation/config.json", "utf8"));
    bibtex2json.parse('./citation/' + fileContent.bibtex_file, 'utf8', './citation/' + fileContent.bibtex_style, './citation/output');
})
// --------------------------------------------------------------->
{'fraction_non_alphanumeric': 0.252948191334305, 'fraction_numerical': 0.0034450775142440705, 'mean_word_length': 2.1306511820821235, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45023', 'n_tokens_mistral': 2470, 'n_tokens_neox': 2450, 'n_words': 343}