<commit_before>var gulp = require('gulp');
var path = require('path');
var config = require('config');
var through2 = require('through2');
var mongo = require('../lib/db/mongo-client.js');
var log = require("../lib/logger");
var DATASET_PATH = path.resolve(__dirname + '/../', 'config', config.get('dataset.path'));
var collectionName = config.get("dbConfig.mongo.collections")[0].collection;

gulp.task('mongo:insertListing', function (done) {
    gulp.src(path.join(DATASET_PATH, 'listing', '*.json'))
        .pipe(new through2.obj(function (file, enc, cb) {
            var doc = JSON.parse(file.contents);
            doc['id'] = doc['n'] + '-' + doc['verc'];
            mongo.upsertOne(collectionName, doc, doc)
                .then(function () {
                    cb();
                })
                .error(function (e) {
                    log.error('Error in insert:listing task: ' + e.message);
                });
        }, function () {
            done();
        }));
});

gulp.task('mongo:indexListing', function (done) {
    mongo.createIndex(collectionName, {id: 1}, {unique: true})
        .then(function () {
            mongo.createIndexes(collectionName, [{
                    key: {
                        t: "text",
                        desc: "text",
                        new: "text"
                    },
                    weights: {
                        t: 10,
                        desc: 5,
                        new: 1
                    },
                    name: "FullTextIndex"
                }],
                function (err, res) {
                    if (err) {
                        log.error('Error in mongoIndex task: ' + err);
                        done();
                    }
                    else {
                        console.log(res);
                        done();
                    }
                }
            )
        })
});<commit_msg>refactoring<commit_after>var gulp = require('gulp');
var path = require('path');
var config = require('config');
var through2 = require('through2');
var mongo = require('../lib/db/mongo-client.js');
var log = require("../lib/logger");
var DATASET_PATH = path.resolve(__dirname + '/../', 'config', config.get('dataset.path'));
var collectionName = config.get("dbConfig.mongo.collections")[0].collection;

gulp.task('mongo:insertListing', function (done) {
    gulp.src(path.join(DATASET_PATH, 'listing', '*.json'))
        .pipe(new through2.obj(function (file, enc, cb) {
            var doc = JSON.parse(file.contents);
            doc['id'] = doc['n'] + '-' + doc['verc'];
            mongo.upsertOne(collectionName, doc, doc)
                .then(function () {
                    cb();
                })
                .error(function (e) {
                    log.error('Error in insert:listing task: ' + e.message);
                });
        }, function () {
            done();
        }));
});

gulp.task('mongo:indexListing', function (done) {
    mongo.createIndex(collectionName, {id: 1}, {unique: true})
        .then(function () {
            mongo.createIndexes(collectionName, [{
                    key: {
                        t: "text",
                        desc: "text",
                        new: "text"
                    },
                    weights: {
                        t: 10,
                        desc: 5,
                        new: 1
                    },
                    name: "FullTextIndex"
                }],
                function (err, res) {
                    console.log(res);
                    done(err);
                }
            )
        })
});
{'fraction_non_alphanumeric': 0.15062875888463642, 'fraction_numerical': 0.004920721705850191, 'mean_word_length': 1.271260086902545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1020172', 'n_tokens_mistral': 1019, 'n_tokens_neox': 960, 'n_words': 185}