<commit_before>var mongoose = require('mongoose');
var async = require('async');
var debug = require('debug')('dgv:fetch');

// var q = require('../config/kue');
var geogw = require('../geogw');

var Organization = mongoose.model('Organization');
var Producer = mongoose.model('Producer');
var Dataset = mongoose.model('Dataset');

module.exports = function (job, jobDone) {
    var organizationId = job.data.organizationId;

    var organization, producers;

    function fetchOrganization(done) {
        Organization.findById(organizationId, function (err, organizationFound) {
            if (err) return done(err);
            if (!organizationFound) return done(new Error('Organization not found'));
            organization = organizationFound;
            done();
        });
    }

    function fetchProducers(done) {
        Producer.find({ associatedTo: organizationId }).exec(function (err, producersFound) {
            if (err) return done(err);
            if (producersFound.length === 0) debug('No associated producers');
            producers = producersFound;
            done();
        });
    }

    function cleanMatchingDatasets(done) {
        var query = { matchingFor: organizationId };
        var updateOperations = { $pull: { matchingFor: organizationId } };
        Dataset.update(query, updateOperations, { multi: true }, function (err) {
            if (err) {
                debug('Unable to clean matching datasets for %s: an error has occurred', organizationId);
                return done(err);
            }
            debug('Matching datasets for %s successfully cleaned', organizationId);
            done();
        });
    }

    function fetchAndSaveDatasets(done) {
        var datasetsProcessed = {};

        function fetchAndProcessByProducer(producer, producerDone) {
            var query = {
                limit: 100,
                opendata: 'yes',
                availability: 'yes',
                organization: producer._id
            };

            geogw.fetchRecordsFromCatalog(organization.sourceCatalog, query, function (err, resp) {
                if (err) return producerDone(err);
                debug('%d datasets found for producer `%s`', resp.count, producer._id);

                async.eachLimit(resp.results, 20, function (dataset, datasetDone) {
                    if (dataset.hashedId in datasetsProcessed) {
                        debug('Dataset %s already processed', dataset.hashedId);
                        datasetDone();
                    } else {
                        datasetsProcessed[dataset.hashedId] = true;

                        var updateOperations = {
                            $push: { matchingFor: organizationId },
                            $set: { title: dataset.metadata.title, producers: dataset.organizations }
                        };
                        var opts = { upsert: true };<commit_msg>Increase dataset fetching limit<commit_after>var mongoose = require('mongoose');
var async = require('async');
var debug = require('debug')('dgv:fetch');

// var q = require('../config/kue');
var geogw = require('../geogw');

var Organization = mongoose.model('Organization');
var Producer = mongoose.model('Producer');
var Dataset = mongoose.model('Dataset');

module.exports = function (job, jobDone) {
    var organizationId = job.data.organizationId;

    var organization, producers;

    function fetchOrganization(done) {
        Organization.findById(organizationId, function (err, organizationFound) {
            if (err) return done(err);
            if (!organizationFound) return done(new Error('Organization not found'));
            organization = organizationFound;
            done();
        });
    }

    function fetchProducers(done) {
        Producer.find({ associatedTo: organizationId }).exec(function (err, producersFound) {
            if (err) return done(err);
            if (producersFound.length === 0) debug('No associated producers');
            producers = producersFound;
            done();
        });
    }

    function cleanMatchingDatasets(done) {
        var query = { matchingFor: organizationId };
        var updateOperations = { $pull: { matchingFor: organizationId } };
        Dataset.update(query, updateOperations, { multi: true }, function (err) {
            if (err) {
                debug('Unable to clean matching datasets for %s: an error has occurred', organizationId);
                return done(err);
            }
            debug('Matching datasets for %s successfully cleaned', organizationId);
            done();
        });
    }

    function fetchAndSaveDatasets(done) {
        var datasetsProcessed = {};

        function fetchAndProcessByProducer(producer, producerDone) {
            var query = {
                limit: 500,
                opendata: 'yes',
                availability: 'yes',
                organization: producer._id
            };

            geogw.fetchRecordsFromCatalog(organization.sourceCatalog, query, function (err, resp) {
                if (err) return producerDone(err);
                debug('%d datasets found for producer `%s`', resp.count, producer._id);

                async.eachLimit(resp.results, 20, function (dataset, datasetDone) {
                    if (dataset.hashedId in datasetsProcessed) {
                        debug('Dataset %s already processed', dataset.hashedId);
                        datasetDone();
                    } else {
                        datasetsProcessed[dataset.hashedId] = true;

                        var updateOperations = {
                            $push: { matchingFor: organizationId },
                            $set: { title: dataset.metadata.title, producers: dataset.organizations }
                        };
                        var opts = { upsert: true };
{'fraction_non_alphanumeric': 0.1096311475409836, 'fraction_numerical': 0.0020491803278688526, 'mean_word_length': 2.0190721649484535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6049935', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1413, 'n_words': 367}