<commit_before>#!/usr/bin/env node

var fs = require('fs');
var path = require('path');
var argv = process.argv;
var api = {
    '.s3': require('../api-s3'),
    '.mbtiles': require('../api-mbtiles')
};
var Carmen = require('../index.js');
var tokenize = require('../lib/tokenize');
var Queue = require('../queue');
var _ = require('underscore');
var f = argv[2];

if (!f) {
    console.warn('Usage: carmen-analyze.js <file>');
    process.exit(1);
}
if (!fs.existsSync(f)) {
    console.warn('File %s does not exist.', f);
    process.exit(1);
}
if (!api[path.extname(f)]) {
    console.warn('File %s format not recognized.', f);
    process.exit(1);
}

console.log('Analyzing %s ...', f);

var s = new api[path.extname(f)](f, function() {});
var carmen = new Carmen({ s: s });
var stats = {};

carmen._open(function(err) {
    if (err) throw err;
    var shardlevel = s._geocoder.shardlevel;
    stats.shardlevel = shardlevel;
    stats.term = {
        min:Infinity,
        max:0,
        mean:0,
        count:0,
        maxes:[]
    };
    stats.grid = {
        min:Infinity,
        max:0,
        mean:0,
        count:0,
        maxes:[]
    };

    function termlookup(maxes, i, callback) {
        if (i >= maxes.length) return callback();
        var term = maxes[i][0];
        var grid = maxes[i][2];
        var ids = [grid];
        s._geocoder.getall(s.getCarmen.bind(s), 'grid', ids, function(err, result) {
            if (err) return callback(err);
            if (!result.length) return callback(new Error('Grid ' + grid + ' not found'));
            var id = result[0] % Math.pow(2,25);
            s.getFeature(id, function(err, doc) {
                if (err) return callback(err);
                var text = doc.search || doc.name || '';
                var query = tokenize(text);
                var terms = Carmen.terms(text);
                var idx = terms.indexOf(+term);
                if (idx !== -1) {
                    maxes[i].unshift(query[idx]);
                } else {
                    maxes[i].unshift(text);
                }
                termlookup(maxes, ++i, callback);
            });
        });
    };

    function phraselookup(maxes, i, callback) {
        if (i >= maxes.length) return callback();
        var grid = maxes[i][0];
        var ids = [grid];
        s._geocoder.getall(s.getCarmen.bind(s), 'grid', ids, function(err, result) {
            if (!result.length) return callback(new Error('Grid ' + grid + ' not found'));
            var id = result[0] % Math.pow(2,25);
            s.getFeature(id, function(err, doc) {
                if (err) return callback(err);
                var text = doc.search || doc.name || '';
                _(text.split(',')).each(function(syn) {
                    if (Carmen.phrase(syn) === +grid) {
                        maxes[i].unshift(tokenize(syn).join(' '));
                    }
                });
                phraselookup(maxes, ++i, callback);
            });
        });
    };

    function relstats(type, i, callback) {<commit_msg>Update carmen-analyze.<commit_after>#!/usr/bin/env node

var fs = require('fs');
var path = require('path');
var argv = process.argv;
var api = {
    '.s3': require('../api-s3'),
    '.mbtiles': require('../api-mbtiles')
};
var Carmen = require('../index.js');
var termops = require('../lib/util/termops.js');
var Queue = require('../queue');
var _ = require('underscore');
var f = argv[2];

if (!f) {
    console.warn('Usage: carmen-analyze.js <file>');
    process.exit(1);
}
if (!fs.existsSync(f)) {
    console.warn('File %s does not exist.', f);
    process.exit(1);
}
if (!api[path.extname(f)]) {
    console.warn('File %s format not recognized.', f);
    process.exit(1);
}

console.log('Analyzing %s ...', f);

var s = new api[path.extname(f)](f, function() {});
var carmen = new Carmen({ s: s });
var stats = {};

carmen._open(function(err) {
    if (err) throw err;
    var shardlevel = s._geocoder.shardlevel;
    stats.shardlevel = shardlevel;
    stats.term = {
        min:Infinity,
        max:0,
        mean:0,
        count:0,
        maxes:[]
    };
    stats.grid = {
        min:Infinity,
        max:0,
        mean:0,
        count:0,
        maxes:[]
    };

    function termlookup(maxes, i, callback) {
        if (i >= maxes.length) return callback();
        var term = maxes[i][0];
        var grid = maxes[i][2];
        var ids = [grid];
        s._geocoder.getall(s.getCarmen.bind(s), 'grid', ids, function(err, result) {
            if (err) return callback(err);
            if (!result.length) return callback(new Error('Grid ' + grid + ' not found'));
            var id = result[0] % Math.pow(2,25);
            s.getFeature(id, function(err, doc) {
                if (err) return callback(err);
                var text = doc.search || doc.name || '';
                var query = termops.tokenize(text);
                var terms = termops.terms(text);
                var idx = terms.indexOf(+term);
                if (idx !== -1) {
                    maxes[i].unshift(query[idx]);
                } else {
                    maxes[i].unshift(text);
                }
                termlookup(maxes, ++i, callback);
            });
        });
    };

    function phraselookup(maxes, i, callback) {
        if (i >= maxes.length) return callback();
        var grid = maxes[i][0];
        var ids = [grid];
        s._geocoder.getall(s.getCarmen.bind(s), 'grid', ids, function(err, result) {
            if (!result.length) return callback(new Error('Grid ' + grid + ' not found'));
            var id = result[0] % Math.pow(2,25);
            s.getFeature(id, function(err, doc) {
                if (err) return callback(err);
                var text = doc.search || doc.name || '';
                _(text.split(',')).each(function(syn) {
                    if (termops.phrase(syn) === +grid) {
                        maxes[i].unshift(termops.tokenize(syn).join(' '));
                    }
                });
                phraselookup(maxes, ++i, callback);
            });
        });
    };

    function relstats(type, i, callback) {
{'fraction_non_alphanumeric': 0.1742806047797106, 'fraction_numerical': 0.007803609169240774, 'mean_word_length': 1.8560817084493966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5363749', 'n_tokens_mistral': 2021, 'n_tokens_neox': 1972, 'n_words': 433}