<commit_before>// This code is largely copied from original MapReduce-plugin
// (https://github.com/daleharvey/pouchdb/blob/6830a273e1839c7ea3b7df1998f81050152e1322/src/plugins/pouchdb.mapreduce.js)
// released under Apache License (https://github.com/daleharvey/pouchdb/blob/6830a273e1839c7ea3b7df1998f81050152e1322/LICENSE)

(function() {
"use strict";

var NoEvalMapReduce = function(db) {

  function viewQuery(fun, options) {
    if (!options.complete) {
      return;
    }

    if (!fun.reduce) {
      options.reduce = false;
    }

    function sum(values) {
      return values.reduce(function(a, b) { return a + b; }, 0);
    }

    var builtInReduce = {
      "_sum": function(keys, values){
        return sum(values);
      },

      "_count": function(keys, values, rereduce){
        if (rereduce){
          return sum(values);
        } else {
          return values.length;
        }
      },

      "_stats": function(keys, values, rereduce) {
        return {
          'sum': sum(values),
          'min': Math.min.apply(null, values),
          'max': Math.max.apply(null, values),
          'count': values.length,
          'sumsqr': (function(){
            var _sumsqr = 0;
            for(var idx in values) {
              if (typeof values[idx] === 'number') {
              _sumsqr += values[idx] * values[idx];
              }
            }
            return _sumsqr;
          })()
        };
      }
    };

    var results = [];
    var current = null;
    var num_started= 0;
    var completed= false;

    var emit = function(key, val) {
      var viewRow = {
        id: current.doc._id,
        key: key,
        value: val
      };

      if (options.startkey && Pouch.collate(key, options.startkey) < 0) return;
      if (options.endkey && Pouch.collate(key, options.endkey) > 0) return;
      if (options.key && Pouch.collate(key, options.key) !== 0) return;

      num_started++;
      if (options.include_docs) {
        //in this special case, join on _id (issue #106)
        if (val && typeof val === 'object' && val._id){
          db.get(val._id,
              function(_, joined_doc){
                if (joined_doc) {
                  viewRow.doc = joined_doc;
                }
                results.push(viewRow);
                checkComplete();
              });
          return;
        } else {
          viewRow.doc = current.doc;
        }
      }
      results.push(viewRow);
    };

    // bind emit by `this` reference
    var context = {emit: emit};
    fun.map = fun.map.bind(context);
    if (fun.reduce && builtInReduce[fun.reduce]) {
      fun.reduce = builtInReduce[fun.reduce];
    }

    //only proceed once all documents are mapped and joined
    var checkComplete = function(){
      if (completed && results.length == num_started){
        results.sort(function(a, b) {
          return Pouch.collate(a.key, b.key);
        });
        if (options.descending) {
          results.reverse();
        }
        if (options.reduce === false) {
          return options.complete(null, {
            rows: ('limit' in options)
              ? results.slice(0, options.limit)
              : results,
            total_rows: results.length
          });
        }

        var groups = [];
        results.forEach(function(e) {
          var last = groups[groups.length-1] || null;
          if (last && Pouch.collate(last.key[0][0], e.key) === 0) {
            last.key.push([e.key, e.id]);
            last.value.push(e.value);
            return;
          }
          groups.push({key: [[e.key, e.id]], value: [e.value]});
        });
        groups.forEach(function(e) {
          e.value = fun.reduce(e.key, e.value) || null;
          e.key = e.key[0][0];
        });

        options.complete(null, {
          rows: ('limit' in options)
            ? groups.slice(0, options.limit)
            : groups,
          total_rows: groups.length
        });
      }
    };

    db.changes({
      conflicts: true,
      include_docs: true,
      onChange: function(doc) {
        if (!('deleted' in doc)) {
          current = {doc: doc.doc};
          fun.map.call(this, doc.doc);
        }<commit_msg>Embedded `pouch.collate` module<commit_after>// This code is largely copied from original MapReduce-plugin
// (https://github.com/daleharvey/pouchdb/blob/6830a273e1839c7ea3b7df1998f81050152e1322/src/plugins/pouchdb.mapreduce.js)
// released under Apache License (https://github.com/daleharvey/pouchdb/blob/6830a273e1839c7ea3b7df1998f81050152e1322/LICENSE)

(function() {
"use strict";

var NoEvalMapReduce = function(db) {
  var pouchCollate = function(a, b) {
    var ai = collationIndex(a);
    var bi = collationIndex(b);
    if ((ai - bi) !== 0) {
      return ai - bi;
    }
    if (a === null) {
      return 0;
    }
    if (typeof a === 'number') {
      return a - b;
    }
    if (typeof a === 'boolean') {
      return a < b ? -1 : 1;
    }
    if (typeof a === 'string') {
      return stringCollate(a, b);
    }
    if (Array.isArray(a)) {
      return arrayCollate(a, b);
    }
    if (typeof a === 'object') {
      return objectCollate(a, b);
    }
  };

  var stringCollate = function(a, b) {
    // See: https://github.com/daleharvey/pouchdb/issues/40
    // This is incompatible with the CouchDB implementation, but its the
    // best we can do for now
    return (a === b) ? 0 : ((a > b) ? 1 : -1);
  };

  var objectCollate = function(a, b) {
    var ak = Object.keys(a), bk = Object.keys(b);
    var len = Math.min(ak.length, bk.length);
    for (var i = 0; i < len; i++) {
      // First sort the keys
      var sort = pouchCollate(ak[i], bk[i]);
      if (sort !== 0) {
        return sort;
      }
      // if the keys are equal sort the values
      sort = pouchCollate(a[ak[i]], b[bk[i]]);
      if (sort !== 0) {
        return sort;
      }

    }
    return (ak.length === bk.length) ? 0 :
      (ak.length > bk.length) ? 1 : -1;
  };

  var arrayCollate = function(a, b) {
    var len = Math.min(a.length, b.length);
    for (var i = 0; i < len; i++) {
      var sort = pouchCollate(a[i], b[i]);
      if (sort !== 0) {
        return sort;
      }
    }
    return (a.length === b.length) ? 0 :
      (a.length > b.length) ? 1 : -1;
  };

  // The collation is defined by erlangs ordered terms
  // the atoms null, true, false come first, then numbers, strings,
  // arrays, then objects
  var collationIndex = function(x) {
    var id = ['boolean', 'number', 'string', 'object'];
    if (id.indexOf(typeof x) !== -1) {
      if (x === null) {
        return 1;
      }
      return id.indexOf(typeof x) + 2;
    }
    if (Array.isArray(x)) {
      return 4.5;
    }
  };

  function viewQuery(fun, options) {
    if (!options.complete) {
      return;
    }

    if (!fun.reduce) {
      options.reduce = false;
    }

    function sum(values) {
      return values.reduce(function(a, b) { return a + b; }, 0);
    }

    var builtInReduce = {
      "_sum": function(keys, values){
        return sum(values);
      },

      "_count": function(keys, values, rereduce){
        if (rereduce){
          return sum(values);
        } else {
          return values.length;
        }
      },

      "_stats": function(keys, values, rereduce) {
        return {
          'sum': sum(values),
          'min': Math.min.apply(null, values),
          'max': Math.max.apply(null, values),
          'count': values.length,
          'sumsqr': (function(){
            var _sumsqr = 0;
            for(var idx in values) {
              if (typeof values[idx] === 'number') {
              _sumsqr += values[idx] * values[idx];
              }
            }
            return _sumsqr;
          })()
        };
      }
    };

    var results = [];
    var current = null;
    var num_started= 0;
    var completed= false;

    var emit = function(key, val) {
      var viewRow = {
        id: current.doc._id,
        key: key,
        value: val
      };

      if (options.startkey && pouchCollate(key, options.startkey) < 0) return;
      if (options.endkey && pouchCollate(key, options.endkey) > 0) return;
      if (options.key && pouchCollate(key, options.key) !== 0) return;

      num_started++;
      if (options.include_docs) {
        //in this special case, join on _id (issue #106)
        if (val && typeof val === 'object' && val._id){
          db.get(val._id,
              function(_, joined_doc){
                if (joined_doc) {
                  viewRow.doc = joined_doc;
                }
                results.push(viewRow);
                checkComplete();
              });
          return;
        } else {
          viewRow.doc = current.doc;
        }
      }
      results.push(viewRow);
    };

    // bind emit by `this` reference
    var context = {emit: emit};
    fun.map = fun.map.bind(context);
    if (fun.reduce && builtInReduce[fun.reduce]) {
      fun.reduce = builtInReduce[fun.reduce];
    }

    //only proceed once all documents are mapped and joined
    var checkComplete = function(){
      if (completed && results.length == num_started){
        results.sort(function(a, b) {
          return pouchCollate(a.key, b.key);
        });
        if (options.descending) {
          results.reverse();
        }
        if (options.reduce === false) {
          return options.complete(null, {
            rows: ('limit' in options)
              ? results.slice(0, options.limit)
              : results,
            total_rows: results.length
          });
        }

        var groups = [];
        results.forEach(function(e) {
          var last = groups[groups.length-1] || null;
          if (last && pouchCollate(last.key[0][0], e.key) === 0) {
            last.key.push([e.key, e.id]);
            last.value.push(e.value);
            return;
          }
          groups.push({key: [[e.key, e.id]], value: [e.value]});
        });
        groups.forEach(function(e) {
          e.value = fun.reduce(e.key, e.value) || null;
          e.key = e.key[0][0];
        });

        options.complete(null, {
          rows: ('limit' in options)
            ? groups.slice(0, options.limit)
            : groups,
          total_rows: groups.length
        });
      }
    };

    db.changes({
      conflicts: true,
      include_docs: true,
      onChange: function(doc) {
        if (!('deleted' in doc)) {
          current = {doc: doc.doc};
          fun.map.call(this, doc.doc);
        }
{'fraction_non_alphanumeric': 0.14665646225963838, 'fraction_numerical': 0.017124270544341336, 'mean_word_length': 1.7539515279241307, 'pattern_counts': {'":': 6, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7559947', 'n_tokens_mistral': 3466, 'n_tokens_neox': 3303, 'n_words': 769}