<commit_before>(function(){
"use strict";

hilarious.use_web_speech_recognition_api = (window.annyang != null);
if(!hilarious.use_web_speech_recognition_api) {
  return;
}

bililiteRange.fn.expandToInclude = function(otherRange) {
  return this.bounds([
    Math.min(this.bounds()[0], otherRange.bounds()[0]),
    Math.max(this.bounds()[1], otherRange.bounds()[1])
    ]);
};

// For commands that are often heard as ambiguous enough
// that it helps to capture homophones, we check some of those.
XRegExp.addToken(
  /{number}/,
  function() {return '(?:(?:-|−|negative|minus|dash|hyphen)? ?[0-9]+|a|one|two|to|too|three|for|four|infinit[yei]|every|all)';},
  {leadChar: '{'}
);
XRegExp.addToken(
  /{through}/,
  function() {return '(?:-|to|two|too|2|through|thru)';},
  {leadChar: '{'}
);

// Use with the custom XRegExp token "{number}".
function parse_spoken_count(count) {
  if(count == null) {
    return 1;
  }
  count = count.trim();
  var negated = /^(-(?=.)|−|negative|minus|hyphen|dash) ?(.*)$/.exec(count);
  if(negated) { count = negated[2]; }
  var sign = (negated ? -1 : 1);
  if(count === '' || count === 'a' || count === 'an' || count === 'one') {
    return sign * 1;
  } else if(count === 'two' || count === 'to' || count === 'too' || count === '-') {
    return sign * 2;
  } else if(count === 'three') {
    return sign * 3;
  } else if(count === 'four' || count == 'for') {
    return sign * 4;
  //} else if(match = /^(-|negative|minus|hyphen|dash)? ?(?:completely|every|infinit[eyi]|all)$/i.exec(count)) {
  } else if(/^(?:infinit[eyi]|every|all)$/i.test(count)) {
    return sign * Infinity;
  } else if(/^[0-9]+$/.test(count)) {
    return sign * +count;
  } else {
    return null;
  }
}

var unicode_names_map = null;
var unicode_names_string = '';
$.ajax({
  method: 'GET',
  cache: true,
  dataType: 'json',
  url: 'unicode_names_map.json',
}).done(function(data) {
  unicode_names_map = data;
  // TODO is this too slow for the main thread?
  // also what order will it be in?
  _.each(unicode_names_map, function(character, name) {
    unicode_names_string += (name + '\n');
  });
});

// For recognizing the user saying "U+03C1",
// the recognizer isn't very good at it.
// I could match "you plus" and then a pause and then
// each individual hex character separated by pauses.
// It's better at "0x..." (but not great) so hmm.
// I wonder if testing more than five possible matches
// would be useful at all here.
// If I have a command to take the preceding text
// and turn it into a Unicode character(s), then
// that could work with copy/pasted u+ info too.
// Also a command to turn unicode into hex or a sidebar
// so you can see what weird things are happening


// TODO make this configurable and/or autodetected:
var num_tab_spaces = 2;
var tab_spaces = ' '.repeat(num_tab_spaces);
function artificially_type(text) {
  var selection = bililiteRange(document.activeElement).bounds('selection');
  if(text == '\n') {
    selection.insertEOL().select();
  } else {
    selection.text(text, 'end').select();
  }
}
var added_commands = {};
var added_literal_commands = {};
// this depends on a patched annyang that exposes registerCommand
// also maybe I should hack around its trimming: done
// also TODO case-fold rather than lowercase
annyang.debug();
annyang.registerCommand({
    exec: function(str) {
      var normalizedstr = str.trim().toLowerCase();
      var matchfunc = added_literal_commands[normalizedstr];
      return (matchfunc ? [null, normalizedstr, matchfunc] : null);
    }},
    function(normalizedstr, matchfunc) {
      matchfunc(normalizedstr);
    },
    '(literal match)'
    );
function showCodePoint(codepointNumber) {
  var s = codepointNumber.toString(16).toUpperCase();
  if(s.length < 4) {
    s = '0'.repeat(4 - s.length) + s;
  }
  return 'U+' + s;
}
function search_for_unicode_characters(searched) {
      console.log("Trying to find unicode matches for: " + searched);
      var words = searched.split(/[- \t]+/);
      var searcher = new RegExp('^' +
        _.map(words, function(word) { return '(?=.*'+word+')'; }).join('') +
        '.*$',
        'gm');
      //console.log(searcher.source);
      // TODO sorted by codepoint maybe??
      // also what about speech recognition multiple results hm.
      var max_to_find = 35;
      for(var num_found = 0; num_found < max_to_find; num_found++) {
        var match = searcher.exec(unicode_names_string);
        if(!match) { break; }
        var name = match[0];
        var character = unicode_names_map[name];
        var codepoint = character.codePointAt(0);
        // When HTMLing this, use <bdi> around the character, IIRC
        console.log("Possible match: " + character + ' ' + showCodePoint(codepoint) + ' ' + name);
      }
      if(num_found === max_to_find) {
        console.log("And more unshown matches...");
      }
      return num_found !== 0;
}
annyang.registerCommand({
  exec: function(str) {
    var match = /^unicode (.*)$/i.exec(str);
    if(!match) { return false; }
    var possible_name = match[1].toUpperCase();
    if(!_.has(unicode_names_map, possible_name)) {
       var any_found = search_for_unicode_characters(possible_name);
       return (any_found ? [null, possible_name, null] : null);
    }
    return [null, possible_name, unicode_names_map[possible_name]];
  }},
  function(name, character) {
    if(character != null) {
      artificially_type(character);
    }
  },
  'unicode <unicode-character-name>');
      
function add_command(regex_or_str, fn) {
  var name = regex_or_str;<commit_msg>improve unicode searching a bit<commit_after>(function(){
"use strict";

hilarious.use_web_speech_recognition_api = (window.annyang != null);
if(!hilarious.use_web_speech_recognition_api) {
  return;
}

bililiteRange.fn.expandToInclude = function(otherRange) {
  return this.bounds([
    Math.min(this.bounds()[0], otherRange.bounds()[0]),
    Math.max(this.bounds()[1], otherRange.bounds()[1])
    ]);
};

// For commands that are often heard as ambiguous enough
// that it helps to capture homophones, we check some of those.
XRegExp.addToken(
  /{number}/,
  function() {return '(?:(?:-|−|negative|minus|dash|hyphen)? ?[0-9]+|a|one|two|to|too|three|for|four|infinit[yei]|every|all)';},
  {leadChar: '{'}
);
XRegExp.addToken(
  /{through}/,
  function() {return '(?:-|to|two|too|2|through|thru)';},
  {leadChar: '{'}
);

// Use with the custom XRegExp token "{number}".
function parse_spoken_count(count) {
  if(count == null) {
    return 1;
  }
  count = count.trim();
  var negated = /^(-(?=.)|−|negative|minus|hyphen|dash) ?(.*)$/.exec(count);
  if(negated) { count = negated[2]; }
  var sign = (negated ? -1 : 1);
  if(count === '' || count === 'a' || count === 'an' || count === 'one') {
    return sign * 1;
  } else if(count === 'two' || count === 'to' || count === 'too' || count === '-') {
    return sign * 2;
  } else if(count === 'three') {
    return sign * 3;
  } else if(count === 'four' || count == 'for') {
    return sign * 4;
  //} else if(match = /^(-|negative|minus|hyphen|dash)? ?(?:completely|every|infinit[eyi]|all)$/i.exec(count)) {
  } else if(/^(?:infinit[eyi]|every|all)$/i.test(count)) {
    return sign * Infinity;
  } else if(/^[0-9]+$/.test(count)) {
    return sign * +count;
  } else {
    return null;
  }
}

var unicode_names_map = null;
var unicode_names_string = '';
$.ajax({
  method: 'GET',
  cache: true,
  dataType: 'json',
  url: 'unicode_names_map.json',
}).done(function(data) {
  unicode_names_map = data;
  // TODO is this too slow for the main thread?
  // also what order will it be in?
  _.each(unicode_names_map, function(character, name) {
    unicode_names_string += (name + '\n');
  });
});

// For recognizing the user saying "U+03C1",
// the recognizer isn't very good at it.
// I could match "you plus" and then a pause and then
// each individual hex character separated by pauses.
// It's better at "0x..." (but not great) so hmm.
// I wonder if testing more than five possible matches
// would be useful at all here.
// If I have a command to take the preceding text
// and turn it into a Unicode character(s), then
// that could work with copy/pasted u+ info too.
// Also a command to turn unicode into hex or a sidebar
// so you can see what weird things are happening


// TODO make this configurable and/or autodetected:
var num_tab_spaces = 2;
var tab_spaces = ' '.repeat(num_tab_spaces);
function artificially_type(text) {
  var selection = bililiteRange(document.activeElement).bounds('selection');
  if(text == '\n') {
    selection.insertEOL().select();
  } else {
    selection.text(text, 'end').select();
  }
}
var added_commands = {};
var added_literal_commands = {};
// this depends on a patched annyang that exposes registerCommand
// also maybe I should hack around its trimming: done
// also TODO case-fold rather than lowercase
annyang.debug();
annyang.registerCommand({
    exec: function(str) {
      var normalizedstr = str.trim().toLowerCase();
      var matchfunc = added_literal_commands[normalizedstr];
      return (matchfunc ? [null, normalizedstr, matchfunc] : null);
    }},
    function(normalizedstr, matchfunc) {
      matchfunc(normalizedstr);
    },
    '(literal match)'
    );
function showCodePoint(codepointNumber) {
  var s = codepointNumber.toString(16).toUpperCase();
  if(s.length < 4) {
    s = '0'.repeat(4 - s.length) + s;
  }
  return 'U+' + s;
}
function search_for_unicode_characters(searched) {
      console.log("Trying to find unicode matches for: " + searched);
      var words = searched.split(/[- \t]+/);
      var searcher = new RegExp('^' +
        _.map(words, function(word) { return '(?=.*'+word+')'; }).join('') +
        '.*$',
        'gm');
      //console.log(searcher.source);
      // TODO sorted by codepoint maybe??
      // also what about speech recognition multiple results hm.
      var max_to_find = 35;
      // There are some intentional duplicate name keys ("-" replaced with
      // spaces for example), so skip the duplicated ones. TODO show
      // the user the *canonical* name (with-dash if duplicates, for example).
      var found_chars = {};
      for(var num_found = 0; num_found < max_to_find;) {
        var match = searcher.exec(unicode_names_string);
        if(!match) { break; }
        var name = match[0];
        var character = unicode_names_map[name];
        if(found_chars[character]) { continue; }
        found_chars[character] = true;
        var codepoint = character.codePointAt(0);
        // When HTMLing this, use <bdi> around the character, IIRC
        console.log("Possible match: " + character + ' ' + showCodePoint(codepoint) + ' ' + name);
        num_found++;
      }
      if(num_found === max_to_find) {
        console.log("And more unshown matches...");
      }
      return num_found !== 0;
}
annyang.registerCommand({
  exec: function(str) {
    var match = /^unicode (.*)$/i.exec(str);
    if(!match) { return false; }
    var possible_name = match[1].toUpperCase();
    if(!_.has(unicode_names_map, possible_name)) {
       var any_found = search_for_unicode_characters(possible_name);
       return (any_found ? [null, possible_name, null] : null);
    }
    return [null, possible_name, unicode_names_map[possible_name]];
  }},
  function(name, character) {
    if(character != null) {
      artificially_type(character);
    }
  },
  'unicode <unicode-character-name>');
      
function add_command(regex_or_str, fn) {
  var name = regex_or_str;
{'fraction_non_alphanumeric': 0.16278260869565217, 'fraction_numerical': 0.00591304347826087, 'mean_word_length': 3.408202376389421, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4018764', 'n_tokens_mistral': 3887, 'n_tokens_neox': 3762, 'n_words': 994}