<commit_before>"use strict";

function script_to_narrative(script, options) {
  var options = options || {};
  var too_many = options.too_many || 5;
  var fixed_characters = options.fixed_characters || false;
  var num_frequent = options.num_frequent || 5;

  var char_ids = {};
  var next_id = 0;
  var char_to_id = function(name) {
    if (char_ids[name] == null) {
      char_ids[name] = next_id;
      next_id += 1;
    }
    return char_ids[name];
  };

  var locations = get_locations(script);
  var frequent = options.characters || most_frequent_characters(locations, num_frequent);

  var pos = 0;
  var scenes = [];
  locations.forEach(function(loc) {
    var scene = {
      duration: 2,
      start: pos,
      chars: [...new Set(get_characters(loc))]
        .map(char_to_id).sort()
    };

    if (fixed_characters || scene.chars.length > too_many) {
      scene.chars = scene.chars.filter(function(c) {
        return frequent.has(c);
      });
    }
    scene.chars = [...new Set(scene.chars).keys()].sort();
    pos += 1

    if (scene.chars.length > 0) {
      scenes.push(scene);
    }
  });

  return {
    narrative: scenes,
    chars: char_ids,
  }
}

function get_json(url, cb) {
  var xhr = new XMLHttpRequest()
  xhr.open("GET", url)
  xhr.send()
  xhr.onload = function() { cb(JSON.parse(xhr.responseText)); };
  return xhr;
}
<commit_msg>"Fix" the character selection<commit_after>"use strict";

function script_to_narrative(script, options) {
  var options = options || {};
  var too_many = options.too_many || 5;
  var fixed_characters = options.fixed_characters || false;
  var num_frequent = options.num_frequent || 5;

  var char_ids = {};
  var next_id = 0;
  var char_to_id = function(name) {
    if (char_ids[name] == null) {
      char_ids[name] = next_id;
      next_id += 1;
    }
    return char_ids[name];
  };

  var locations = get_locations(script);
  var frequent = options.characters || most_frequent_characters(locations, num_frequent);

  var pos = 0;
  var scenes = [];
  locations.forEach(function(loc) {
    get_characters(loc).map(char_to_id);
    var scene = {
      duration: 2,
      start: pos,
      chars: [...new Set(get_characters(loc))].sort()
    };

    if (fixed_characters || scene.chars.length > too_many) {
      scene.chars = scene.chars.filter(function(c) {
        return frequent.has(c);
      });
    }
    scene.chars = [...new Set(scene.chars).keys()].map(char_to_id).sort();
    pos += 1;

    if (scene.chars.length > 0) {
      scenes.push(scene);
    }
  });

  return {
    narrative: scenes,
    chars: char_ids,
  }
}

function get_json(url, cb) {
  var xhr = new XMLHttpRequest()
  xhr.open("GET", url)
  xhr.send()
  xhr.onload = function() { cb(JSON.parse(xhr.responseText)); };
  return xhr;
}

{'fraction_non_alphanumeric': 0.15016204537270436, 'fraction_numerical': 0.005761613251710479, 'mean_word_length': 2.985652797704448, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4979181', 'n_tokens_mistral': 1014, 'n_tokens_neox': 979, 'n_words': 199}