<commit_before>var tagIndex = {
  totalDocs: 0,
  termDocCounts: {},
  termTags: {},
  tagTagMap: {},
  tagCounts: {},
  getPageTokens: function (page) {
    var urlChunk = ''
    try {
      urlChunk = new URL(page.url).hostname
    } catch (e) { }
    var tokens = tokenize(page.title + ' ' + urlChunk.split(/\W/g).join(' '))

    var generic = ['www', 'com', 'net', 'html', 'pdf', 'file']
    tokens = tokens.filter(t => t.length > 2 && !generic.includes(t))

    return tokens
  },
  addPage: function (page) {
    if (page.tags.length === 0) {
      return
    }

    tagIndex.totalDocs++

    var tokens = tagIndex.getPageTokens(page)

    tokens.filter((t, i) => tokens.indexOf(t) === i).forEach(function (token) {
      if (!tagIndex.termDocCounts[token]) {
        tagIndex.termDocCounts[token] = 1
      } else {
        tagIndex.termDocCounts[token]++
      }
    })
<commit_msg>a couple minor improvements for tagIndex<commit_after>var tagIndex = {
  totalDocs: 0,
  termDocCounts: {},
  termTags: {},
  tagTagMap: {},
  tagCounts: {},
  getPageTokens: function (page) {
    var urlChunk = ''
    try {
      // ignore the TLD, since it doesn't predict tags very well
      urlChunk = new URL(page.url).hostname.split('.').slice(0, -1).join('.')
    } catch (e) { }
    var tokens = tokenize(page.title + ' ' + urlChunk)

    var generic = ['www', 'com', 'net', 'html', 'pdf', 'file']
    tokens = tokens.filter(t => t.length > 2 && !generic.includes(t))

    return tokens
  },
  addPage: function (page) {
    if (page.tags.length === 0) {
      return
    }

    tagIndex.totalDocs++

    var tokens = tagIndex.getPageTokens(page)

    tokens.filter((t, i) => tokens.indexOf(t) === i).forEach(function (token) {
      if (!tagIndex.termDocCounts[token]) {
        tagIndex.termDocCounts[token] = 1
      } else {
        tagIndex.termDocCounts[token]++
      }
    })

{'fraction_non_alphanumeric': 0.17100371747211895, 'fraction_numerical': 0.0053106744556558685, 'mean_word_length': 2.7233201581027666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5285864', 'n_tokens_mistral': 662, 'n_tokens_neox': 649, 'n_words': 136}