<commit_before># encoding: UTF-8
pattern = /\$\$(\w[\w: ]*\w)\$\$/
DocuBot.handle_snippet pattern do |match, page|
  parts = match[ 2..-3 ].split(':',2)
  text, term = parts.first, parts.last
  if page.bundle.glossary[ term ]
  	"<span class='glossary'>#{text}</span>"
  else
    page.bundle.glossary.add_missing_term( term )
  	"<span class='missing'>#{text}</span>"
  end
end<commit_msg>Glossary entries record term for lookup<commit_after># encoding: UTF-8
pattern = /\$\$(\w[\w: ]*\w)\$\$/
DocuBot.handle_snippet pattern do |match, page|
  parts = match[ 2..-3 ].split(':',2)
  text, term = parts.first, parts.last
  if page.bundle.glossary[ term ]
  	"<span class='glossary' term='#{term}'>#{text}</span>"
  else
    page.bundle.glossary.add_missing_term( term )
  	"<span class='missing'>#{text}</span>"
  end
end
{'fraction_non_alphanumeric': 0.21001221001221002, 'fraction_numerical': 0.009768009768009768, 'mean_word_length': 3.9696969696969697, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '905988', 'n_tokens_mistral': 320, 'n_tokens_neox': 316, 'n_words': 67}