<commit_before><%
user_agent = UserAgent.parse(request.user_agent)

$id_vars[:os] = user_agent.os
$id_vars[:platform] = user_agent.platform
$id_vars[:browser] = user_agent.browser
$id_vars[:version] = user_agent.version
$id_vars[:build] = user_agent.build
$id_vars[:localization] = user_agent.localization

id_str = String.new

$id_vars.each do |key, val|
  val = 'nil' if val.nil?
  val = 'false' if val == false
  val = 'true' if val == true
  id_str << val
end

id_md5 = Digest::MD5.hexdigest(id_str)
$redis.incr(id_md5)
%>
<table class="table table-bordered table-condensed">
  <tr><td>ID String</td><td class="text-center"><%=id_str%></td></tr>
  <tr><td>ID Fingerprint</td><td class="text-center"><%=id_md5%></td></tr>
  <tr><td>ID Uniqueness</td><td class="text-center"><%=$redis.get(id_md5)%></td></tr>
</table><commit_msg>Keep count of total unique fingerprints<commit_after><%
user_agent = UserAgent.parse(request.user_agent)

$id_vars[:os] = user_agent.os
$id_vars[:platform] = user_agent.platform
$id_vars[:browser] = user_agent.browser
$id_vars[:version] = user_agent.version
$id_vars[:build] = user_agent.build
$id_vars[:localization] = user_agent.localization

id_str = String.new

$id_vars.each do |key, val|
  val = 'nil' if val.nil?
  val = 'false' if val == false
  val = 'true' if val == true
  id_str << "#{key}:#{val}, "
end

id_md5 = Digest::MD5.hexdigest(id_str)

$redis.incr(id_md5)
id_match = $redis.get(id_md5)
id_total = $redis.keys.count
%>
<table class="table table-bordered table-condensed">
  <tr><td>ID String</td><td class="text-center"><%=id_str%></td></tr>
  <tr><td>ID Fingerprint</td><td class="text-center"><%=id_md5%></td></tr>
  <tr><td>ID Uniqueness</td><td class="text-center"><%="#{id_match} / #{id_total}"%></td></tr>
</table>
{'fraction_non_alphanumeric': 0.20327498588368154, 'fraction_numerical': 0.00564652738565782, 'mean_word_length': 6.003952569169961, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3665691', 'n_tokens_mistral': 715, 'n_tokens_neox': 708, 'n_words': 119}