<commit_before>#!/usr/bin/env ruby
require './printarray.rb'

# Convert initial value of CABAC context.
#    m, n are to be derived by:
#        m = (elem >> 4) * 5 - 45
#        n = ((elem & 15) << 3) - 16


INITIAL_VALUE =
  [
   [
    [153], [200], [139, 141, 157], [154], [0, 0, 0], [0],
    [184, 0, 0, 0], [184], [63], [0],
    [0], [0], [0, 0, 0, 0, 0], [0, 0], [0],
    [153, 138, 138], [111, 141], [94, 138, 182, 154],
    [0, 0], [154, 154],
    [139, 139],
    [110, 110, 124, 125, 140, 153, 125, 127, 140, 109, 111, 143, 127, 111, 79, 108, 123, 63],
    [110, 110, 124, 125, 140, 153, 125, 127, 140, 109, 111, 143, 127, 111, 79, 108, 123, 63],
    [91, 171, 134, 141],
    [111, 111, 125, 110, 110, 94, 124, 108, 124, 107, 125, 141, 179, 153, 125, 107,
     125, 141, 179, 153, 125, 107, 125, 141, 179, 153, 125, 140, 139, 182, 182, 152,
     136, 152, 136, 153, 136, 139, 111, 136, 139, 111],
    [140, 92, 137, 138, 140, 152, 138, 139, 153, 74, 149, 92, 139, 107, 122, 152,
     140, 179, 166, 182, 140, 227, 122, 197],
    [138, 153, 136, 167, 152, 152]
   ],
   [
    [153], [185], [107, 139, 126], [154], [197, 185, 201], [149],
    [154, 139, 154, 154], [154], [152], [79],
    [110], [122], [95, 79, 63, 31, 31], [153, 153], [168],
    [124, 138, 94], [153, 111], [149, 107, 167, 154],
    [140, 198], [154, 154],
    [139, 139],
    [125, 110, 94, 110, 95, 79, 125, 111, 110, 78, 110, 111, 111, 95, 94, 108, 123, 108],
    [125, 110, 94, 110, 95, 79, 125, 111, 110, 78, 110, 111, 111, 95, 94, 108, 123, 108],
    [121, 140, 61, 154],
    [155, 154, 139, 153, 139, 123, 123, 63, 153, 166, 183, 140, 136, 153, 154, 166,
     183, 140, 136, 153, 154, 166, 183, 140, 136, 153, 154, 170, 153, 123, 123, 107,
     121, 107, 121, 167, 151, 183, 140, 151, 183, 140],
    [154, 196, 196, 167, 154, 152, 167, 182, 182, 134, 149, 136, 153, 121, 136, 137,
     169, 194, 166, 167, 154, 167, 137, 182],
    [107, 167, 91, 122, 107, 167]
   ]
  ]

class MnArray < PrintableArray
  def elem(v)
    sprintf("{%d, %d}", v[0], v[1])
  end<commit_msg>Update LUT generator.<commit_after>#!/usr/bin/env ruby
require './printarray.rb'

# Convert initial value of CABAC context.
#    m, n are to be derived by:
#        m = (elem >> 4) * 5 - 45
#        n = ((elem & 15) << 3) - 16


INITIAL_VALUE =
  [
   [
    [153], [200], [139, 141, 157], [154], [0, 0, 0], [0],
    [184, 0, 0, 0], [184], [63], [0],
    [0], [0], [0, 0, 0, 0, 0], [0, 0], [0],
    [153, 138, 138], [111, 141], [94, 138, 182, 154],
    [0, 0], [154, 154],
    [139, 139],
    [110, 110, 124, 125, 140, 153, 125, 127, 140, 109, 111, 143, 127, 111, 79, 108, 123, 63],
    [110, 110, 124, 125, 140, 153, 125, 127, 140, 109, 111, 143, 127, 111, 79, 108, 123, 63],
    [91, 171, 134, 141],
    [111, 111, 125, 110, 110, 94, 124, 108, 124, 107, 125, 141, 179, 153, 125, 107,
     125, 141, 179, 153, 125, 107, 125, 141, 179, 153, 125, 140, 139, 182, 182, 152,
     136, 152, 136, 153, 136, 139, 111, 136, 139, 111],
    [140, 92, 137, 138, 140, 152, 138, 139, 153, 74, 149, 92, 139, 107, 122, 152,
     140, 179, 166, 182, 140, 227, 122, 197],
    [138, 153, 136, 167, 152, 152]
   ],
   [
    [153], [185], [107, 139, 126], [154], [197, 185, 201], [149],
    [154, 139, 154, 154], [154], [152], [79],
    [110], [122], [95, 79, 63, 31, 31], [153, 153], [168],
    [124, 138, 94], [153, 111], [149, 107, 167, 154],
    [140, 198], [154, 154],
    [139, 139],
    [125, 110, 94, 110, 95, 79, 125, 111, 110, 78, 110, 111, 111, 95, 94, 108, 123, 108],
    [125, 110, 94, 110, 95, 79, 125, 111, 110, 78, 110, 111, 111, 95, 94, 108, 123, 108],
    [121, 140, 61, 154],
    [155, 154, 139, 153, 139, 123, 123, 63, 153, 166, 183, 140, 136, 153, 154, 166,
     183, 140, 136, 153, 154, 166, 183, 140, 136, 153, 154, 170, 153, 123, 123, 107,
     121, 107, 121, 167, 151, 183, 140, 151, 183, 140],
    [154, 196, 196, 167, 154, 152, 167, 182, 182, 134, 149, 136, 153, 121, 136, 137,
     169, 194, 166, 167, 154, 167, 137, 182],
    [107, 167, 91, 122, 107, 167]
   ],
   [
    [153], [160], [107, 139, 126], [154], [197, 185, 201], [134],
    [154, 139, 154, 154], [183], [152], [79],
    [154], [137], [95, 79, 63, 31, 31], [153, 153], [168],
    [224, 167, 122], [153, 111], [149, 92, 167, 154],
    [169, 198], [154, 154],
    [139, 139],
    [125, 110, 124, 110, 95, 94, 125, 111, 111, 79, 125, 126, 111, 111, 79, 108, 123, 93],
    [125, 110, 124, 110, 95, 94, 125, 111, 111, 79, 125, 126, 111, 111, 79, 108, 123, 93],
    [121, 140, 61, 154],
    [170, 154, 139, 153, 139, 123, 123, 63, 124, 166, 183, 140, 136, 153, 154, 166,
     183, 140, 136, 153, 154, 166, 183, 140, 136, 153, 154, 170, 153, 138, 138, 122,
     121, 122, 121, 167, 151, 183, 140, 151, 183, 140],
    [154, 196, 167, 167, 154, 152, 167, 182, 182, 134, 149, 136, 153, 121, 136, 122,
     169, 208, 166, 167, 154, 152, 167, 182],
    [107, 167, 91, 107, 107, 167]
   ]
  ]

class MnArray < PrintableArray
  def elem(v)
    sprintf("{%d, %d}", v[0], v[1])
  end
{'fraction_non_alphanumeric': 0.2331128482661856, 'fraction_numerical': 0.43415514131088395, 'mean_word_length': 2.78316906747536, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '768457', 'n_tokens_mistral': 4135, 'n_tokens_neox': 2112, 'n_words': 847}