<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Generated by ngram_merge on 2011-01-21 10:50:46 from:
// score_me file /export/hda3/cld/pre2010/b0_samp_prune_20100722.utf8
// cld_generated_quad.bin, built 20110121 bytes 4443812 hash 462c-16c4
// cld_generated_deltaocta.bin, built 20110121 bytes 1053284 hash c834-81f5

// 1 # dsites Added text for
// ak haw ig kha ks mfe mo nd nso ny ve
// bs-Cyrl/Latn hr-Latn sr-Cyrl/Latn sr-ME-Latn


namespace CLD2 {

// Average score per 1024 bytes
extern const short kAvgDeltaOctaScore[614 * 4] = {
// Latn  Cyrl  Arab  Other script
// Updated 20140204 for CLD2 full
  1163,    0,    0,    0,  // 0 ENGLISH en
   983,    0,    0,    0,  // 1 DANISH da
  1036,    0,    0,    0,  // 2 DUTCH nl
  1245,    0,    0,    0,  // 3 FINNISH fi
   873,    0,    0,    0,  // 4 FRENCH fr
  1146,    0,    0,    0,  // 5 GERMAN de
     0,    0,    0,  901,  // 6 HEBREW iw
   736,    0,    0,    0,  // 7 ITALIAN it
     0,    0,    0, 3100,  // 8 Japanese ja
     0,    0,    0, 3669,  // 9 Korean ko
   836,    0,    0,    0,  // 10 NORWEGIAN no
  1372,    0,    0,    0,  // 11 POLISH pl
  1044,    0,    0,    0,  // 12 PORTUGUESE pt
     0,  648,    0,    0,  // 13 RUSSIAN ru
   640,    0,    0,    0,  // 14 SPANISH es
   866,    0,    0,    0,  // 15 SWEDISH sv
     0,    0,    0, 1928,  // 16 Chinese zh
  1417,    0,    0,    0,  // 17 CZECH cs
     0,    0,    0, 1024,  // 18 GREEK el
  1212,    0,    0,    0,  // 19 ICELANDIC is
  1295,    0,    0,    0,  // 20 LATVIAN lv
  1169,    0,    0,    0,  // 21 LITHUANIAN lt
   898,  894,    0,    0,  // 22 ROMANIAN ro
  1370,    0,    0,    0,  // 23 HUNGARIAN hu
  1114,    0,    0,    0,  // 24 ESTONIAN et
     0,    0,    0,    0,  // 25 Ignore xxx
     0,    0,    0,    0,  // 26 Unknown un
     0,  684,    0,    0,  // 27 BULGARIAN bg
   575,    0,    0,    0,  // 28 CROATIAN hr<commit_msg>Add named constant for array size<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Generated by ngram_merge on 2011-01-21 10:50:46 from:
// score_me file /export/hda3/cld/pre2010/b0_samp_prune_20100722.utf8
// cld_generated_quad.bin, built 20110121 bytes 4443812 hash 462c-16c4
// cld_generated_deltaocta.bin, built 20110121 bytes 1053284 hash c834-81f5

// 1 # dsites Added text for
// ak haw ig kha ks mfe mo nd nso ny ve
// bs-Cyrl/Latn hr-Latn sr-Cyrl/Latn sr-ME-Latn


namespace CLD2 {

// Average score per 1024 bytes
static const int kAvgDeltaOctaScoreSize = 614 * 4;
extern const short kAvgDeltaOctaScore[kAvgDeltaOctaScoreSize] = {
// Latn  Cyrl  Arab  Other script
// Updated 20140204 for CLD2 full
  1163,    0,    0,    0,  // 0 ENGLISH en
   983,    0,    0,    0,  // 1 DANISH da
  1036,    0,    0,    0,  // 2 DUTCH nl
  1245,    0,    0,    0,  // 3 FINNISH fi
   873,    0,    0,    0,  // 4 FRENCH fr
  1146,    0,    0,    0,  // 5 GERMAN de
     0,    0,    0,  901,  // 6 HEBREW iw
   736,    0,    0,    0,  // 7 ITALIAN it
     0,    0,    0, 3100,  // 8 Japanese ja
     0,    0,    0, 3669,  // 9 Korean ko
   836,    0,    0,    0,  // 10 NORWEGIAN no
  1372,    0,    0,    0,  // 11 POLISH pl
  1044,    0,    0,    0,  // 12 PORTUGUESE pt
     0,  648,    0,    0,  // 13 RUSSIAN ru
   640,    0,    0,    0,  // 14 SPANISH es
   866,    0,    0,    0,  // 15 SWEDISH sv
     0,    0,    0, 1928,  // 16 Chinese zh
  1417,    0,    0,    0,  // 17 CZECH cs
     0,    0,    0, 1024,  // 18 GREEK el
  1212,    0,    0,    0,  // 19 ICELANDIC is
  1295,    0,    0,    0,  // 20 LATVIAN lv
  1169,    0,    0,    0,  // 21 LITHUANIAN lt
   898,  894,    0,    0,  // 22 ROMANIAN ro
  1370,    0,    0,    0,  // 23 HUNGARIAN hu
  1114,    0,    0,    0,  // 24 ESTONIAN et
     0,    0,    0,    0,  // 25 Ignore xxx
     0,    0,    0,    0,  // 26 Unknown un
     0,  684,    0,    0,  // 27 BULGARIAN bg
   575,    0,    0,    0,  // 28 CROATIAN hr
{'fraction_non_alphanumeric': 0.11177406523468576, 'fraction_numerical': 0.1328560063643596, 'mean_word_length': 1.7095905172413792, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '925853', 'n_tokens_mistral': 2367, 'n_tokens_neox': 1965, 'n_words': 719}