<commit_before>enablecompression false
usev8geopositions false
documenttype[0].id -1368624373
documenttype[0].name "other_doc"
documenttype[0].version 0
documenttype[0].headerstruct 1631005140
documenttype[0].bodystruct 0
documenttype[0].inherits[0].id 8
documenttype[0].datatype[0].id 1631005140
documenttype[0].datatype[0].type STRUCT
documenttype[0].datatype[0].array.element.id 0
documenttype[0].datatype[0].map.key.id 0
documenttype[0].datatype[0].map.value.id 0
documenttype[0].datatype[0].wset.key.id 0
documenttype[0].datatype[0].wset.createifnonexistent false
documenttype[0].datatype[0].wset.removeifzero false
documenttype[0].datatype[0].annotationref.annotation.id 0
documenttype[0].datatype[0].sstruct.name "other_doc.header"
documenttype[0].datatype[0].sstruct.version 0
documenttype[0].datatype[0].sstruct.compression.type NONE
documenttype[0].datatype[0].sstruct.compression.level 0
documenttype[0].datatype[0].sstruct.compression.threshold 95
documenttype[0].datatype[0].sstruct.compression.minsize 200
documenttype[1].id -853072901
documenttype[1].name "types"
documenttype[1].version 0
documenttype[1].headerstruct 1328581348
documenttype[1].bodystruct 0
documenttype[1].inherits[0].id 8
documenttype[1].datatype[0].id -1368624373
documenttype[1].datatype[0].type STRUCT
documenttype[1].datatype[0].array.element.id 0
documenttype[1].datatype[0].map.key.id 0
documenttype[1].datatype[0].map.value.id 0
documenttype[1].datatype[0].wset.key.id 0
documenttype[1].datatype[0].wset.createifnonexistent false
documenttype[1].datatype[0].wset.removeifzero false
documenttype[1].datatype[0].annotationref.annotation.id 0
documenttype[1].datatype[0].sstruct.name "other_doc"
documenttype[1].datatype[0].sstruct.version 0
documenttype[1].datatype[0].sstruct.compression.type NONE
documenttype[1].datatype[0].sstruct.compression.level 0
documenttype[1].datatype[0].sstruct.compression.threshold 95
documenttype[1].datatype[0].sstruct.compression.minsize 200
documenttype[1].datatype[1].id 1328581348
documenttype[1].datatype[1].type STRUCT
documenttype[1].datatype[1].array.element.id 0
documenttype[1].datatype[1].map.key.id 0
documenttype[1].datatype[1].map.value.id 0
documenttype[1].datatype[1].wset.key.id 0
documenttype[1].datatype[1].wset.createifnonexistent false
documenttype[1].datatype[1].wset.removeifzero false
documenttype[1].datatype[1].annotationref.annotation.id 0
documenttype[1].datatype[1].sstruct.name "types.header"
documenttype[1].datatype[1].sstruct.version 0
documenttype[1].datatype[1].sstruct.compression.type NONE
documenttype[1].datatype[1].sstruct.compression.level 0
documenttype[1].datatype[1].sstruct.compression.threshold 95
documenttype[1].datatype[1].sstruct.compression.minsize 200
documenttype[1].datatype[1].sstruct.field[0].name "doc_field"
documenttype[1].datatype[1].sstruct.field[0].id 819293364
documenttype[1].datatype[1].sstruct.field[0].datatype -1368624373
documenttype[1].datatype[1].sstruct.field[0].detailedtype ""
documenttype[1].fieldsets{[document]}.fields[0] "doc_field"
<commit_msg>this looks more correct now<commit_after>enablecompression false
usev8geopositions false
documenttype[0].id -1368624373
documenttype[0].name "other_doc"
documenttype[0].version 0
documenttype[0].headerstruct 1631005140
documenttype[0].bodystruct 0
documenttype[0].inherits[0].id 8
documenttype[0].datatype[0].id 1631005140
documenttype[0].datatype[0].type STRUCT
documenttype[0].datatype[0].array.element.id 0
documenttype[0].datatype[0].map.key.id 0
documenttype[0].datatype[0].map.value.id 0
documenttype[0].datatype[0].wset.key.id 0
documenttype[0].datatype[0].wset.createifnonexistent false
documenttype[0].datatype[0].wset.removeifzero false
documenttype[0].datatype[0].annotationref.annotation.id 0
documenttype[0].datatype[0].sstruct.name "other_doc.header"
documenttype[0].datatype[0].sstruct.version 0
documenttype[0].datatype[0].sstruct.compression.type NONE
documenttype[0].datatype[0].sstruct.compression.level 0
documenttype[0].datatype[0].sstruct.compression.threshold 95
documenttype[0].datatype[0].sstruct.compression.minsize 200
documenttype[1].id -853072901
documenttype[1].name "types"
documenttype[1].version 0
documenttype[1].headerstruct 1328581348
documenttype[1].bodystruct 0
documenttype[1].inherits[0].id 8
documenttype[1].datatype[0].id 1328581348
documenttype[1].datatype[0].type STRUCT
documenttype[1].datatype[0].array.element.id 0
documenttype[1].datatype[0].map.key.id 0
documenttype[1].datatype[0].map.value.id 0
documenttype[1].datatype[0].wset.key.id 0
documenttype[1].datatype[0].wset.createifnonexistent false
documenttype[1].datatype[0].wset.removeifzero false
documenttype[1].datatype[0].annotationref.annotation.id 0
documenttype[1].datatype[0].sstruct.name "types.header"
documenttype[1].datatype[0].sstruct.version 0
documenttype[1].datatype[0].sstruct.compression.type NONE
documenttype[1].datatype[0].sstruct.compression.level 0
documenttype[1].datatype[0].sstruct.compression.threshold 95
documenttype[1].datatype[0].sstruct.compression.minsize 200
documenttype[1].datatype[0].sstruct.field[0].name "doc_field"
documenttype[1].datatype[0].sstruct.field[0].id 819293364
documenttype[1].datatype[0].sstruct.field[0].datatype -1368624373
documenttype[1].datatype[0].sstruct.field[0].detailedtype ""
documenttype[1].fieldsets{[document]}.fields[0] "doc_field"

{'fraction_non_alphanumeric': 0.14793342060968767, 'fraction_numerical': 0.08341125864971012, 'mean_word_length': 8.448763250883392, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6722108', 'n_tokens_mistral': 2089, 'n_tokens_neox': 1896, 'n_words': 228}