<commit_before>#!/usr/bin/env python

########################################################
#
# fregion.py : read structured data files
#
#    to load a file from the path P into the variable f:
#      f = fregion.FRegion(P)
#
#    to read the stored field 'x' out of f:
#      f.x
#
#    to read the 'metadata' for the field 'x' (type and offset details):
#      meta(f).x
#
#    to add reading support for a custom user type:
#      fregion.FRegion.addType("MyTypeName", lambda renv, td: makeMyTypeNameReader(renv,td))
#    where:
#      'renv' will be the "reader environment" (necessary for any call to makeReader)
#      'td'   will be the full type description where "MyTypeName" appears at root (e.g. for 'fileref', App(Prim('fileref', ...), [Arr(Prim('char'))]))
#    and:
#      the returned 'reader' must be a class with a "read" function like:
#        def read(m,v):
#      where
#        'm' will give access to the memory for the file being read out of
#        'v' will be the "representation value" (as specified in the type description where "MyTypeName" appears)
#      and:
#        the returned value may be whatever the application decides is sensible
#
########################################################

import os
import mmap
import struct
import math
import datetime

#######
#<commit_msg>update comment header to better reflect API usage for adding custom user-defined types<commit_after>#!/usr/bin/env python

########################################################
#
# fregion.py : read structured data files
#
#    to load a file from the path P into the variable f:
#      f = fregion.FRegion(P)
#
#    to read the stored field 'x' out of f:
#      f.x
#
#    to read the 'metadata' for the field 'x' (type and offset details):
#      meta(f).x
#
#    to add reading support for a custom user type:
#      fregion.FRegion.addType("MyTypeName", lambda renv, td, repty: makeMyTypeNameReader(renv,td,repty))
#    where:
#      'renv'  will be the "reader environment" (necessary for any call to makeReader)
#      'td'    will be the full type description where "MyTypeName" appears at root (e.g. for 'fileref', App(Prim('fileref', ...), [Arr(Prim('char'))]))
#      'repty' will be the determined 'representation type' (which can also be determined through 'td')
#    and:
#      the returned 'reader' must be a class with a "read" function like:
#        def read(self,m,o):
#      where
#        'm' will give access to the memory for the file being read out of
#        'o' will be the memory offset where the value to be read is placed
#      and:
#        the returned value may be whatever the application decides is sensible
#
########################################################

import os
import mmap
import struct
import math
import datetime

#######
#
{'fraction_non_alphanumeric': 0.17802354620049946, 'fraction_numerical': 0.0, 'mean_word_length': 3.450793650793651, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2596725', 'n_tokens_mistral': 808, 'n_tokens_neox': 791, 'n_words': 336}