<commit_before>from collections import namedtuple, OrderedDict
import locale
import os
import re
import struct
import logging
import yaml
import codecs
import yaml.constructor

from array import array
import parsedatetime as pdt
from binascii import hexlify, unhexlify

from .block import TokenType, decode_vli
from .blockchain import BlockChain, BlockChainIter
from .datafield import DataField

from .psqlexporter import PsqlExporter
from .postgresexporter import PostgresExporter


class FP5File(object):
    """Wrapper for FP5 file object"""

    def __init__(self, filename, encoding=None, locale=None):
        super(FP5File, self).__init__()

        self.logging = logging.getLogger('fp5dump.fp5file.fp5file')

        self.filename = filename

        self.locale = locale
        self.encoding = encoding if encoding else 'latin1'
        self.ptd_parser = pdt.Calendar(pdt.Constants())

        self.basename = os.path.splitext(os.path.basename(filename))[0]
        self.dirname = os.path.dirname(os.path.abspath(os.path.expanduser(filename)))
        self.db_name = self.basename

        self.export_definition = None

        self.error_report_columns = []

        self.block_chains = []
        self.block_chain_levels = 0
        self.fields = {}

        self.index = None
        self.data = None

        self.enums = []

        self.file_size = 0

        self.records_index = []
        self.records_count = 0

        self.version_string = ""
        self.filename_string = ""
        self.server_addr_string = ""

        self.block_prev_id_to_block_pos = None
        self.block_id_to_block_pos = None

        self.logging.info("opening %s" % self.basename)
        print(os.path.abspath(os.path.expanduser(self.filename)))

        self.file = open(os.path.abspath(os.path.expanduser(self.filename)), "rb", buffering=0)

        self.largest_block_id = 0x00000000

        self.read_header()
        self.get_blocks()
        self.order_block_indices()
        self.get_field_index()
        self.get_record_index()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.close()

    def close(self):
        self.logging.info("closing %s" % self.basename)
<commit_msg>removes superfluous print statement<commit_after>from collections import namedtuple, OrderedDict
import locale
import os
import re
import struct
import logging
import yaml
import codecs
import yaml.constructor

from array import array
import parsedatetime as pdt
from binascii import hexlify, unhexlify

from .block import TokenType, decode_vli
from .blockchain import BlockChain, BlockChainIter
from .datafield import DataField

from .psqlexporter import PsqlExporter
from .postgresexporter import PostgresExporter


class FP5File(object):
    """Wrapper for FP5 file object"""

    def __init__(self, filename, encoding=None, locale=None):
        super(FP5File, self).__init__()

        self.logging = logging.getLogger('fp5dump.fp5file.fp5file')

        self.filename = filename

        self.locale = locale
        self.encoding = encoding if encoding else 'latin1'
        self.ptd_parser = pdt.Calendar(pdt.Constants())

        self.basename = os.path.splitext(os.path.basename(filename))[0]
        self.dirname = os.path.dirname(os.path.abspath(os.path.expanduser(filename)))
        self.db_name = self.basename

        self.export_definition = None

        self.error_report_columns = []

        self.block_chains = []
        self.block_chain_levels = 0
        self.fields = {}

        self.index = None
        self.data = None

        self.enums = []

        self.file_size = 0

        self.records_index = []
        self.records_count = 0

        self.version_string = ""
        self.filename_string = ""
        self.server_addr_string = ""

        self.block_prev_id_to_block_pos = None
        self.block_id_to_block_pos = None

        self.logging.info("opening %s" % self.basename)

        self.file = open(os.path.abspath(os.path.expanduser(self.filename)), "rb", buffering=0)

        self.largest_block_id = 0x00000000

        self.read_header()
        self.get_blocks()
        self.order_block_indices()
        self.get_field_index()
        self.get_record_index()

    def __enter__(self):
        return self

    def __exit__(self, type, value, tb):
        self.close()

    def close(self):
        self.logging.info("closing %s" % self.basename)

{'fraction_non_alphanumeric': 0.09522737505628096, 'fraction_numerical': 0.00945520036019811, 'mean_word_length': 2.7780612244897958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5494105', 'n_tokens_mistral': 1408, 'n_tokens_neox': 1321, 'n_words': 290}