<commit_before># -*- coding: utf-8 -*-
# FIXME: implement more specialized operational pattern parser.

""" MXF Parser. """

from mxf.common import InterchangeObject
from mxf.s377m import MXFPartition, MXFDataSet, MXFPreface, MXFPrimer, KLVFill, KLVDarkComponent, RandomIndexMetadata, S377MException
from mxf.avid import AvidObjectDirectory, AvidAAFDefinition, AvidMetadataPreface, AvidMXFDataSet

SMPTE_PARTITION_PACK_LABEL = '060e2b34020501010d010201'

def mxf_kind(filename):
    """ Lookup the MXF data start position and returns appropriate parser. """

    mxf = MXFParser(filename)
    mxf.open()

    # SMTPE 377M: Header Partition Pack, first thing in a MXF file
    header_partition_pack = MXFPartition(mxf.fd)
    try:
        header_partition_pack.read()
    except S377MException, error:
        print error

    print "Selecting", str(PARSERS[header_partition_pack.data['operational_pattern'].encode('hex_codec')])

    return PARSERS[header_partition_pack.data['operational_pattern'].encode('hex_codec')](filename)


class MXFParser(object):

    def __init__(self, filename, debug=False):
        self.filename = filename
        self.fd = None
        self.data = None
        self.debug = debug

    def open(self):
        # SMTPE 377M: ability to skip over RunIn sequence
        self.fd = open(self.filename, 'r')
        data = self.fd.read(65536)
        idx = data.find(SMPTE_PARTITION_PACK_LABEL.decode('hex_codec'))
        if idx == -1:
            raise Exception('Not a valid SMTPE 377m MXF file.')

        # Real MXF data position
        self.fd.seek(idx)

    def close(self):
        self.fd.close()


class AvidParser(MXFParser):

    def read(self):
        if not self.fd:
            self.open()

        # SMTPE 377M: Header Partition Pack, first thing in a MXF file
        header_partition_pack = MXFPartition(self.fd)
        header_partition_pack.read()

        header_klvs = []
        header_klvs_hash = {}
        header_end = self.fd.tell() + header_partition_pack.data['header_byte_count']

        header_metadata_preface = None
        avid_metadata_preface = None
        dark = 0

        while self.fd.tell() <= header_end:<commit_msg>Fix header parsing (header_byte_count does not include optional KLVFill)<commit_after># -*- coding: utf-8 -*-
# FIXME: implement more specialized operational pattern parser.

""" MXF Parser. """

from mxf.common import InterchangeObject
from mxf.s377m import MXFPartition, MXFDataSet, MXFPreface, MXFPrimer, KLVFill, KLVDarkComponent, RandomIndexMetadata, S377MException
from mxf.avid import AvidObjectDirectory, AvidAAFDefinition, AvidMetadataPreface, AvidMXFDataSet

SMPTE_PARTITION_PACK_LABEL = '060e2b34020501010d010201'

def mxf_kind(filename):
    """ Lookup the MXF data start position and returns appropriate parser. """

    mxf = MXFParser(filename)
    mxf.open()

    # SMTPE 377M: Header Partition Pack, first thing in a MXF file
    header_partition_pack = MXFPartition(mxf.fd)
    try:
        header_partition_pack.read()
    except S377MException, error:
        print error

    print "Selecting", str(PARSERS[header_partition_pack.data['operational_pattern'].encode('hex_codec')])

    return PARSERS[header_partition_pack.data['operational_pattern'].encode('hex_codec')](filename)


class MXFParser(object):

    def __init__(self, filename, debug=False):
        self.filename = filename
        self.fd = None
        self.data = None
        self.debug = debug

    def open(self):
        # SMTPE 377M: ability to skip over RunIn sequence
        self.fd = open(self.filename, 'r')
        data = self.fd.read(65536)
        idx = data.find(SMPTE_PARTITION_PACK_LABEL.decode('hex_codec'))
        if idx == -1:
            raise Exception('Not a valid SMTPE 377m MXF file.')

        # Real MXF data position
        self.fd.seek(idx)

    def close(self):
        self.fd.close()


class AvidParser(MXFParser):

    def read(self):
        if not self.fd:
            self.open()

        # SMTPE 377M: Header Partition Pack, first thing in a MXF file
        header_partition_pack = MXFPartition(self.fd)
        header_partition_pack.read()

        header_klvs = []
        header_klvs_hash = {}

        # SMPTE 377M: klv fill behind Header Partition is not counted in HeaderByteCount
        key = InterchangeObject.get_key(self.fd)
        if key in ('<KEY>', \
            '060e2b34010101010301021001000000'):
            # KLV Fill item
            klv = KLVFill(self.fd)
            klv.read()

        header_end = self.fd.tell() + header_partition_pack.data['header_byte_count']

        header_metadata_preface = None
        avid_metadata_preface = None
        dark = 0

        while self.fd.tell() < header_end:
{'fraction_non_alphanumeric': 0.09470575827884413, 'fraction_numerical': 0.028053153343176544, 'mean_word_length': 2.9983136593591904, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491437', 'n_tokens_mistral': 1612, 'n_tokens_neox': 1464, 'n_words': 356}