# Path: appAux.py
# def loadFile(file_fullpath, max_chunk_size = 0):
#     """Abstracts loading a regular file and a file from within a zip archive.
#     Args:
#         file_fullpath (str): Full path to file to load
#         max_chunk_size (int): maximum number of bytes to be read from the file
#     Returns:
#         file_pointer (StringIO): Data read from fileFullPath
#     """
#     global zipCache
#     global zipCache_hits
#     global zipCache_misses
#     logger.debug("Loading file %s" % file_fullpath)
#     if ".zip/" in file_fullpath:
#         m = re.match(r'^((?:.*)\.zip)[\\/](.*)$', file_fullpath)
#         if m:
#             zip_container = m.group(1)
#             if m.lastindex > 1:
#                 file_fullpath_unprocessed = m.group(2)
#                 if '.zip' in file_fullpath_unprocessed:
#                     m = re.match(r'^(.*)(?:\.zip[\\/](.*))*$', file_fullpath_unprocessed)
#                     if m:
#                         zip_container_relative_path = m.group(1)
#                         if m.lastindex > 1:
#                             file_fullpath_unprocessed = m.group(2)
#                         else:
#                             file_fullpath_unprocessed = None
#                     else:
#                         logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
#                 else:
#                     zip_container_relative_path = file_fullpath_unprocessed
# 
#                 # If not in the zipCache we add it
#             if zip_container not in zipCache:
#                 zipCache_misses +=1
#                 if zipfile.is_zipfile(loadFile(zip_container)):
#                     zipCache[zip_container] = zipfile.ZipFile(loadFile(zip_container))
#                 else:
#                     logger.error("Invalid ZIP file found: %s" % file_fullpath)
#                     return None
#             else:
#                 zipCache_hits += 1
#                 # print("ZipCache hits / misses: %d/%d" % (zipCache_hits, zipCache_misses))
# 
#             # Extract file_pointer from zip:
#             file_pointer = StringIO(zipCache[zip_container].read(zip_container_relative_path))
#             if max_chunk_size != 0:
#                 file_pointer = StringIO(file_pointer.read(max_chunk_size))
# 
#         else:
#             logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
#     else:
#             # Extract file_pointer:
#             input_file = open(file_fullpath, 'rb')
#             if max_chunk_size == 0:
#                 file_pointer = StringIO(input_file.read())
#             else: file_pointer = StringIO(input_file.read(max_chunk_size))
#     assert(file_pointer is not None)
# 
#     logger.debug("Read %d bytes [%s]" % (getFileSize(file_pointer), toHex(file_pointer.read(20))))
#     # Return file pointer to pos 0
#     file_pointer.seek(0, 0)
# 
#     return file_pointer


import settings
import logging
import xml.etree.ElementTree as ET
import hashlib
import ntpath
import sys, traceback
import os
    import xml.etree.cElementTree as etree
    import xml.etree.ElementTree as etree
from ingest import Ingest
from appAux import loadFile
from datetime import datetime
        minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0)
        maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0)
        rowNumber = 0
        check_tags = ['LastModified', 'AppCompatPath']
        try:
            xml_data = loadFile(file_fullpath)
            for event, element in etree.iterparse(xml_data, events=("end",)):
                skip_entry = False
                tag_dict = {}
                if element.tag == "AppCompatItemExtended":
                    self._processElement(element, tag_dict)

                    # From time to time we get some entries with no real data on them for some unknown reason, skip for now
                    if 'AppCompatPath' in tag_dict:
                        if tag_dict['AppCompatPath'] == 'N/A':
                            logger.debug("ShimCache entry with no AppCompatPath (Sequence # %s) on %s. (skipping entry)" % (tag_dict['Sequence'], file_fullpath))
                            break

                    # Check we have everything we need and ignore entries with critical XML errors on them
                    for tag in check_tags:
                        if tag not in tag_dict or tag_dict[tag] is None:
                            if tag not in tag_dict:
                                if 'AppCompatPath' in tag_dict:
                                    logger.warning("Missing tag [%s] in %s, entry: %s (skipping entry)" % (tag, tag_dict['AppCompatPath'], file_fullpath))
                                else:
                                    logger.warning("Malformed tag [%s] in %s, entry: Unknown (skipping entry)" % (tag, file_fullpath))
                                skip_entry = True
                                break
                            if tag_dict[tag] is None:
                                if 'AppCompatPath' in tag_dict:

# Path: appAux.py
# def loadFile(file_fullpath, max_chunk_size = 0):
#     """Abstracts loading a regular file and a file from within a zip archive.
#     Args:
#         file_fullpath (str): Full path to file to load
#         max_chunk_size (int): maximum number of bytes to be read from the file
#     Returns:
#         file_pointer (StringIO): Data read from fileFullPath
#     """
#     global zipCache
#     global zipCache_hits
#     global zipCache_misses
#     logger.debug("Loading file %s" % file_fullpath)
#     if ".zip/" in file_fullpath:
#         m = re.match(r'^((?:.*)\.zip)[\\/](.*)$', file_fullpath)
#         if m:
#             zip_container = m.group(1)
#             if m.lastindex > 1:
#                 file_fullpath_unprocessed = m.group(2)
#                 if '.zip' in file_fullpath_unprocessed:
#                     m = re.match(r'^(.*)(?:\.zip[\\/](.*))*$', file_fullpath_unprocessed)
#                     if m:
#                         zip_container_relative_path = m.group(1)
#                         if m.lastindex > 1:
#                             file_fullpath_unprocessed = m.group(2)
#                         else:
#                             file_fullpath_unprocessed = None
#                     else:
#                         logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
#                 else:
#                     zip_container_relative_path = file_fullpath_unprocessed
# 
#                 # If not in the zipCache we add it
#             if zip_container not in zipCache:
#                 zipCache_misses +=1
#                 if zipfile.is_zipfile(loadFile(zip_container)):
#                     zipCache[zip_container] = zipfile.ZipFile(loadFile(zip_container))
#                 else:
#                     logger.error("Invalid ZIP file found: %s" % file_fullpath)
#                     return None
#             else:
#                 zipCache_hits += 1
#                 # print("ZipCache hits / misses: %d/%d" % (zipCache_hits, zipCache_misses))
# 
#             # Extract file_pointer from zip:
#             file_pointer = StringIO(zipCache[zip_container].read(zip_container_relative_path))
#             if max_chunk_size != 0:
#                 file_pointer = StringIO(file_pointer.read(max_chunk_size))
# 
#         else:
#             logger.error("Issue extracting container and relative path from ZIP file: %s" % file_fullpath)
#     else:
#             # Extract file_pointer:
#             input_file = open(file_fullpath, 'rb')
#             if max_chunk_size == 0:
#                 file_pointer = StringIO(input_file.read())
#             else: file_pointer = StringIO(input_file.read(max_chunk_size))
#     assert(file_pointer is not None)
# 
#     logger.debug("Read %d bytes [%s]" % (getFileSize(file_pointer), toHex(file_pointer.read(20))))
#     # Return file pointer to pos 0
#     file_pointer.seek(0, 0)
# 
#     return file_pointer


# Path: Ingest/appcompat_mirlua_v2.py
import settings
import logging
import xml.etree.ElementTree as ET
import hashlib
import ntpath
import sys, traceback
import os
    import xml.etree.cElementTree as etree
    import xml.etree.ElementTree as etree
from ingest import Ingest
from appAux import loadFile
from datetime import datetime

        minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0)
        maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0)
        rowNumber = 0
        check_tags = ['LastModified', 'AppCompatPath']
        try:
            xml_data = loadFile(file_fullpath)
            for event, element in etree.iterparse(xml_data, events=("end",)):
                skip_entry = False
                tag_dict = {}
                if element.tag == "AppCompatItemExtended":
                    self._processElement(element, tag_dict)

                    # From time to time we get some entries with no real data on them for some unknown reason, skip for now
                    if 'AppCompatPath' in tag_dict:
                        if tag_dict['AppCompatPath'] == 'N/A':
                            logger.debug("ShimCache entry with no AppCompatPath (Sequence # %s) on %s. (skipping entry)" % (tag_dict['Sequence'], file_fullpath))
                            break

                    # Check we have everything we need and ignore entries with critical XML errors on them
                    for tag in check_tags:
                        if tag not in tag_dict or tag_dict[tag] is None:
                            if tag not in tag_dict:
                                if 'AppCompatPath' in tag_dict:
                                    logger.warning("Missing tag [%s] in %s, entry: %s (skipping entry)" % (tag, tag_dict['AppCompatPath'], file_fullpath))
                                else:
                                    logger.warning("Malformed tag [%s] in %s, entry: Unknown (skipping entry)" % (tag, file_fullpath))
                                skip_entry = True
                                break
                            if tag_dict[tag] is None:
                                if 'AppCompatPath' in tag_dict:

                                    logger.warning("Malformed tag [%s: %s] in %s, entry: %s (skipping entry)" % (tag, tag_dict[tag], tag_dict['AppCompatPath'], file_fullpath))