<commit_before>from followthemoney import model

from ingestors.ingestor import Ingestor
from ingestors.support.cellebrite import CellebriteSupport
from ingestors.support.encoding import EncodingSupport


class CellebriteIngestor(Ingestor, EncodingSupport, CellebriteSupport):
    "Ingestor for Cellebrite XML reports"

    MIME_TYPES = ['text/xml']
    EXTENSIONS = ['xml']
    SCORE = 5

    def ingest(self, file_path, entity):
        """Ingestor implementation."""
        entity.schema = model.get('Document')

        doc = self.parse_xml_path(file_path)
        root = doc.getroot()
        ns = {"ns": root.nsmap[None]}
        self.ns = ns

        self.device_owner = self.manager.make_entity('LegalEntity')
        self.device_id = root.xpath('./ns:metadata/ns:item[@name="DeviceInfoUniqueID"][1]/text()', namespaces=ns)[0]  # noqa
        imei = root.xpath('./ns:metadata/ns:item[@name="IMEI"][1]/text()', namespaces=ns)[0]  # noqa
        self.device_owner.make_id(self.device_id, imei)
        emails = root.xpath('./ns:metadata/ns:item[@name="DeviblceInfoAppleID"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('email', emails)
        name = root.xpath('./ns:metadata/ns:item[@name="DeviceInfoOwnerName"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('name', name)
        phone_numbers = root.xpath('./ns:metadata/ns:item[@name="MSISDN"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('phone', phone_numbers)
        self.manager.emit_entity(self.device_owner)

        self.parse_calls(root)
        self.parse_messages(root)
        self.parse_notes(root)
        self.parse_sms(root)
        self.parse_contacts(root)

    @classmethod
    def match(cls, file_path, entity):
        score = super(CellebriteIngestor, cls).match(file_path, entity)
        if score > 0 and cls.inspect_metadata(file_path):
            score = cls.SCORE * 2
        return score<commit_msg>Scoring adjustment for cellebrite ingestor<commit_after>from followthemoney import model

from ingestors.ingestor import Ingestor
from ingestors.support.cellebrite import CellebriteSupport
from ingestors.support.encoding import EncodingSupport


class CellebriteIngestor(Ingestor, EncodingSupport, CellebriteSupport):
    "Ingestor for Cellebrite XML reports"

    MIME_TYPES = ['text/xml']
    EXTENSIONS = ['xml']
    SCORE = 0.5

    def ingest(self, file_path, entity):
        """Ingestor implementation."""
        entity.schema = model.get('Document')

        doc = self.parse_xml_path(file_path)
        root = doc.getroot()
        ns = {"ns": root.nsmap[None]}
        self.ns = ns

        self.device_owner = self.manager.make_entity('LegalEntity')
        self.device_id = root.xpath('./ns:metadata/ns:item[@name="DeviceInfoUniqueID"][1]/text()', namespaces=ns)[0]  # noqa
        imei = root.xpath('./ns:metadata/ns:item[@name="IMEI"][1]/text()', namespaces=ns)[0]  # noqa
        self.device_owner.make_id(self.device_id, imei)
        emails = root.xpath('./ns:metadata/ns:item[@name="DeviblceInfoAppleID"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('email', emails)
        name = root.xpath('./ns:metadata/ns:item[@name="DeviceInfoOwnerName"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('name', name)
        phone_numbers = root.xpath('./ns:metadata/ns:item[@name="MSISDN"]/text()', namespaces=ns)  # noqa
        self.device_owner.add('phone', phone_numbers)
        self.manager.emit_entity(self.device_owner)

        self.parse_calls(root)
        self.parse_messages(root)
        self.parse_notes(root)
        self.parse_sms(root)
        self.parse_contacts(root)

    @classmethod
    def match(cls, file_path, entity):
        score = super(CellebriteIngestor, cls).match(file_path, entity)
        if score > 0 and cls.inspect_metadata(file_path):
            score = cls.SCORE * 20
        return score
{'fraction_non_alphanumeric': 0.13631713554987213, 'fraction_numerical': 0.004092071611253197, 'mean_word_length': 3.404279279279279, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3377294', 'n_tokens_mistral': 1312, 'n_tokens_neox': 1288, 'n_words': 209}