<commit_before>import hashlib
from django.db import IntegrityError
from pillowtop.listener import BasicPillow
from couchforms.models import XFormInstance
from phonelog.models import UserLog, Log
import dateutil.parser as dparser

def force_list(obj_or_list):
    return obj_or_list if isinstance(obj_or_list, list) else [obj_or_list]

class PhoneLogPillow(BasicPillow):
    document_class = XFormInstance
    couch_filter = 'phonelog/all_logs'

    def change_transport(self, doc_dict):
        xform_id = doc_dict.get('_id')
        form = doc_dict.get('form', {})
        userlogs = (form.get('user_subreport', {}) or {}).get('user', [])
        for log in force_list(userlogs):
            u = UserLog(user_id=log["user_id"], username=log["username"],
                        sync_token=log["sync_token"], xform_id=xform_id)
            u.save()

        domain = doc_dict.get('domain')
        logs = (form.get('log_subreport', {}) or {}).get('log', [])
        logged_in_user = 'unknown'
        for log in filter(None, force_list(logs)):
            if log["type"] == 'login':
                logged_in_user = log["msg"].split('-')[1]<commit_msg>allow pillow to process oddly formatted phone logs<commit_after>import hashlib
from django.db import IntegrityError
from pillowtop.listener import BasicPillow
from couchforms.models import XFormInstance
from phonelog.models import UserLog, Log
import dateutil.parser as dparser

def force_list(obj_or_list):
    return obj_or_list if isinstance(obj_or_list, list) else [obj_or_list]

def get_logs(form, report_name, report_slug):
    report = form.get(report_name, {}) or {}
    if isinstance(report, list):
        return filter(None, [log.get(report_slug) for log in report])
    return report.get(report_slug, [])

class PhoneLogPillow(BasicPillow):
    document_class = XFormInstance
    couch_filter = 'phonelog/all_logs'

    def change_transport(self, doc_dict):
        xform_id = doc_dict.get('_id')
        form = doc_dict.get('form', {})
        userlogs = get_logs(form, 'user_subreport', 'user')
        for log in force_list(userlogs):
            u = UserLog(user_id=log["user_id"], username=log["username"],
                        sync_token=log["sync_token"], xform_id=xform_id)
            u.save()

        domain = doc_dict.get('domain')
        logs = get_logs(form, 'log_subreport', 'log')
        logged_in_user = 'unknown'
        for log in filter(None, force_list(logs)):
            if log["type"] == 'login':
                logged_in_user = log["msg"].split('-')[1]
{'fraction_non_alphanumeric': 0.12171181782489203, 'fraction_numerical': 0.0007852375343541421, 'mean_word_length': 3.3704974271012005, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3670695', 'n_tokens_mistral': 806, 'n_tokens_neox': 809, 'n_words': 179}