<commit_before>from django.conf import settings
import simplejson as json
from maldini import models
from dateutil import parser
import os
import tika
import tika.parser
import tika.language

tika.tika.TikaClientOnly = True


def extract_meta(meta):
    def _get_flat(dict, *keys):
        """
        Select the first non-null item from the dict that matches a key.
        If it's a list, take the first element from it.
        """
        item = None
        for key in keys:
            item = dict.get(key)
            if item is not None:
                break

        if type(item) is list:
            return item[0]

        return item

    def _get_bool(dict, *keys):
        """
        Select the first non-null item from the dict that matches a key.
        Cast it to bool.
        """
        item = _get_flat(dict, *keys)

        if not item:
            return False  # check for None

        if type(item) is bool:
            return item  # check if it's already a bool
        return item.lower() == "true"

    data = {
        'content-type':     _get_flat(meta, 'Content-Type', 'content-type'),
        'author':           _get_flat(meta, 'Author', 'meta:author', 'creator'),
        'date_created':     _get_flat(meta, 'Creation-Date', 'dcterms:created', 'meta:created', 'created'),
        'date':             _get_flat(meta, 'Last-Modified', 'Last-Saved-Date', 'dcterms:modified',
                                      'meta:modified', 'created'),
        'encrypted-pdf':    _get_bool(meta, 'pdf:encrypted'),
        'tika':             meta
    }

    for key in ['date', 'date_created']:
        if key in data:
            data[key] = parser.parse(data[key]).isoformat()

    return data

<commit_msg>save to cache in a transaction<commit_after>from django.conf import settings
from django.db import transaction
import simplejson as json
from maldini import models
from dateutil import parser
import os
import tika
import tika.parser
import tika.language

tika.tika.TikaClientOnly = True


def extract_meta(meta):
    def _get_flat(dict, *keys):
        """
        Select the first non-null item from the dict that matches a key.
        If it's a list, take the first element from it.
        """
        item = None
        for key in keys:
            item = dict.get(key)
            if item is not None:
                break

        if type(item) is list:
            return item[0]

        return item

    def _get_bool(dict, *keys):
        """
        Select the first non-null item from the dict that matches a key.
        Cast it to bool.
        """
        item = _get_flat(dict, *keys)

        if not item:
            return False  # check for None

        if type(item) is bool:
            return item  # check if it's already a bool
        return item.lower() == "true"

    data = {
        'content-type':     _get_flat(meta, 'Content-Type', 'content-type'),
        'author':           _get_flat(meta, 'Author', 'meta:author', 'creator'),
        'date_created':     _get_flat(meta, 'Creation-Date', 'dcterms:created', 'meta:created', 'created'),
        'date':             _get_flat(meta, 'Last-Modified', 'Last-Saved-Date', 'dcterms:modified',
                                      'meta:modified', 'created'),
        'encrypted-pdf':    _get_bool(meta, 'pdf:encrypted'),
        'tika':             meta
    }

    for key in ['date', 'date_created']:
        if key in data:
            data[key] = parser.parse(data[key]).isoformat()

    return data


@transaction.atomic
{'fraction_non_alphanumeric': 0.10697149308495625, 'fraction_numerical': 0.0005644933672029354, 'mean_word_length': 1.960735171261487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2596446', 'n_tokens_mistral': 1044, 'n_tokens_neox': 1014, 'n_words': 332}