<commit_before># coding: utf-8
import contextlib
import copy
import re
from collections import defaultdict

from django.apps import apps
from taggit.models import Tag, TaggedItem

'''
This circular import will bite you if you don't import kpi.models before
importing kpi.model_utils:
  File "kpi/model_utils.py", line 6, in <module>
    from .models import Asset
  File "kpi/models/__init__.py", line 5, in <module>
    from kpi.models.import_task import ImportTask
  File "kpi/models/import_task.py", line 6, in <module>
    from kpi.model_utils import create_assets
'''
from .constants import ASSET_TYPE_COLLECTION
from .models import Asset


TAG_RE = r'tag:(.*)'


def _load_library_content(structure):
    content = structure.get('content', {})
    if 'library' not in content:
        raise Exception('to load a library, you must have a sheet called "library"')
    library_sheet = content.get('library', [])
    del content['library']

    tag_name_to_pk = {} # Both a cache and a record of what to index later
    created_asset_pks = [] # A list of what to index at the end of the import

    grouped = defaultdict(list)
    for row in library_sheet:
        # preserve the additional sheets of imported library (but not the library)
        row_tags = []
        for key, val in row.items():
            if str(val).lower() in ['false', '0', 'no', 'n', '', 'none']:
                continue
            if re.search(TAG_RE, key):
                tag_name = re.match(TAG_RE, key).groups()[0]
                row_tags.append(tag_name)
                tag_name_to_pk[tag_name] = None # Will be filled in later
                del row[key]
        block_name = row.get('block', None)
        grouped[block_name].append((row, row_tags,))

    # Resolve tag names to PKs
    existing_tags = Tag.objects.filter(
        name__in=tag_name_to_pk.keys()).values_list('name', 'pk')
    existing_tags_dict = dict(existing_tags)
    tag_name_to_pk.update(existing_tags_dict)
    if existing_tags.count() < len(tag_name_to_pk.keys()):
        import_tag_names = set(tag_name_to_pk.keys())
        existing_tag_names = set(existing_tags_dict.keys())
        for new_tag_name in import_tag_names.difference(existing_tag_names):
            # We're not atomic, but get_or_create should be
            new_tag, created = Tag.objects.get_or_create(name=new_tag_name)
            tag_name_to_pk[new_tag_name] = new_tag.pk

    collection_name = structure['name']
    if not collection_name:
        collection_name = 'Collection'
    collection = Asset.objects.create(
        asset_type=ASSET_TYPE_COLLECTION, owner=structure['owner'],
        name=collection_name
    )
<commit_msg>Remove `block` column from asset content when…<commit_after># coding: utf-8
import contextlib
import copy
import re
from collections import defaultdict

from django.apps import apps
from taggit.models import Tag, TaggedItem

'''
This circular import will bite you if you don't import kpi.models before
importing kpi.model_utils:
  File "kpi/model_utils.py", line 6, in <module>
    from .models import Asset
  File "kpi/models/__init__.py", line 5, in <module>
    from kpi.models.import_task import ImportTask
  File "kpi/models/import_task.py", line 6, in <module>
    from kpi.model_utils import create_assets
'''
from .constants import ASSET_TYPE_COLLECTION
from .models import Asset


TAG_RE = r'tag:(.*)'


def _load_library_content(structure):
    content = structure.get('content', {})
    if 'library' not in content:
        raise Exception('to load a library, you must have a sheet called "library"')
    library_sheet = content.get('library', [])
    del content['library']

    tag_name_to_pk = {} # Both a cache and a record of what to index later
    created_asset_pks = [] # A list of what to index at the end of the import

    grouped = defaultdict(list)
    for row in library_sheet:
        # preserve the additional sheets of imported library (but not the library)
        row_tags = []
        for key, val in row.items():
            if str(val).lower() in ['false', '0', 'no', 'n', '', 'none']:
                continue
            if re.search(TAG_RE, key):
                tag_name = re.match(TAG_RE, key).groups()[0]
                row_tags.append(tag_name)
                tag_name_to_pk[tag_name] = None # Will be filled in later
                del row[key]
        block_name = row.pop('block', None)
        grouped[block_name].append((row, row_tags,))

    # Resolve tag names to PKs
    existing_tags = Tag.objects.filter(
        name__in=tag_name_to_pk.keys()).values_list('name', 'pk')
    existing_tags_dict = dict(existing_tags)
    tag_name_to_pk.update(existing_tags_dict)
    if existing_tags.count() < len(tag_name_to_pk.keys()):
        import_tag_names = set(tag_name_to_pk.keys())
        existing_tag_names = set(existing_tags_dict.keys())
        for new_tag_name in import_tag_names.difference(existing_tag_names):
            # We're not atomic, but get_or_create should be
            new_tag, created = Tag.objects.get_or_create(name=new_tag_name)
            tag_name_to_pk[new_tag_name] = new_tag.pk

    collection_name = structure['name']
    if not collection_name:
        collection_name = 'Collection'
    collection = Asset.objects.create(
        asset_type=ASSET_TYPE_COLLECTION, owner=structure['owner'],
        name=collection_name
    )

{'fraction_non_alphanumeric': 0.10061372512553468, 'fraction_numerical': 0.0022317277292170355, 'mean_word_length': 3.1401077752117015, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4694986', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1645, 'n_words': 458}