<commit_before>from celery import shared_task

from isisdata.models import *

import logging
import smart_open
import unicodecsv as csv

COLUMN_NAME_ATTR_SUBJ_ID = 'ATT Subj ID'
COLUMN_NAME_ATTR_RELATED_NAME = 'Related Record Name'
COLUMN_NAME_ATTR_TYPE = 'ATT Type'
COLUMN_NAME_ATTR_VALUE = 'ATT Value'
COLUMN_NAME_ATTR_DATE_FREE = 'ATT DateFree'
COLUMN_NAME_ATTR_DATE_BEGIN = 'ATT DateBegin'
COLUMN_NAME_ATTR_DATE_END = 'ATT DateEnd'
COLUMN_NAME_ATTR_PLACE_NAME = 'ATT PlaceName'
COLUMN_NAME_ATTR_PLACE_LINK = 'ATT PlaceLink'
COLUMN_NAME_ATTR_NOTES = 'ATT Notes'

logger = logging.getLogger(__name__)

@shared_task
def add_attributes_to_authority(file_path, error_path, task_id):
    logging.info('Adding attributes from %s.' % (file_path))
    # this is a hack but the best I can come up with right now :op
    logging.debug('Make AuthorityValue exists in ContentType table...')
    ContentType.objects.get_or_create(model='authorityvalue', app_label='isisdata')

    with smart_open.smart_open(file_path, 'r') as f:
        reader = csv.reader(f, encoding='utf-8')
        task = AsyncTask.objects.get(pk=task_id)
        # we want to avoid loading everything in memory, in case it's a large file
        # we do not count the header, so we start at -1
        row_count = -1
        for row in reader:
            row_count += 1
        task.max_value = row_count
        task.save()
        # reset file cursor to first data line
        f.seek(1)
        current_count = 0
        not_matching_subject_names = []<commit_msg>[ISISCB-1021] trying a differen reader<commit_after>from celery import shared_task

from isisdata.models import *

import logging
import smart_open
import unicodecsv as csv

COLUMN_NAME_ATTR_SUBJ_ID = 'ATT Subj ID'
COLUMN_NAME_ATTR_RELATED_NAME = 'Related Record Name'
COLUMN_NAME_ATTR_TYPE = 'ATT Type'
COLUMN_NAME_ATTR_VALUE = 'ATT Value'
COLUMN_NAME_ATTR_DATE_FREE = 'ATT DateFree'
COLUMN_NAME_ATTR_DATE_BEGIN = 'ATT DateBegin'
COLUMN_NAME_ATTR_DATE_END = 'ATT DateEnd'
COLUMN_NAME_ATTR_PLACE_NAME = 'ATT PlaceName'
COLUMN_NAME_ATTR_PLACE_LINK = 'ATT PlaceLink'
COLUMN_NAME_ATTR_NOTES = 'ATT Notes'

logger = logging.getLogger(__name__)

@shared_task
def add_attributes_to_authority(file_path, error_path, task_id):
    logging.info('Adding attributes from %s.' % (file_path))
    # this is a hack but the best I can come up with right now :op
    logging.debug('Make AuthorityValue exists in ContentType table...')
    ContentType.objects.get_or_create(model='authorityvalue', app_label='isisdata')

    with smart_open.smart_open(file_path, 'r') as f:
        reader = csv.reader(f, encoding='utf-8')
        task = AsyncTask.objects.get(pk=task_id)
        # we want to avoid loading everything in memory, in case it's a large file
        # we do not count the header, so we start at -1
        row_count = -1
        for row in csv.DictReader(f):
            row_count += 1
        task.max_value = row_count
        task.save()
        # reset file cursor to first data line
        f.seek(1)
        current_count = 0
        not_matching_subject_names = []
{'fraction_non_alphanumeric': 0.08069722401549387, 'fraction_numerical': 0.005164622336991607, 'mean_word_length': 3.8346333853354135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4031735', 'n_tokens_mistral': 1027, 'n_tokens_neox': 1015, 'n_words': 282}