<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import json
from datetime import datetime

from django.db.models import Count
from django.core.exceptions import ValidationError
from django.contrib.gis.db import models

from treemap.models import User, Instance

from importer import errors


class GenericImportEvent(models.Model):

    class Meta:
        abstract = True

    PENDING_VERIFICATION = 1
    LOADING = 7
    VERIFIYING = 2
    FINISHED_VERIFICATION = 3
    CREATING = 4
    FINISHED_CREATING = 5
    FAILED_FILE_VERIFICATION = 6

    # Original Name of the file
    file_name = models.CharField(max_length=255)

    # Global errors and notices (json)
    errors = models.TextField(default='')

    field_order = models.TextField(default='')

    # Metadata about this particular import
    owner = models.ForeignKey(User)
    instance = models.ForeignKey(Instance)
    created = models.DateTimeField(auto_now=True)
    completed = models.DateTimeField(null=True, blank=True)

    status = models.IntegerField(default=PENDING_VERIFICATION)

    @property
    def row_count(self):
        return self.rows().count()

    @property
    def can_export(self):
        return self.status != self.FAILED_FILE_VERIFICATION

    def status_summary(self):
        summaries = {
            self.PENDING_VERIFICATION: "Not Yet Started",
            self.LOADING: "Loading",
            self.VERIFIYING: "Verifying",
            self.FINISHED_VERIFICATION: "Verification Complete",
            self.CREATING: "Creating Trees",
            self.FAILED_FILE_VERIFICATION: "Invalid File Structure",
        }
        return summaries.get(self.status, "Finished")

    def active(self):
        return self.status != GenericImportEvent.FINISHED_CREATING

    def is_loading(self):
        return self.status == self.LOADING

    def is_running(self):
        return (
            self.status == self.VERIFIYING or
            self.status == self.CREATING)

    def is_finished(self):
        return (
            self.status == self.FINISHED_VERIFICATION or
            self.status == self.FINISHED_CREATING or
            self.status == self.FAILED_FILE_VERIFICATION)

    def row_counts_by_status(self):
        q = self.row_set()\
                .values('status')\
                .annotate(Count('status'))

        return {r['status']: r['status__count'] for r in q}

    def completed_row_count(self):
        n_left = self.row_counts_by_status().get(GenericImportRow.WAITING, 0)
        return self.row_set().count() - n_left

    def update_status(self):
        """ Update the status field based on current row statuses """
        pass

    def append_error(self, err, data=None):
        code, msg, fatal = err

        if data and not isinstance(data, list):
            raise ValidationError("For this class, data must be a list")

        if self.errors is None or self.errors == '':
            self.errors = '[]'

        self.errors = json.dumps(
            self.errors_as_array() + [
                {'code': code,
                 'msg': msg,
                 'data': data,
                 'fatal': fatal}])

        return self

    def errors_as_array(self):
        if self.errors is None or self.errors == '':
            return []
        else:
            return json.loads(self.errors)

    def has_errors(self):
        return len(self.errors_as_array()) > 0

    def row_set(self):
        raise Exception('Abstract Method')

    def rows(self):
        return self.row_set().order_by('idx').all()

    def legal_and_required_fields(self):
        raise Exception('Abstract Method')

    def validate_field_names(self, input_fields):
        """
        Make sure the imported file has valid columns
        """
        is_valid = True

        legal_fields, required_fields = self.legal_and_required_fields()
        input_fields = set(input_fields)

        # Extra input fields cause a fatal error
        extra = input_fields.difference(legal_fields)
        if len(extra) > 0:
            is_valid = False
            self.append_error(errors.UNMATCHED_FIELDS, list(extra))

        missing_required_fields = required_fields - input_fields
        for field in missing_required_fields:
            is_valid = False
            self.append_error(errors.MISSING_FIELD, data=[field])

        return is_valid
<commit_msg>Show unknown fields in their original order<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division

import json
from datetime import datetime

from django.db.models import Count
from django.core.exceptions import ValidationError
from django.contrib.gis.db import models

from treemap.models import User, Instance

from importer import errors


class GenericImportEvent(models.Model):

    class Meta:
        abstract = True

    PENDING_VERIFICATION = 1
    LOADING = 7
    VERIFIYING = 2
    FINISHED_VERIFICATION = 3
    CREATING = 4
    FINISHED_CREATING = 5
    FAILED_FILE_VERIFICATION = 6

    # Original Name of the file
    file_name = models.CharField(max_length=255)

    # Global errors and notices (json)
    errors = models.TextField(default='')

    field_order = models.TextField(default='')

    # Metadata about this particular import
    owner = models.ForeignKey(User)
    instance = models.ForeignKey(Instance)
    created = models.DateTimeField(auto_now=True)
    completed = models.DateTimeField(null=True, blank=True)

    status = models.IntegerField(default=PENDING_VERIFICATION)

    @property
    def row_count(self):
        return self.rows().count()

    @property
    def can_export(self):
        return self.status != self.FAILED_FILE_VERIFICATION

    def status_summary(self):
        summaries = {
            self.PENDING_VERIFICATION: "Not Yet Started",
            self.LOADING: "Loading",
            self.VERIFIYING: "Verifying",
            self.FINISHED_VERIFICATION: "Verification Complete",
            self.CREATING: "Creating Trees",
            self.FAILED_FILE_VERIFICATION: "Invalid File Structure",
        }
        return summaries.get(self.status, "Finished")

    def active(self):
        return self.status != GenericImportEvent.FINISHED_CREATING

    def is_loading(self):
        return self.status == self.LOADING

    def is_running(self):
        return (
            self.status == self.VERIFIYING or
            self.status == self.CREATING)

    def is_finished(self):
        return (
            self.status == self.FINISHED_VERIFICATION or
            self.status == self.FINISHED_CREATING or
            self.status == self.FAILED_FILE_VERIFICATION)

    def row_counts_by_status(self):
        q = self.row_set()\
                .values('status')\
                .annotate(Count('status'))

        return {r['status']: r['status__count'] for r in q}

    def completed_row_count(self):
        n_left = self.row_counts_by_status().get(GenericImportRow.WAITING, 0)
        return self.row_set().count() - n_left

    def update_status(self):
        """ Update the status field based on current row statuses """
        pass

    def append_error(self, err, data=None):
        code, msg, fatal = err

        if data and not isinstance(data, list):
            raise ValidationError("For this class, data must be a list")

        if self.errors is None or self.errors == '':
            self.errors = '[]'

        self.errors = json.dumps(
            self.errors_as_array() + [
                {'code': code,
                 'msg': msg,
                 'data': data,
                 'fatal': fatal}])

        return self

    def errors_as_array(self):
        if self.errors is None or self.errors == '':
            return []
        else:
            return json.loads(self.errors)

    def has_errors(self):
        return len(self.errors_as_array()) > 0

    def row_set(self):
        raise Exception('Abstract Method')

    def rows(self):
        return self.row_set().order_by('idx').all()

    def legal_and_required_fields(self):
        raise Exception('Abstract Method')

    def validate_field_names(self, input_fields):
        """
        Make sure the imported file has valid columns
        """
        is_valid = True

        legal_fields, required_fields = self.legal_and_required_fields()

        # Extra input fields cause a fatal error
        extra = [field for field in input_fields
                 if field not in legal_fields]
        if len(extra) > 0:
            is_valid = False
            self.append_error(errors.UNMATCHED_FIELDS, list(extra))

        for field in required_fields:
            if field not in input_fields:
                is_valid = False
                self.append_error(errors.MISSING_FIELD, data=[field])

        return is_valid

{'fraction_non_alphanumeric': 0.09116393810271362, 'fraction_numerical': 0.0031397174254317113, 'mean_word_length': 2.3964204112718965, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1669257', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2473, 'n_words': 639}