<commit_before>import os
import csv

import logging
log = logging.getLogger(__name__)

from .models import (
    Organization,
    Convention,
    Session,
)


def import_convention(path, kind, division=False):
    with open(path) as f:
        reader = csv.reader(f, skipinitialspace=True)
        rows = [row for row in reader]
        row_one = rows[0]
        row_two = rows[1]
        stix_name = row_one[2]
        if division:
            try:
                district = row_two[0].strip()
                division = row_two[1].strip()
                location = ", ".join([row_two[2], row_two[3]])
                dates = ", ".join([row_two[4], row_two[5]])
                year = int(row_two[5])
            except IndexError:
                log.error("Could not build: {0}".format(row_two))
        else:
            try:
                district = row_two[0].strip()
                location = ", ".join([row_two[1], row_two[2]])
                dates = ", ".join([row_two[3], row_two[4]])
                year = int(row_two[4])
            except IndexError:
                log.error("Could not build: {0}".format(row_two))
        try:
            organization = Organization.objects.get(
                long_name=district,
            )
        except Organization.DoesNotExist:
            raise RuntimeError("No Match for: {0}".format(district))
        convention = Convention(
            stix_name=stix_name,
            location=location,
            dates=dates,
            year=year,
            organization=organization,
            kind=getattr(Convention.KIND, kind),
        )
        if division:
            convention.stix_div = division
            convention.division = getattr(Convention.DIVISION, division)
    return convention


def extract_sessions(convention):
    reader = csv.reader(convention.stix_file, skipinitialspace=True)
    rows = [row for row in reader]
    quartet = False
    chorus = False
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith('Subsessions:'):
            if 'Quartet' in row[0]:
                quartet = True
                parts = row[0].partition(":")
                stix_name = parts[2].strip()
            if 'Chorus' in row[0]:
                chorus = True
                parts = row[0].partition(":")
                stix_name = parts[2].strip()
    if quartet:
        quartet = Session.objects.create(
            convention=convention,
            year=convention.year,
            kind=Session.KIND.quartet,
            stix_name=stix_name,
        )

    if chorus:
        chorus = Session.objects.create(
            convention=convention,
            year=convention.year,
            kind=Session.KIND.chorus,
            stix_name=stix_name,
        )

    quartet_semis = False
    chorus_semis = False
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith("Subsessions:"):
            if "Quartet Semi-Finals" in row[0]:
                quartet_semis = True
            if "Chorus Semi-Finals" in row[0]:
                chorus_semis = True

    # TODO This is super kludgy
    if quartet:
        if quartet_semis:
            quartet.rounds.create(
                kind=quartet.rounds.model.KIND.semis,
                num=1,
            )
            quartet.rounds.create(
                kind=quartet.rounds.model.KIND.finals,
                num=2,
            )
        else:
            quartet.rounds.create(
                kind=quartet.rounds.model.KIND.finals,
                num=1,
            )

    if chorus:
        if chorus_semis:
            chorus.rounds.create(
                kind=chorus.rounds.model.KIND.semis,
                num=1,
            )
            chorus.rounds.create(
                kind=chorus.rounds.model.KIND.finals,
                num=2,
            )
        else:
            chorus.rounds.create(
                kind=chorus.rounds.model.KIND.finals,
                num=1,
            )


def extract_contests(convention):
    reader = csv.reader(convention.stix_file, skipinitialspace=True)
    rows = [row for row in reader]
    l = []
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith('Subsessions:'):
            stix_name = row[0].partition(":")[2].strip()
            session = Session.objects.get(
                stix_name=stix_name,
            )
            contest_list = row[1:]
            for c in contest_list:
                # Parse each list item for id, name
                parts = c.partition('=')
                contest_text = parts[2]
                l.append([session, contest_text])
    return l


def deinterlace(path):
    with open(path) as csvfile:
        raw = [row for row in csv.reader(
            csvfile,
        )]
        l = len(raw)
        data = []
        i = 0
        #  Deinterlace and build list of dictionaries.
        while i < l:
            if (i % 2 == 0):  # zero-indexed rows
                row = raw[i]
                row.extend(raw[i + 1])
                data.append(row)
            else:  # Skip interlaced row; added supra
                pass
            i += 1
        return data


def strip_penalties(output):
    for row in output:
        scores = [6, 7, 8, 10, 11, 12]<commit_msg>Utils through round creation<commit_after>import os
import csv

import logging
log = logging.getLogger(__name__)

from .models import (
    Organization,
    Convention,
    Session,
    Round,
)


def import_convention(path, kind, division=False):
    with open(path) as f:
        reader = csv.reader(f, skipinitialspace=True)
        rows = [row for row in reader]
        row_one = rows[0]
        row_two = rows[1]
        stix_name = row_one[2]
        if division:
            try:
                district = row_two[0].strip()
                division = row_two[1].strip()
                location = ", ".join([row_two[2], row_two[3]])
                dates = ", ".join([row_two[4], row_two[5]])
                year = int(row_two[5])
            except IndexError:
                log.error("Could not build: {0}".format(row_two))
        else:
            try:
                district = row_two[0].strip()
                location = ", ".join([row_two[1], row_two[2]])
                dates = ", ".join([row_two[3], row_two[4]])
                year = int(row_two[4])
            except IndexError:
                log.error("Could not build: {0}".format(row_two))
        try:
            organization = Organization.objects.get(
                long_name=district,
            )
        except Organization.DoesNotExist:
            raise RuntimeError("No Match for: {0}".format(district))
        convention = Convention(
            stix_name=stix_name,
            location=location,
            dates=dates,
            year=year,
            organization=organization,
            kind=getattr(Convention.KIND, kind),
        )
        if division:
            convention.stix_div = division
            convention.division = getattr(Convention.DIVISION, division)
    return convention


def extract_sessions(convention):
    reader = csv.reader(convention.stix_file, skipinitialspace=True)
    rows = [row for row in reader]
    sessions = {}
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith('Subsessions:'):
            parts = row[0].partition(":")
            contest_name = parts[2].strip()
            kind = contest_name.partition(" ")[0].lower()
            sessions[kind] = True
    for key, value in sessions.viewitems():
        Session.objects.create(
            convention=convention,
            year=convention.year,
            kind=getattr(Session.KIND, key),
            stix_name=contest_name,
        )
    return


def extract_rounds(convention):
    reader = csv.reader(convention.stix_file, skipinitialspace=True)
    rows = [row for row in reader]
    # Determine meta-data
    counts = {}
    counts['chorus'] = 0
    counts['quartet'] = 0
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith('Subsessions:'):
            # Parse session into components
            parts = row[0].partition(':')
            # Parse session meta-data
            session_text = parts[2].strip()
            if session_text.startswith('Chorus'):
                counts['chorus'] += 1
            elif session_text.startswith('Quartet'):
                counts['quartet'] += 1
            else:
                raise RuntimeError("Can't determine session kind")
    for key, value in counts.viewitems():
        if value:
            i = 1
            j = value
            while i <= value:
                try:
                    session = convention.sessions.get(
                        kind=getattr(Session.KIND, key),
                    )
                except Session.DoesNotExist:
                    print "No session: {0} {1}".format(key, value)
                rnd, created = session.rounds.get_or_create(
                    num=i,
                    kind=j,
                )
                i += 1
                j -= 1
    return


def extract_contests(convention):
    reader = csv.reader(convention.stix_file, skipinitialspace=True)
    rows = [row for row in reader]
    l = []
    for row in rows:
        if len(row) == 0:
            continue
        if row[0].startswith('Subsessions:'):
            stix_name = row[0].partition(":")[2].strip()
            l.append(stix_name)
            # try:
            #     session = convention.sessions.get(
            #         stix_name=stix_name,
            #     )
            # except Session.DoesNotExist:
            #     print "SESSION DOES NOT EXIST: {0}".format(row)
            #     continue
            # except Session.MultipleObjectsReturned:
            #     print "MULTIPLE SESSIONS: {0}".format(row)
            #     continue
            # contest_list = row[1:]
            # for c in contest_list:
            #     # Parse each list item for id, name
            #     parts = c.partition('=')
            #     contest_text = parts[2]
            #     l.append([session, contest_text])
    return l


def deinterlace(path):
    with open(path) as csvfile:
        raw = [row for row in csv.reader(
            csvfile,
        )]
        l = len(raw)
        data = []
        i = 0
        #  Deinterlace and build list of dictionaries.
        while i < l:
            if (i % 2 == 0):  # zero-indexed rows
                row = raw[i]
                row.extend(raw[i + 1])
                data.append(row)
            else:  # Skip interlaced row; added supra
                pass
            i += 1
        return data


def strip_penalties(output):
    for row in output:
        scores = [6, 7, 8, 10, 11, 12]
{'fraction_non_alphanumeric': 0.1060480530240265, 'fraction_numerical': 0.010678449783669336, 'mean_word_length': 1.4923147510896995, 'pattern_counts': {'":': 5, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5906344', 'n_tokens_mistral': 3259, 'n_tokens_neox': 3102, 'n_words': 706}