<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import re
import functools
from lxml import etree
import logging
import dateutil
from pytz import timezone
from django.conf import settings
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.db import IntegrityError
from django_orghierarchy.models import Organization

from .base import Importer, register_importer, recur_dict
from .yso import KEYWORDS_TO_ADD_TO_AUDIENCE
from .util import unicodetext
from events.models import DataSource, Event, EventAggregate, EventAggregateMember, Keyword, Place, License
from events.keywords import KeywordMatcher
from events.translation_utils import expand_model_fields

# Per module logger
logger = logging.getLogger(__name__)

LOCATION_TPREK_MAP = {
    'malmitalo': '8740',
    'malms kulturhus': '8740',
    'malms bibliotek - malms kulturhus': '8192',
    '<NAME>': '8192',
    'helsingin kaupungintalo': '28473',
    'stoa': '7259',
    'östra centrums bibliotek': '8184',
    'parvigalleria': '7259',
    'musiikkisali': '7259',
    'kanneltalo': '7255',
    'vuotalo': '7260',
    'vuosali': '7260',
    'savoy-teatteri': '7258',
    'savoy': '7258',
    'annantalo': '7254',
    'annegården': '7254',
    'espan lava': '7265',
    'caisa': '7256',
    'nuorisokahvila clubi': '8006',
    'haagan nuorisotalo': '8023',
    'vuosaaren kirjasto': '8310',
    'riistavuoren palvelukeskus': '47695',
    'kannelmäen palvelukeskus': '51869',
    'leikkipuisto lampi': '57117',
}

ADDRESS_TPREK_MAP = {
    'annankatu 30': 'annantalo',
    'annegatan 30': 'annantalo',
    'mosaiikkitori 2': 'vuotalo',
    'ala-malmin tori 1': 'malmitalo',
    'ala-malmin tori': 'malmitalo',
    'klaneettitie 5': 'kanneltalo',
    'klarinettvägen 5': 'kanneltalo',
    'turunlinnantie 1': 'stoa'
}

CATEGORIES_TO_IGNORE = [
    286, 596, 614, 307, 632, 645, 675, 231, 364, 325, 324, 319, 646, 640,
    641, 642, 643, 670, 671, 673, 674, 725, 312, 344, 365, 239, 240, 308, 623,
    229, 230, 323, 320, 357, 358, 728, 729, 730, 735, 736,

    # The categories below are languages, ignore as categories
    # todo: add as event languages
    53, 54, 55
]

# Events having one of these categories are courses - they are excluded when importing events
# and only they are included when importing courses.
COURSE_CATEGORIES = {
    70, 71, 72, 73, 75, 77, 79, 80,
    81, 83, 84, 85, 87, 316, 629, 632,
    728, 729, 730, 735,
}


def _query_courses():
    filter_out_keywords = map(
        make_kulke_id,
        COURSE_CATEGORIES
    )
    return Event.objects.filter(
        data_source='kulke'
    ).filter(
        keywords__id__in=set(filter_out_keywords)
    )


def _delete_courses():
    courses_q = _query_courses()
    courses_q.delete()


SPORTS = ['p965']
GYMS = ['p8504']
MOVIES = ['p1235', 'p16327']
CHILDREN = ['p4354']
YOUTH = ['p11617']
ELDERLY = ['p2434']
FAMILIES = ['p4363']

MANUAL_CATEGORIES = {
    # urheilu
    546: SPORTS, 547: SPORTS, 431: SPORTS, 638: SPORTS,
    # kuntosalit
    607: GYMS, 615: GYMS,
    # harrastukset
    626: ['p2901'],
    # erityisliikunta
    634: ['p3093'],
    # monitaiteisuus
    223: ['p25216'],
    # seniorit > ikääntyneet ja vanhukset
    354: ['p2433'] + ELDERLY,
    # saunominen
    371: ['p11049'],
    # lastentapahtumat > lapset (!)
    105: CHILDREN,
    # steppi
    554: ['p19614'],
    # liikuntaleiri
    710: ['p143', 'p916'],
    # teatteri ja sirkus
    351: ['p2850'],
    # elokuva ja media
    205: MOVIES + ['p2445'],
    # skidikino
    731: CHILDREN + MOVIES,
    # luennot ja keskustelut
    733: ['p15875', 'p14004'],
    # nuorille
    734: YOUTH,
    # elokuva
    737: MOVIES,
    # perheliikunta
    628: SPORTS + FAMILIES,
    # lapset ja nuoret
    355: CHILDREN + YOUTH,
    # lapsi ja aikuinen yhdessä > perheet
    747: FAMILIES
}

# these are added to all courses
COURSE_KEYWORDS = ('p9270',)

# retain the above for simplicity, even if kulke importer internally requires full keyword ids
KEYWORDS_TO_ADD_TO_AUDIENCE = ['yso:{}'.format(i) for i in KEYWORDS_TO_ADD_TO_AUDIENCE]

LOCAL_TZ = timezone('Europe/Helsinki')


def make_kulke_id(num):
    return "kulke:{}".format(num)


def make_event_name(title, subtitle):
    if title and subtitle:
        return "{} – {}".format(title, subtitle)
    elif title:
        return title
    elif subtitle:
        return subtitle


def get_event_name(event):
    if 'fi' in event['name']:
        return event['name']['fi']
    else:
        names = list(event['name'].values())
        if len(names):
            return None
        else:
            return names[0]


def parse_age_range(secondary_headline):
    pattern = r'^\D*(\d{1,2}).(\d{1,2}).[v.|år].+$'
    match = re.match(pattern, secondary_headline)

    if match:
        age_range_in_string = match.groups()
        age_range = (int(age_range_in_string[0]), int(age_range_in_string[1]))
        return age_range
    else:
        return (None, None)


def parse_course_time(secondary_headline):
    return secondary_headline


@register_importer
class KulkeImporter(Importer):
    name = "kulke"
    supported_languages = ['fi', 'sv', 'en']
    languages_to_detect = []

    def setup(self):
        self.languages_to_detect = [lang[0].replace('-', '_') for lang in settings.LANGUAGES
                                    if lang[0] not in self.supported_languages]
        ds_args = dict(id=self.name)
        defaults = dict(name='Kulttuurikeskus')
        self.tprek_data_source = DataSource.objects.get(id='tprek')
        self.data_source, _ = DataSource.objects.get_or_create(defaults=defaults, **ds_args)

        ds_args = dict(id='ahjo')
        defaults = dict(name='Ahjo')
        ahjo_ds, _ = DataSource.objects.get_or_create(defaults=defaults, **ds_args)

        org_args = dict(origin_id='u4804001050', data_source=ahjo_ds)
        defaults = dict(name='Yleiset kulttuuripalvelut')
        self.organization, _ = Organization.objects.get_or_create(defaults=defaults, **org_args)

        # Build a cached list of Places to avoid frequent hits to the db
        id_list = LOCATION_TPREK_MAP.values()
        place_list = Place.objects.filter(data_source=self.tprek_data_source).filter(origin_id__in=id_list)
        self.tprek_by_id = {p.origin_id: p.id for p in place_list}<commit_msg>partially implement the regex necessary to parse course time from course titles<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import re
import functools
from lxml import etree
import logging
import dateutil
from datetime import time
from pytz import timezone
from django.conf import settings
from django.core.validators import URLValidator
from django.core.exceptions import ValidationError, ObjectDoesNotExist
from django.db import IntegrityError
from django_orghierarchy.models import Organization

from .base import Importer, register_importer, recur_dict
from .yso import KEYWORDS_TO_ADD_TO_AUDIENCE
from .util import unicodetext
from events.models import DataSource, Event, EventAggregate, EventAggregateMember, Keyword, Place, License
from events.keywords import KeywordMatcher
from events.translation_utils import expand_model_fields

# Per module logger
logger = logging.getLogger(__name__)

LOCATION_TPREK_MAP = {
    'malmitalo': '8740',
    'malms kulturhus': '8740',
    'malms bibliotek - malms kulturhus': '8192',
    'malmin kirjasto': '8192',
    'helsingin kaupungintalo': '28473',
    'stoa': '7259',
    'östra centrums bibliotek': '8184',
    'parvigalleria': '7259',
    'musiikkisali': '7259',
    'kanneltalo': '7255',
    'vuotalo': '7260',
    'vuosali': '7260',
    'savoy-teatteri': '7258',
    'savoy': '7258',
    'annantalo': '7254',
    'annegården': '7254',
    'espan lava': '7265',
    'caisa': '7256',
    'nuorisokahvila clubi': '8006',
    'haagan nuorisotalo': '8023',
    'vuosaaren kirjasto': '8310',
    'riistavuoren palvelukeskus': '47695',
    'kannelmäen palvelukeskus': '51869',
    'leikkipuisto lampi': '57117',
}

ADDRESS_TPREK_MAP = {
    'annankatu 30': 'annantalo',
    'annegatan 30': 'annantalo',
    'mosaiikkitori 2': 'vuotalo',
    'ala-malmin tori 1': 'malmitalo',
    'ala-malmin tori': 'malmitalo',
    'klaneettitie 5': 'kanneltalo',
    'klarinettvägen 5': 'kanneltalo',
    'turunlinnantie 1': 'stoa'
}

CATEGORIES_TO_IGNORE = [
    286, 596, 614, 307, 632, 645, 675, 231, 364, 325, 324, 319, 646, 640,
    641, 642, 643, 670, 671, 673, 674, 725, 312, 344, 365, 239, 240, 308, 623,
    229, 230, 323, 320, 357, 358, 728, 729, 730, 735, 736,

    # The categories below are languages, ignore as categories
    # todo: add as event languages
    53, 54, 55
]

# Events having one of these categories are courses - they are excluded when importing events
# and only they are included when importing courses.
COURSE_CATEGORIES = {
    70, 71, 72, 73, 75, 77, 79, 80,
    81, 83, 84, 85, 87, 316, 629, 632,
    728, 729, 730, 735,
}


def _query_courses():
    filter_out_keywords = map(
        make_kulke_id,
        COURSE_CATEGORIES
    )
    return Event.objects.filter(
        data_source='kulke'
    ).filter(
        keywords__id__in=set(filter_out_keywords)
    )


def _delete_courses():
    courses_q = _query_courses()
    courses_q.delete()


SPORTS = ['p965']
GYMS = ['p8504']
MOVIES = ['p1235', 'p16327']
CHILDREN = ['p4354']
YOUTH = ['p11617']
ELDERLY = ['p2434']
FAMILIES = ['p4363']

MANUAL_CATEGORIES = {
    # urheilu
    546: SPORTS, 547: SPORTS, 431: SPORTS, 638: SPORTS,
    # kuntosalit
    607: GYMS, 615: GYMS,
    # harrastukset
    626: ['p2901'],
    # erityisliikunta
    634: ['p3093'],
    # monitaiteisuus
    223: ['p25216'],
    # seniorit > ikääntyneet ja vanhukset
    354: ['p2433'] + ELDERLY,
    # saunominen
    371: ['p11049'],
    # lastentapahtumat > lapset (!)
    105: CHILDREN,
    # steppi
    554: ['p19614'],
    # liikuntaleiri
    710: ['p143', 'p916'],
    # teatteri ja sirkus
    351: ['p2850'],
    # elokuva ja media
    205: MOVIES + ['p2445'],
    # skidikino
    731: CHILDREN + MOVIES,
    # luennot ja keskustelut
    733: ['p15875', 'p14004'],
    # nuorille
    734: YOUTH,
    # elokuva
    737: MOVIES,
    # perheliikunta
    628: SPORTS + FAMILIES,
    # lapset ja nuoret
    355: CHILDREN + YOUTH,
    # lapsi ja aikuinen yhdessä > perheet
    747: FAMILIES
}

# these are added to all courses
COURSE_KEYWORDS = ('p9270',)

# retain the above for simplicity, even if kulke importer internally requires full keyword ids
KEYWORDS_TO_ADD_TO_AUDIENCE = ['yso:{}'.format(i) for i in KEYWORDS_TO_ADD_TO_AUDIENCE]

LOCAL_TZ = timezone('Europe/Helsinki')


def make_kulke_id(num):
    return "kulke:{}".format(num)


def make_event_name(title, subtitle):
    if title and subtitle:
        return "{} – {}".format(title, subtitle)
    elif title:
        return title
    elif subtitle:
        return subtitle


def get_event_name(event):
    if 'fi' in event['name']:
        return event['name']['fi']
    else:
        names = list(event['name'].values())
        if len(names):
            return None
        else:
            return names[0]


def parse_age_range(secondary_headline):
    pattern = r'^\D*(\d{1,2}).(\d{1,2}).[v.|år].+$'
    match = re.match(pattern, secondary_headline)

    if match:
        age_range_in_string = match.groups()
        age_range = (int(age_range_in_string[0]), int(age_range_in_string[1]))
        return age_range
    else:
        return (None, None)


def parse_course_time(secondary_headline):
    pattern = r'^.*klo?\s(\d{1,2})\.(\d{1,2}).(\d{1,2})\.(\d{1,2})*$'
    match = re.match(pattern, secondary_headline)

    if match:
        course_time_beginning_hour = int(match.groups()[0])
        course_time_beginning_minute = int(match.groups()[1])
        course_time_end_hour = int(match.groups()[2])
        course_time_end_minute = int(match.groups()[3])
        course_time_beginning = time(hour=course_time_beginning_hour, minute=course_time_beginning_minute)
        course_time_end = time(hour=course_time_end_hour, minute=course_time_end_minute)
        return (course_time_beginning, course_time_end)
    else:
        return (None, None)


@register_importer
class KulkeImporter(Importer):
    name = "kulke"
    supported_languages = ['fi', 'sv', 'en']
    languages_to_detect = []

    def setup(self):
        self.languages_to_detect = [lang[0].replace('-', '_') for lang in settings.LANGUAGES
                                    if lang[0] not in self.supported_languages]
        ds_args = dict(id=self.name)
        defaults = dict(name='Kulttuurikeskus')
        self.tprek_data_source = DataSource.objects.get(id='tprek')
        self.data_source, _ = DataSource.objects.get_or_create(defaults=defaults, **ds_args)

        ds_args = dict(id='ahjo')
        defaults = dict(name='Ahjo')
        ahjo_ds, _ = DataSource.objects.get_or_create(defaults=defaults, **ds_args)

        org_args = dict(origin_id='u4804001050', data_source=ahjo_ds)
        defaults = dict(name='Yleiset kulttuuripalvelut')
        self.organization, _ = Organization.objects.get_or_create(defaults=defaults, **org_args)

        # Build a cached list of Places to avoid frequent hits to the db
        id_list = LOCATION_TPREK_MAP.values()
        place_list = Place.objects.filter(data_source=self.tprek_data_source).filter(origin_id__in=id_list)
        self.tprek_by_id = {p.origin_id: p.id for p in place_list}
{'fraction_non_alphanumeric': 0.12564747669083912, 'fraction_numerical': 0.06881752256918751, 'mean_word_length': 3.405149934810952, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3709139', 'n_tokens_mistral': 5676, 'n_tokens_neox': 4970, 'n_words': 1159}