<commit_before>from funcy import flatten, some
from datetime import datetime, time, timedelta

from .utils import (
    find_first,
    date_from_timestamp,
    strip_links,
    time_from_seconds,
)


# main objects

def transform_event(kudago_event):
    tags = kudago_event['tags']
    categories = kudago_event['categories']
    place = kudago_event['place']

    type_ = find_first(('festival', 'exhibition', 'theater'), categories)
    dates = filter(is_date_finite, kudago_event['dates'])

    if type_ not in ('festival', 'exhibition'):
        dates = flatten(map(split_date, dates))

    return {
        'id': kudago_event['id'],
        'type': type_,
        'is_premiere': 'премьера' in tags,

        'name': kudago_event['short_title'],
        'full_name': kudago_event['title'],
        'tagline': kudago_event['tagline'],
        'lead': strip_links(kudago_event['description']),
        'description': strip_links(kudago_event['body_text']),

        'location': kudago_event['location']['slug'],
        'place': place['id'] if place else None,

        'age_restriction': kudago_event['age_restriction'],
        'price': transform_price(kudago_event['price'], kudago_event['is_free']),

        'favorites_count': kudago_event['favorites_count'],
        'comments_count': kudago_event['comments_count'],

        'images': kudago_event['images'],
        'dates': list(map(transform_date, dates)),

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_event['site_url'],
        }
    }


def transform_place(kudago_place):
    return {
        'id': kudago_place['id'],

        'is_closed': kudago_place['is_closed'],
        'is_stub': kudago_place['is_stub'],

        'name': kudago_place['short_title'],
        'full_name': kudago_place['title'],
        'lead': strip_links(kudago_place['description']),
        'description': strip_links(kudago_place['body_text']),

        'location': kudago_place['location'],
        'address': kudago_place['address'],
        'subway': kudago_place['subway'],
        'coords': kudago_place['coords'],

        'age_restriction': kudago_place['age_restriction'],
        'phone_numbers': kudago_place['phone'].split(','),
        'working_hours': kudago_place['timetable'],
        'url': kudago_place['foreign_url'],

        'favorites_count': kudago_place['favorites_count'],
        'comments_count': kudago_place['comments_count'],

        'images': kudago_place['images'],

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_place['site_url'],
        }
    }


def transform_stub_place(kudago_place):
    return {
        'id': kudago_place['id'],

        'is_closed': kudago_place['is_closed'],
        'is_stub': kudago_place['is_stub'],

        'full_name': kudago_place['title'],

        'location': kudago_place['location'],
        'address': kudago_place['address'],
        'subway': kudago_place['subway'],
        'coords': kudago_place['coords'],

        'phone_numbers': kudago_place['phone'].split(','),

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_place['site_url'],
        }
    }


# related objects

def transform_price(price_text, is_free):
    # TODO: extract from/to from the text
    return {
        'text': price_text,
    }


def transform_date(spec):
    start_date = date_from_timestamp(spec['start_date'])
    start_time = time_from_seconds(spec['start_time']) or time(0, 0)
    end_date = date_from_timestamp(spec['end_date']) or start_date
    end_time = time_from_seconds(spec['end_time']) or start_time

    is_continuous = spec['is_continuous']
    is_date_based = spec['start_time'] is None

    if is_continuous:
        start = datetime.combine(start_date, start_time)
        end = datetime.combine(end_date, end_time)
    else:
        overflows_into_next_day = (
            spec['end_time'] and
            spec['start_time'] and
            spec['end_time'] <= spec['start_time']
        )
        start = datetime.combine(start_date, start_time)<commit_msg>Properly transform coords into ES-readable format<commit_after>from funcy import flatten, some
from datetime import datetime, time, timedelta

from .utils import (
    find_first,
    date_from_timestamp,
    strip_links,
    time_from_seconds,
)


# main objects

def transform_event(kudago_event):
    tags = kudago_event['tags']
    categories = kudago_event['categories']
    place = kudago_event['place']

    type_ = find_first(('festival', 'exhibition', 'theater'), categories)
    dates = filter(is_date_finite, kudago_event['dates'])

    if type_ not in ('festival', 'exhibition'):
        dates = flatten(map(split_date, dates))

    return {
        'id': kudago_event['id'],
        'type': type_,
        'is_premiere': 'премьера' in tags,

        'name': kudago_event['short_title'],
        'full_name': kudago_event['title'],
        'tagline': kudago_event['tagline'],
        'lead': strip_links(kudago_event['description']),
        'description': strip_links(kudago_event['body_text']),

        'location': kudago_event['location']['slug'],
        'place': place['id'] if place else None,

        'age_restriction': kudago_event['age_restriction'],
        'price': transform_price(kudago_event['price'], kudago_event['is_free']),

        'favorites_count': kudago_event['favorites_count'],
        'comments_count': kudago_event['comments_count'],

        'images': kudago_event['images'],
        'dates': list(map(transform_date, dates)),

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_event['site_url'],
        }
    }


def transform_place(kudago_place):
    return {
        'id': kudago_place['id'],

        'is_closed': kudago_place['is_closed'],
        'is_stub': kudago_place['is_stub'],

        'name': kudago_place['short_title'],
        'full_name': kudago_place['title'],
        'lead': strip_links(kudago_place['description']),
        'description': strip_links(kudago_place['body_text']),

        'location': kudago_place['location'],
        'address': kudago_place['address'],
        'subway': kudago_place['subway'],
        'coords': transform_coords(kudago_place['coords']),

        'age_restriction': kudago_place['age_restriction'],
        'phone_numbers': kudago_place['phone'].split(','),
        'working_hours': kudago_place['timetable'],
        'url': kudago_place['foreign_url'],

        'favorites_count': kudago_place['favorites_count'],
        'comments_count': kudago_place['comments_count'],

        'images': kudago_place['images'],

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_place['site_url'],
        }
    }


def transform_stub_place(kudago_place):
    return {
        'id': kudago_place['id'],

        'is_closed': kudago_place['is_closed'],
        'is_stub': kudago_place['is_stub'],

        'full_name': kudago_place['title'],

        'location': kudago_place['location'],
        'address': kudago_place['address'],
        'subway': kudago_place['subway'],
        'coords': kudago_place['coords'],

        'phone_numbers': kudago_place['phone'].split(','),

        'source': {
            'name': 'KudaGo.com',
            'url': kudago_place['site_url'],
        }
    }


# related objects

def transform_price(price_text, is_free):
    # TODO: extract from/to from the text
    return {
        'text': price_text,
    }


def transform_coords(coords):
    if coords is None:
        return None

    lat, lon = coords.get('lat'), coords.get('lon')
    if not lat or not lon:
        return None

    return {
        'lat': float(lat),
        'lon': float(lon),
    }


def transform_date(spec):
    start_date = date_from_timestamp(spec['start_date'])
    start_time = time_from_seconds(spec['start_time']) or time(0, 0)
    end_date = date_from_timestamp(spec['end_date']) or start_date
    end_time = time_from_seconds(spec['end_time']) or start_time

    is_continuous = spec['is_continuous']
    is_date_based = spec['start_time'] is None

    if is_continuous:
        start = datetime.combine(start_date, start_time)
        end = datetime.combine(end_date, end_time)
    else:
        overflows_into_next_day = (
            spec['end_time'] and
            spec['start_time'] and
            spec['end_time'] <= spec['start_time']
        )
        start = datetime.combine(start_date, start_time)
{'fraction_non_alphanumeric': 0.14755650218909006, 'fraction_numerical': 0.0004733167672464797, 'mean_word_length': 3.182088075210292, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '946101', 'n_tokens_mistral': 2836, 'n_tokens_neox': 2752, 'n_words': 463}