<commit_before># -*- coding: utf-8 -*-

import os
import datetime
import simplejson
from skylines import files
from tg import config
from skylines.model.geo import Location
import logging

log = logging.getLogger(__name__)


def helper_path(helper):
    return os.path.join(config['skylines.analysis.path'], helper)


def import_location_attribute(node, name):
    if node is None or name not in node:
        return None

    location = node[name]
    if 'latitude' not in location or 'longitude' not in location:
        return None

    try:
        latitude = float(location['latitude'])
        longitude = float(location['longitude'])
        return Location(latitude=latitude, longitude=longitude)
    except ValueError:
        return None


def import_datetime_attribute(node, name):
    if node is None or name not in node:
        return None
    try:
        return datetime.datetime.strptime(node[name], '%Y-%m-%dT%H:%M:%SZ')
    except ValueError:
        return None


def find_contest(root, name):
    if 'contests' not in root:
        return None

    if name not in root['contests']:
        return None

    return root['contests'][name]


def find_trace(contest, name):
    if not name in contest:
        return None

    return contest[name]


def analyse_flight(flight):
    path = files.filename_to_path(flight.igc_file.filename)
    log.info('Analyzing ' + path)

    root = None
    with os.popen(helper_path('AnalyseFlight') + ' "' + path + '"') as f:
        try:
            root = simplejson.load(f)
        except simplejson.JSONDecodeError:
            log.error('Parsing the output of AnalyseFlight failed.')

            if log.isEnabledFor(logging.DEBUG):
                with os.popen(helper_path('AnalyseFlight') + ' "' + path + '"') as f_debug:
                    log.debug(helper_path('AnalyseFlight') +
                              ' "' + path + '" = ' + str(f_debug.readlines()))

            return False

    if not root:
        return False

    times = root['times'] if 'times' in root else None
    flight.takeoff_time = import_datetime_attribute(times, "takeoff")
    flight.landing_time = import_datetime_attribute(times, "landing")

    locations = root['locations'] if 'locations' in root else None
    flight.takeoff_location = import_location_attribute(locations, 'takeoff')
    flight.landing_location = import_location_attribute(locations, 'landing')

    contest = find_contest(root, 'olc_plus')
    if contest is not None:
        trace = find_trace(contest, 'classic')
        if trace is not None and 'distance' in trace:<commit_msg>lib.analysis: insert takeoff airport if autodetectable<commit_after># -*- coding: utf-8 -*-

import os
import datetime
import simplejson
from skylines import files
from tg import config
from skylines.model.geo import Location
from skylines.model import Airport
import logging

log = logging.getLogger(__name__)


def helper_path(helper):
    return os.path.join(config['skylines.analysis.path'], helper)


def import_location_attribute(node, name):
    if node is None or name not in node:
        return None

    location = node[name]
    if 'latitude' not in location or 'longitude' not in location:
        return None

    try:
        latitude = float(location['latitude'])
        longitude = float(location['longitude'])
        return Location(latitude=latitude, longitude=longitude)
    except ValueError:
        return None


def import_datetime_attribute(node, name):
    if node is None or name not in node:
        return None
    try:
        return datetime.datetime.strptime(node[name], '%Y-%m-%dT%H:%M:%SZ')
    except ValueError:
        return None


def find_contest(root, name):
    if 'contests' not in root:
        return None

    if name not in root['contests']:
        return None

    return root['contests'][name]


def find_trace(contest, name):
    if not name in contest:
        return None

    return contest[name]


def analyse_flight(flight):
    path = files.filename_to_path(flight.igc_file.filename)
    log.info('Analyzing ' + path)

    root = None
    with os.popen(helper_path('AnalyseFlight') + ' "' + path + '"') as f:
        try:
            root = simplejson.load(f)
        except simplejson.JSONDecodeError:
            log.error('Parsing the output of AnalyseFlight failed.')

            if log.isEnabledFor(logging.DEBUG):
                with os.popen(helper_path('AnalyseFlight') + ' "' + path + '"') as f_debug:
                    log.debug(helper_path('AnalyseFlight') +
                              ' "' + path + '" = ' + str(f_debug.readlines()))

            return False

    if not root:
        return False

    times = root['times'] if 'times' in root else None
    flight.takeoff_time = import_datetime_attribute(times, "takeoff")
    flight.landing_time = import_datetime_attribute(times, "landing")

    locations = root['locations'] if 'locations' in root else None
    flight.takeoff_location = import_location_attribute(locations, 'takeoff')
    flight.landing_location = import_location_attribute(locations, 'landing')

    flight.takeoff_airport = Airport.by_location(flight.takeoff_location)

    contest = find_contest(root, 'olc_plus')
    if contest is not None:
        trace = find_trace(contest, 'classic')
        if trace is not None and 'distance' in trace:
{'fraction_non_alphanumeric': 0.1016663546152406, 'fraction_numerical': 0.0003744617112900206, 'mean_word_length': 2.9250551065393093, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '323558', 'n_tokens_mistral': 1600, 'n_tokens_neox': 1542, 'n_words': 434}