<commit_before># -*- coding: utf-8 -*-

from __future__ import absolute_import

import re
from datetime import date, datetime
from skylines.lib import files, base36
from skylines.lib.string import import_ascii, import_alnum

hfdte_re = re.compile(r'HFDTE(\d{6})', re.IGNORECASE)
hfgid_re = re.compile(r'HFGID\s*GLIDER\s*ID\s*:(.*)', re.IGNORECASE)
hfgty_re = re.compile(r'HFGTY\s*GLIDER\s*TYPE\s*:(.*)', re.IGNORECASE)
hfcid_re = re.compile(r'HFCID.*:(.*)', re.IGNORECASE)
afil_re = re.compile(r'AFIL(\d*)FLIGHT', re.IGNORECASE)


def read_igc_headers(filename):
    path = files.filename_to_path(filename)

    try:
        f = file(path, 'r')
    except IOError:
        return None

    igc_headers = dict()
    a_record_found = False

    for i, line in enumerate(f):
        if not a_record_found:
            if line[0] != 'A' or len(line) < 4:
                # if the first record of the file is not an
                # A record the IGC file is broken
                return None

            a_record_found = True
            igc_headers['manufacturer_id'] = line[1:4]

            if len(line) >= 7:
                igc_headers['logger_id'] = parse_logger_id(line)

        if line.startswith('HFDTE'):
            igc_headers['date_utc'] = parse_date(line)

        if line.startswith('HFGTY'):
            igc_headers['model'] = parse_pattern(hfgty_re, line)

        if line.startswith('HFGID'):
            igc_headers['reg'] = parse_pattern(hfgid_re, line)

        if line.startswith('HFCID'):
            igc_headers['cid'] = parse_pattern(hfcid_re, line)

        # don't read more than 100 lines, that should be enough
        if i > 100:
            break

    return igc_headers


def parse_logger_id(line):
    # non IGC loggers may use more than 3 characters as unique ID.
    # we just use the first three, currently no model is known which
    # really uses more than 3 characters.

    if line[1:4] == 'FIL':
        # filser doesn't respect the IGC file specification and
        # stores the unique id as base 10 instead of base 36.<commit_msg>lib/igc: Don't break on missing logger information<commit_after># -*- coding: utf-8 -*-

from __future__ import absolute_import

import re
from datetime import date, datetime
from skylines.lib import files, base36
from skylines.lib.string import import_ascii, import_alnum

hfdte_re = re.compile(r'HFDTE(\d{6})', re.IGNORECASE)
hfgid_re = re.compile(r'HFGID\s*GLIDER\s*ID\s*:(.*)', re.IGNORECASE)
hfgty_re = re.compile(r'HFGTY\s*GLIDER\s*TYPE\s*:(.*)', re.IGNORECASE)
hfcid_re = re.compile(r'HFCID.*:(.*)', re.IGNORECASE)
afil_re = re.compile(r'AFIL(\d*)FLIGHT', re.IGNORECASE)


def read_igc_headers(filename):
    path = files.filename_to_path(filename)

    try:
        f = file(path, 'r')
    except IOError:
        return None

    igc_headers = dict()

    for i, line in enumerate(f):
        if line[0] == 'A':
            length = len(line)

            if length >= 4:
                igc_headers['manufacturer_id'] = line[1:4]

            if length >= 7:
                igc_headers['logger_id'] = parse_logger_id(line)

        if line.startswith('HFDTE'):
            igc_headers['date_utc'] = parse_date(line)

        if line.startswith('HFGTY'):
            igc_headers['model'] = parse_pattern(hfgty_re, line)

        if line.startswith('HFGID'):
            igc_headers['reg'] = parse_pattern(hfgid_re, line)

        if line.startswith('HFCID'):
            igc_headers['cid'] = parse_pattern(hfcid_re, line)

        # don't read more than 100 lines, that should be enough
        if i > 100:
            break

    return igc_headers


def parse_logger_id(line):
    # non IGC loggers may use more than 3 characters as unique ID.
    # we just use the first three, currently no model is known which
    # really uses more than 3 characters.

    if line[1:4] == 'FIL':
        # filser doesn't respect the IGC file specification and
        # stores the unique id as base 10 instead of base 36.
{'fraction_non_alphanumeric': 0.11698396793587175, 'fraction_numerical': 0.011523046092184368, 'mean_word_length': 2.673413063477461, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442613', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1346, 'n_words': 353}