<commit_before>import re
import datetime
import json


def parse_block_schedule_into_blocks(fname):
    """
    Parse block_schedule into blocks of lines.
    
    :param fname: 
        Path to block_schedule file.
    :return: 
        List of blocks with lines containing information about observation.
    """
    with open(fname, 'r') as fo:
        lines = fo.readlines()

    for line in lines:
        if 'Version' in line:
            break
    version_line_idx = lines.index(line)

    lines = lines[version_line_idx+1:]

    if lines.index('\r\n') == 0:
        lines = lines[1:]

    observations = list()
    while True:
        idx = lines.index('\r\n')
        block = lines[:idx]
        if block[0].startswith('Observational') or\
            block[0].startswith('#') or\
            block[0].startswith('Comments:'):
            observations.append(block)
        lines = lines[idx+1:]
        if not lines:
            break

    return observations


def classify_block(block):
    if block[0].startswith('Comments: Lavochkin shortened start time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortened stop time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Observational code:'):
        rank = 0
    elif block[0].startswith('###Cancelled: poor'):
        rank = 0
    elif block[0].startswith('###Cancelled by Lavochkin'):
        rank = 1
    else:
        print block
        raise Exception("Check unknown block starting")
    return rank


def get_ut_source_from_block(block):
    for line in block:
        if 'Start(UT)' in line:
            ut_time = "{} {}".format(line.strip().split()[1],
                                     line.strip().split()[2])
            ut_datetime = datetime.strptime(ut_time, '%d.%m.%Y %H:%M:%S')
        if 'Source:' in line:
            source = line.strip().split()[-1]
    return ut_datetime, source


def parse_racat(fname):
    """
    :return: 
        Dictionary with keys - source names and values - tuples of RA & DEC.
    """

    source_coordinates = dict()

    with open(fname, 'r') as fo:
        lines = fo.readlines()
        for line in lines:
            if line.strip().startswith('!'):
                continue
            try:
                name = re.findall(r".+source='(\S+)'", line)[0]
                ra = re.findall(r".+RA=(\S+) ", line)[0]
                dec = re.findall(r".+DEC=(\S+) ", line)[0]
                source_coordinates.update({str(name): (ra, dec)})
            except IndexError:
                pass

    return source_coordinates


def dump_source_coordinates(fname, outfname):
    """
    Dump source coordinates to json format.
    
    :param fname: 
        Path to RA catalouge.
    :param outfname:
        Path to dump file.
    """
    source_coordinates = parse_racat(fname)
    with open(outfname, 'w') as fo:
        json.dump(source_coordinates, fo)


def load_source_coordinates(fname):
    """
    Load source coordinates dictionary from json-file.
    :return: 
        Dictionary with keys - source names and values - lists of RA & DEC.
    """
    with open(fname, 'r') as fo:
        source_coordinates = json.load(fo)
    return source_coordinates


def datetime_to_fractional_year(dt):
    return  (float(dt.strftime("%j"))-1) / 366

<commit_msg>fetching frac.year, ra, dec, rank from all block schedules<commit_after>import re
import datetime
import pickle
from astropy import units as u
from astropy.coordinates import SkyCoord
import pandas as pd


def parse_block_schedule_into_blocks(fname):
    """
    Parse block_schedule into blocks of lines.
    
    :param fname: 
        Path to block_schedule file.
    :return: 
        List of blocks with lines containing information about observation.
    """
    with open(fname, 'r') as fo:
        lines = fo.readlines()

    for line in lines:
        if 'Version' in line:
            break
    version_line_idx = lines.index(line)

    lines = lines[version_line_idx+1:]

    # This marks that \r\n used for separation of blocks
    use_n = False
    try:
        if lines.index('\r\n') == 0:
            lines = lines[1:]
    except ValueError:
        use_n = True
        if lines.index('\n') == 0:
            lines = lines[1:]

    observations = list()
    while True:
        try:
            if not use_n:
                idx = lines.index('\r\n')
            else:
                idx = lines.index('\n')
        except ValueError:
            break
        block = lines[:idx]
        print(block)
        if not block:
            break
        if block[0].startswith('Observational') or\
            block[0].startswith('#') or\
            block[0].startswith('Comments:'):
            observations.append(block)
        lines = lines[idx+1:]
        if not lines:
            break

    return observations


# FIXME: After poor GRT support sometimes goes comment about Lavochkin
def classify_block(block):
    if block[0].startswith('Comments: Lavochkin shortened start time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortened stop time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shrtened stop time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortened end time on'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortens the'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortened'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: shortened for'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('#Comments: Lavochkin cancelled'):
        rank = 1
    elif block[0].startswith('###Comments: Lavochkin cancelled'):
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shifted'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments:Lavochkin shifted'):
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('###Cancelled: 40min too short here'):
        rank = 1
    elif block[0].startswith('Comments: Lavochkin shortened the segment'):
        rank = 1
    elif block[0].startswith('Comments: Lavochkin will give answer later'):
        rank = 1
    elif block[0].startswith('Comments: Lavochkin prolong'):
        rank = 0
    elif block[0].startswith('Comments: Lavochkin extended'):
        rank = 0
    elif block[0].startswith('Observational code:'):
        rank = 0
    elif block[0].startswith('###Cancelled: poor'):
        rank = 0
    elif block[0].startswith('### Cancelled: por'):
        rank = 0
    elif block[0].startswith('###Cancelled by the grav team'):
        rank = 0
    elif block[0].startswith('### Cancelled: poor GRT support'):
        rank = 0
    elif block[0].startswith('### Cancelled: need more GRTs'):
        rank = 0
    elif block[0].startswith('### Cancelled: poor ground support'):
        rank = 0
    elif block[0].startswith('###Cancelled -- poor GRT support'):
        rank = 0
    elif block[0].startswith('###Cancelled since no GBT support'):
        rank = 0
    elif block[0].startswith('###Cancelled: no ground support'):
        rank = 0
    elif block[0].startswith('#Comments: No Ground support'):
        rank = 0
    elif block[0].startswith('###Cancelled: no GRTs'):
        rank = 0
    elif block[0] == '###Cancelled\n':
        rank = 0
    elif block[0].startswith('###Cancelled: PuTS not ready'):
        rank = 0
    elif block[0].startswith('###Cancelled: no GRT support'):
        rank = 0
    elif block[0].startswith('### Cancelled: no ground support'):
        rank = 0
    elif block[0].startswith('### Cancelled: limited GRT support'):
        rank = 0
    elif block[0].startswith('###Cancelled: GBT not available'):
        rank = 0
    elif block[0].startswith('Comments: Observational time changed from'):
        rank = 0
    elif block[0].startswith('###Cancelled: not enough GRT support'):
        rank = 0
    elif block[0].startswith('###Cancelled: not enoiugh GRT support'):
        rank = 0
    elif block[0].startswith('###Cancelled: not tnough GRT support'):
        rank = 0
    elif block[0].startswith('###Cancelled by Lavochkin'):
        rank = 1
    elif block[0].startswith('#Cancelled by Lavochkin'):
        rank = 1
    elif block[0].startswith('### Cancelled by Lavochkin'):
        rank = 1
    elif block[0].startswith('###Cancelld by Lavochkin'):
        rank = 1
    elif ', Lavochkin shortened' in block[0]:
        shortened_time_mins = block[0].strip().split()[-2]
        rank = 1
    elif block[0].startswith('Comments: start time shifted by 30 min -- GRTs: check, please, that it is OK'):
        rank = 0
    else:
        print block
        raise Exception("Check unknown block starting")
    return rank


def get_ut_source_from_block(block):
    for line in block:
        if 'Start(UT)' in line:
            ut_time = "{} {}".format(line.strip().split()[1],
                                     line.strip().split()[2])
            ut_datetime = datetime.datetime.strptime(ut_time, '%d.%m.%Y %H:%M:%S')
        if 'ource:' in line:
            # Take second word because there could be alternative names
            source = line.strip().split()[1]
            # Why they use commas???
            source = source.rstrip(',')

            source_alt = line.strip().split()[-1]
            source_alt = source_alt.rstrip(')')
            source_alt = source_alt.lstrip('(')
    return ut_datetime, source, source_alt


def parse_racat(fname):
    """
    :return: 
        Dictionary with keys - source names and values - instances of
        ``astropy.coordinates.SkyCoord``.
    """

    source_coordinates = dict()

    with open(fname, 'r') as fo:
        lines = fo.readlines()
        for line in lines:
            # if line.strip().startswith('!'):
            #     continue
            try:
                name = re.findall(r".+source='(\S+)'", line)[0]
                ra = re.findall(r".+RA=(\S+) ", line)[0]
                dec = re.findall(r".+DEC=(\S+) ", line)[0]
                source_coordinates.update({str(name):
                                           SkyCoord(ra + ' ' + dec,
                                                    unit=(u.hourangle, u.deg))})
            except IndexError:
                pass

    return source_coordinates


def dump_source_coordinates(ra_cat_fname, outfname):
    """
    Pickle source coordinates to file.
    
    :param ra_cat_fname: 
        Path to RA catalouge.
    :param outfname:
        Path to dump file.
    """
    source_coordinates = parse_racat(ra_cat_fname)
    with open(outfname, 'wb') as fo:
        pickle.dump(source_coordinates, fo)


def load_source_coordinates(fname):
    """
    Load source coordinates dictionary from pickle-file.
    :return: 
        Dictionary with keys - source names and values - instances of
        ``astropy.coordinates.SkyCoord``.
    """
    with open(fname, 'rb') as fo:
        source_coordinates = pickle.load(fo)
    return source_coordinates


def datetime_to_fractional_year(dt):
    return (float(dt.strftime("%j"))-1) / 366


def create_features_responces_dataset(block_sched_files, ra_cat_pkl):
    features_responces = list()
    columns = ('frac_year', 'ra', 'dec', 'rank')
    source_coordinates = load_source_coordinates(ra_cat_pkl)
    for block_sched_file in block_sched_files:
        print("Parsing block-sched file {}".format(block_sched_file))
        blocks = parse_block_schedule_into_blocks(block_sched_file)
        for block in blocks:
            print("Parsing block :")
            print(block)
            rank = classify_block(block)
            try:
                ut_dt, source, source_alt = get_ut_source_from_block(block)
            # FIXME: ??? in coordinates of block schedule
            except ValueError:
                continue
            print source, source_alt
            frac_year = datetime_to_fractional_year(ut_dt)
            try:
                sky_coord = source_coordinates[source]
            except KeyError:
                try:
                    sky_coord = source_coordinates[source_alt]
                # Some weird maser coordinates
                except KeyError:
                    continue
            features_responces.append((frac_year, sky_coord.ra, sky_coord.dec,
                                       rank))
    features_responces = pd.DataFrame.from_records(features_responces,
                                                   columns=columns)
    return features_responces


if __name__ == '__main__':
    import os
    import glob
    block_sched_dir = '/home/ilya/Dropbox/scheduling/block_schedules'
    block_sched_files = glob.glob(os.path.join(block_sched_dir,
                                               '*_block_schedule.*'))
    dump_source_coordinates('/home/ilya/code/as/Radioastron_Input_Catalog_v054.txt',
                            'RA_cat_v054.pkl')
    fr = create_features_responces_dataset(block_sched_files,
                                           'RA_cat_v054.pkl')


{'fraction_non_alphanumeric': 0.12210325608682898, 'fraction_numerical': 0.01261366969785861, 'mean_word_length': 2.195922193578627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1728741', 'n_tokens_mistral': 4214, 'n_tokens_neox': 4027, 'n_words': 1015}