<commit_before>#Import required packages
import pandas as pd
import numpy as np
import datetime
import os

def format_date(df_date):
    """
    Splits Meeting Times and Dates into separate columns.
    """
    df_date['Days'] = df_date['Meeting_Times'].str.extract('([^\s]+)', expand=True).astype(str)
    df_date['Start_Date'] = df_date['Meeting_Dates'].str.extract('([^\s]+)', expand=True)
    df_date['End_Date'] = df_date['Meeting_Dates'].str.extract('(?<=-)(.*)(?= )', expand=True)
    df_date['Start_Time'] = df_date['Meeting_Times'].str.extract('(?<= )(.*)(?=-)', expand=True).astype(str)
    df_date['End_Time'] = df_date['Meeting_Times'].str.extract('((?<=-).*$)', expand=True).astype(str)
    df_date['Building'] =  df_date['ROOM'].str.extract('([^\s]+)', expand=True).astype(str)

    for index, row in df_date.iterrows():
        
        try:
            start = int(row['Start_Time'][0:2])
            end = int(row['End_Time'][0:2])
            time_array = str(list(range(start, end+1)))
            df_date.loc[index, 'Time_Range'] = time_array          

        except:
            df_date.drop(index, inplace=True)
            continue 

        time_array = list(range(start, end+1))
        class_hr_range = list(range(6, 24))

        for class_hr in class_hr_range:
            for sched_hr in time_array:
                if sched_hr == class_hr:
                    df_date.loc[index, 'Hr_{0}'.format(str(sched_hr))] = False
                else:
                    df_date.loc[index, 'Hr_{0}'.format(str(sched_hr))] = True

    df_date.to_csv('test_sched.csv')
    return df_date

def save_to_csv(df_final):
    """
    Simplifies data file by saving only pertinent data
    """   
    df_final = df_final.loc[
        (df_final['Latitude'] != None) &
        (df_final['Longitude'] != None) &
        (df_final['Actual_Enrl'] > 0)]
    agg_operations = ({'Actual_Enrl' : 'sum',
                       'Latitude' : 'max',
                       'Longitude' : 'max'})
    df_final = df_final.groupby(['Building', 'Start_Time', 'End_Time', 'Days', 'Time_Range'], as_index=False).agg(agg_operations)
    days = ['M', 'T', 'W', 'R', 'F', 'S', 'U']
    for index, row in df_final.iterrows():
        for day in row['Days']:
            for match_day in days: # e.g. MWF in the string
                try:
                    if day == match_day:
                        df_final.loc[index, 'Day_{0}'.format(day)] = False
                    else:
                        df_final.loc[index, 'Day_{0}'.format(day)] = True
                except:
                    continue

    columns = ['Building', 'Actual_Enrl', 'Latitude', 'Longitude', 'Start_Time', 'End_Time', 'Time_Range']
    day_cols = [col for col in df_final.columns if 'Day' in col]
    for col in day_cols:
        columns.append(col) 
    class_hr_cols = [col for col in df_final.columns if 'Hr' in col]
    for col in class_hr_cols:
        columns.append(col)
    df_final.to_csv('full_sched.csv', columns=columns)


def join_coords(df_proc):
    filename = 'map_input/bldg_pt.csv'
    df_new = pd.read_csv(os.path.join(os.path.dirname(__file__), filename))
    df_coord = pd.merge(df_proc, df_new, left_on='Building', right_on='BUILDINGID', how='left')
    return df_coord

def main():
    """
    main program control.
    """
    pd.set_option('display.max_rows', None)  
    filename = 'map_input/PSU_master_classroom.csv'
    df = pd.read_csv(os.path.join(os.path.dirname(__file__), filename), dtype={'Schedule': object, 'Schedule Desc': object})   
    df = df.fillna('')

    terms = [201604]
    df = df.loc[df['Term'].isin(terms)]

    df = format_date(df)

    df = join_coords(df)

    # Avoid classes that only occur on a single day
    df = df.loc[df['Start_Date'] != df['End_Date']]
    df = df.loc[df['Online Instruct Method'] != 'Fully Online']
    save_to_csv(df)

if __name__ == '__main__':
    main()
<commit_msg>Working group by hour column functionality.<commit_after>#Import required packages
import pandas as pd
import numpy as np
import datetime
import os

def split_time(df_time):
    """
    Splits Meeting Times and Dates into separate columns.
    """
    df_time['Days'] = df_time['Meeting_Times'].str.extract('([^\s]+)', expand=True).astype(str)
    df_time['Start_Date'] = df_time['Meeting_Dates'].str.extract('([^\s]+)', expand=True)
    df_time['End_Date'] = df_time['Meeting_Dates'].str.extract('(?<=-)(.*)(?= )', expand=True)
    df_time['Start_Time'] = df_time['Meeting_Times'].str.extract('(?<= )(.*)(?=-)', expand=True).astype(str)
    df_time['End_Time'] = df_time['Meeting_Times'].str.extract('((?<=-).*$)', expand=True).astype(str)
    df_time['Building'] =  df_time['ROOM'].str.extract('([^\s]+)', expand=True).astype(str)

    for index, row in df_time.iterrows():
        
        try:
            start = int(row['Start_Time'][0:2])
            end = int(row['End_Time'][0:2])
            time_array = str(list(range(start, end+1)))
            df_time.loc[index, 'Time_Range'] = time_array          

        except:
            df_time.drop(index, inplace=True)
            continue 

        time_array = list(range(start, end+1))
        class_hr_range = list(range(6, 24))

        for class_hr in class_hr_range:
            for sched_hr in time_array:
                if sched_hr == class_hr:
                    df_time.loc[index, 'Hr_{0}'.format(str(sched_hr))] = False
                else:
                    df_time.loc[index, 'Hr_{0}'.format(str(sched_hr))] = True

    #df_time.to_csv('test_sched.csv')
    return df_time

def group_hour(df_all_hours):
    """
    Loops through every Hr column and groups by hour; output is a final combined
    dataframe.
    """
    # an empty dataframe:
    df_by_hr = pd.DataFrame()
    group_hours = list(range(6, 24))
    for hr in group_hours:
        agg_hr_operations = ({'Actual_Enrl' : 'sum'})
        df_temp = df_all_hours.groupby(['Building', 'Days', 'Hr_{0}'.format(str(hr))], as_index=False).agg(agg_hr_operations)
        df_by_hr = df_by_hr.append(df_temp, ignore_index = True)
    
    df_by_hr.to_csv('test_group_hr.csv')
    return df_by_hr


def save_to_csv(df_final):
    """
    Simplifies data file by saving only pertinent data
    """   
    df_final = df_final.loc[
        (df_final['Latitude'] != None) &
        (df_final['Longitude'] != None) &
        (df_final['Actual_Enrl'] > 0)]
    agg_operations = ({'Actual_Enrl' : 'sum',
                       'Latitude' : 'max',
                       'Longitude' : 'max'})
    df_final = df_final.groupby(['Building', 'Start_Time', 'End_Time', 'Days', 'Time_Range'], as_index=False).agg(agg_operations)
    days = ['M', 'T', 'W', 'R', 'F', 'S', 'U']
    for index, row in df_final.iterrows():
        for day in row['Days']:
            for match_day in days: # e.g. MWF in the string
                try:
                    if day == match_day:
                        df_final.loc[index, 'Day_{0}'.format(day)] = False
                    else:
                        df_final.loc[index, 'Day_{0}'.format(day)] = True
                except:
                    continue

    columns = ['Building', 'Actual_Enrl', 'Latitude', 'Longitude', 'Start_Time', 'End_Time', 'Time_Range']
    day_cols = [col for col in df_final.columns if 'Day' in col]
    for col in day_cols:
        columns.append(col) 
    class_hr_cols = [col for col in df_final.columns if 'Hr' in col]
    for col in class_hr_cols:
        columns.append(col)
    df_final.to_csv('full_sched.csv', columns=columns)


def join_coords(df_proc):
    filename = 'map_input/bldg_pt.csv'
    df_new = pd.read_csv(os.path.join(os.path.dirname(__file__), filename))
    df_coord = pd.merge(df_proc, df_new, left_on='Building', right_on='BUILDINGID', how='left')
    return df_coord

def main():
    """
    main program control.
    """
    pd.set_option('display.max_rows', None)  
    filename = 'map_input/PSU_master_classroom.csv'
    df = pd.read_csv(os.path.join(os.path.dirname(__file__), filename), dtype={'Schedule': object, 'Schedule Desc': object})   
    df = df.fillna('')

    terms = [201604]
    df = df.loc[df['Term'].isin(terms)]
    df = split_time(df)
    df = group_hour(df)
    df = join_coords(df)

    # Avoid classes that only occur on a single day
    df = df.loc[df['Start_Date'] != df['End_Date']]
    df = df.loc[df['Online Instruct Method'] != 'Fully Online']
    #save_to_csv(df)

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.15980172312050042, 'fraction_numerical': 0.005192965891655848, 'mean_word_length': 2.7085339168490155, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3465871', 'n_tokens_mistral': 2993, 'n_tokens_neox': 2895, 'n_words': 583}