<commit_before>#!/usr/bin/env python

import bs4 as bs
import HTMLParser
import csv
from urllib2 import Request, urlopen, URLError

dict = {}
dict_list = {}

def schedule_Picker(line):
    return{
        'mfl_69_w' : "http://www.septa.org/schedules/transit/w/MFL_1.htm",
        'mfl_69_sat' : "http://www.septa.org/schedules/transit/s/MFL_1.htm",
        'mfl_69_sun' : "http://www.septa.org/schedules/transit/h/MFL_1.htm",
        'mfl_f_w' : "http://www.septa.org/schedules/transit/w/MFL_0.htm",
        'mfl_f_sat' : "http://www.septa.org/schedules/transit/s/MFL_0.htm",
        'mfl_f_sun' : "http://www.septa.org/schedules/transit/h/MFL_0.htm",
	'bsl_fern_w' : "http://www.septa.org/schedules/transit/w/BSL_1.htm",
	'bsl_fern_sat' : "http://www.septa.org/schedules/transit/s/BSL_1.htm",
	'bsl_fern_sun' : "http://www.septa.org/schedules/transit/h/BSL_1.htm",
	'bsl_att_w' : "http://www.septa.org/schedules/transit/w/BSL_0.htm",
	'bsl_att_sat' : "http://www.septa.org/schedules/transit/s/BSL_0.htm",
	'bsl_att_sun' : "http://www.septa.org/schedules/transit/h/BSL_0.htm",
        }.get(line,'error')


def source(url_request):
    site = Request(url_request) 
    try:
        source = urlopen(site).read()
    except URLError, e:
        print "Didn't work, got error:", e
    return(source)


def soupify(source):
    soup = bs.BeautifulSoup(source,'html.parser')
    return(soup)


def get_Times(soup):
    for tags in soup.find_all('td'):
        print(tags.string)


## Create a separate list to try to get the ordering correct
def get_ID_Station(soup):
    temp = []
    table = soup.find('table', id="timeTable")
    
    for rows in table.findAll('th'):
        name = rows.text
        data = name.split("StopID:")
        temp.append(data[0])
        dict[data[1]] = data[0]

    list_half = temp[:len(temp)/2]
    return list_half

## Save the csv files with a time stamp
## have a separate script to upload the data
def csv_Writing(output):
    with open('test.csv', 'wb') as csvfile:
        spamwriter = csv.writer(csvfile, delimiter = '|')


def get_Time_Station(soup, list_half):
    print(list_half)
    for key, value in dict.iteritems():
        value2 = value.decode('utf-8')
        dict_list['%s' % value2] = []
    
    for key, value in dict.iteritems():
        time = soup.find('td', title=value)
        for tags in time.findAll('td'):
            #print(tags.string)
            temp = tags.string.replace(u'\xa0','')
            temp2 = temp.replace(u'\u2014', '-')
            dict_list['%s' % value].append(temp2)
    
    with open('test.csv', 'wb') as outfile:
        writer = csv.writer(outfile)
        writer.writerow(dict_list.keys())
        writer.writerows(zip(*dict_list.values()))
    
        

def main():
    website = schedule_Picker('mfl_f_w')
    main_source = source(website)
    soup = soupify(main_source)
    list_half = get_ID_Station(soup)
    get_Time_Station(soup, list_half)

def lambda_handler(event, context):
    print event
    main()


if __name__ == "__main__":
    main()
<commit_msg>Creating a csv for each line<commit_after>#!/usr/bin/env python

import bs4 as bs
import HTMLParser
import csv
import os.path
import time
from urllib2 import Request, urlopen, URLError

dict = {}
dict_list = {}
lines = {
    'mfl_69_w' : "http://www.septa.org/schedules/transit/w/MFL_1.htm",
    'mfl_69_sat' : "http://www.septa.org/schedules/transit/s/MFL_1.htm",
    'mfl_69_sun' : "http://www.septa.org/schedules/transit/h/MFL_1.htm",
    'mfl_f_w' : "http://www.septa.org/schedules/transit/w/MFL_0.htm",
    'mfl_f_sat' : "http://www.septa.org/schedules/transit/s/MFL_0.htm",
    'mfl_f_sun' : "http://www.septa.org/schedules/transit/h/MFL_0.htm",
    'bsl_fern_w' : "http://www.septa.org/schedules/transit/w/BSL_1.htm",
    'bsl_fern_sat' : "http://www.septa.org/schedules/transit/s/BSL_1.htm",
    'bsl_fern_sun' : "http://www.septa.org/schedules/transit/h/BSL_1.htm",
    'bsl_att_w' : "http://www.septa.org/schedules/transit/w/BSL_0.htm",
    'bsl_att_sat' : "http://www.septa.org/schedules/transit/s/BSL_0.htm",
    'bsl_att_sun' : "http://www.septa.org/schedules/transit/h/BSL_0.htm",
    }

'''
def schedule_Picker(line):
    return{
        'mfl_69_w' : "http://www.septa.org/schedules/transit/w/MFL_1.htm",
        'mfl_69_sat' : "http://www.septa.org/schedules/transit/s/MFL_1.htm",
        'mfl_69_sun' : "http://www.septa.org/schedules/transit/h/MFL_1.htm",
        'mfl_f_w' : "http://www.septa.org/schedules/transit/w/MFL_0.htm",
        'mfl_f_sat' : "http://www.septa.org/schedules/transit/s/MFL_0.htm",
        'mfl_f_sun' : "http://www.septa.org/schedules/transit/h/MFL_0.htm",
        'bsl_fern_w' : "http://www.septa.org/schedules/transit/w/BSL_1.htm",
        'bsl_fern_sat' : "http://www.septa.org/schedules/transit/s/BSL_1.htm",
        'bsl_fern_sun' : "http://www.septa.org/schedules/transit/h/BSL_1.htm",
        'bsl_att_w' : "http://www.septa.org/schedules/transit/w/BSL_0.htm",
        'bsl_att_sat' : "http://www.septa.org/schedules/transit/s/BSL_0.htm",
        'bsl_att_sun' : "http://www.septa.org/schedules/transit/h/BSL_0.htm",
    }.get(line,'error')
'''

def csv_Check(line_name, date):
    csv_name = line_name + date
    if os.path.isfile(csv_name):
        return True
    else:
        return False


def source(url_request):
    site = Request(url_request) 
    try:
        source = urlopen(site).read()
    except URLError, e:
        print "Didn't work, got error:", e
    return(source)


def soupify(source):
    soup = bs.BeautifulSoup(source,'html.parser')
    return(soup)


def get_Times(soup):
    for tags in soup.find_all('td'):
        print(tags.string)


## Create a separate list to try to get the ordering correct
def get_ID_Station(soup):
    temp = []
    table = soup.find('table', id="timeTable")
    
    for rows in table.findAll('th'):
        name = rows.text
        data = name.split("StopID:")
        if data[0] == 'AT&T;':
            data[0] = 'AT&T'
        temp.append(data[0])
        dict[data[1]] = data[0]

    list_half = temp[:len(temp)/2]
    return list_half


## Save the csv files with a time stamp
## have a separate script to upload the data
def csv_Writing(output):
    with open('test.csv', 'wb') as csvfile:
        spamwriter = csv.writer(csvfile, delimiter = '|')


def get_Time_Station(soup, list_half, file_ending, key_name):
    for key, value in dict.iteritems():
        value2 = value.decode('utf-8')
        dict_list['%s' % value2] = []
    
    for key, value in dict.iteritems():
        time = soup.find('td', title=value)        
        for tags in time.findAll('td'):
            #print(tags.string)
            temp = tags.string.replace(u'\xa0','')
            temp2 = temp.replace(u'\u2014', '-')
            dict_list['%s' % value].append(temp2)
    
    csv_file_name = key_name + file_ending
    with open(csv_file_name, 'wb') as outfile:
        writer = csv.writer(outfile)
        writer.writerow(dict_list.keys())
        writer.writerows(zip(*dict_list.values()))
    
        
def main():
    current_time = time.strftime("%m_%Y")
    file_ending = "_" + current_time + ".csv"


    for key, value in lines.iteritems():
        if csv_Check(key, file_ending):
            print(key + file_ending + " already exists")
        else:
            main_source = source(value)
            soup = soupify(main_source)
            list_half = get_ID_Station(soup)
            get_Time_Station(soup, list_half, file_ending, key)
            dict.clear()



def lambda_handler(event, context):
    print event
    main()


if __name__ == "__main__":
    main()

{'fraction_non_alphanumeric': 0.16153140159958043, 'fraction_numerical': 0.011537957257112888, 'mean_word_length': 2.8273958855995986, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 36, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4585965', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2912, 'n_words': 467}