<jupyter_start><jupyter_text>Title<jupyter_code>r"""markdown
    TITLE   :
    AUTHOR  :
    PROJECT :
""";

__author__ = ''
__version__ = ''<jupyter_output><empty_output><jupyter_text>About - - - - - -   Prepare  Imports<jupyter_code>## General
# import requests
# from bs4 import BeautifulSoup
# import flask
import pandas as pd

## Custom
from IPython.display import HTML

## Project-Specific<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def scrape_table(inputurl, columns=[], rename={}, sort=None, parse=True,
                 # dates
                 date_col='date',
                 to_next_date=False, is_start_date=True,
                 # saving
                 save=False, combine_to_master=True,
                ):

    # get get from html
    dflist = pd.read_html(inputurl, attrs={"class": ["wikitable","wikitable sortable"]})
    # concatenate and sort
    df = concatenate_tables(*dflist, sort=sort)
    
    # get right columns
    df = df[columns]
    df.rename(columns=rename, inplace=True)

    # add source
    df['source'] = inputurl

    # parse date
    if parse:
        df = parse_dates(df, date_col=date_col)

    # deal with start and end times
    if is_start_date is True:
        if to_next_date is False:  # single date
            df['start_date'] = df[date_col]
            df['end_date'] = df[date_col]
        else: # date range
            df['start_date'] = df[date_col]
            df['end_date'] = [*df[date_col][1:], 0]
    else:
        pass
#         df['start_date'] = df[date_col] - 
#         df['end_date'] = df[date_col] +
    
    if isinstance(save, str):
        df.to_csv(save, index=False)
    if combine_to_master:
        add_to_master_table(df, sort=sort)
    return df


def concatenate_tables(*dfs, sort=None):
    cdf = pd.concat(list(dfs))
    if sort is not None:
        cdf.sort_values(sort, inplace=True)
    cdf.reset_index(drop=True, inplace=True)
    return cdf


def add_to_master_table(df, sort):
    try:
        mdf = pd.read_csv('../data/master_table.csv')
    except:
        df.to_csv('../data/master_table.csv', index=False)
    else:
        new_df = concatenate_tables(mdf, df, sort='date')
        new_df.to_csv('../data/master_table.csv', index=False)
    return df

def parse_dates(df, date_col='date'):
    
    def parse_start(date):
        date = date.strip()
        if date.startswith('by '):
            return date[2:]
        elif date.startswith('c.'):
            return date[2:]
        else:
            return date
        
    def parse_parenthesis(date):
        date = date.strip()
        if '(' in date:
            return date.split('(')[0]
        else:
            return date
    
    def parse_unit(date):
        """return unit in years"""    
        date = date.strip()
        if date.endswith('ka'):
            return date[:-2], 1e3
        # mega
        elif date.endswith('ma'):
            return date[:-2], 1e6
        elif date.endswith('ma-'):
            return date[:-3], 1e6
        # giga
        elif date.endswith('bya'):
            return date[:-3], 1e9
        elif date.endswith('ya'):  # after bya b/c ya in bya
            return date[:-2], 1
        else:
            return date, 1
    # /def
    
    
    def parse_range(date):
        date = date.strip()
        if ('-' in date):
            s = date.split('-')[0]
            np.mean([float(s[0]), float(s[1])])
        elif ('–' in date):
            s = date.split('–')
            return np.mean([float(s[0]), float(s[1])])
        else:
            return date
    
    
    dates = df[date_col]
    
    for i, date in dates.iteritems():
        
        date = date.lower().strip()
        # parse unit
        date = parse_start(date)
        date = parse_parenthesis(date)
        date, unit = parse_unit(date)
        # parse range
        date = parse_range(date)
        
        # assign
        dates.iloc[i] = float(date) * unit
    
    # reassigning
    df[date_col] = dates
    return df<jupyter_output><empty_output><jupyter_text>- - - - - -   Code<jupyter_code>df = scrape_table("https://en.wikipedia.org/wiki/Timeline_of_the_evolutionary_history_of_life",
                  columns=['Date', 'Event'],
                  rename={'Date': 'date', 'Event': 'event'},
                  date_col='date',
                  save='../data/wikipedia_evolutionary_history_of_life.csv',
                  sort='Date',
                  parse=True
)

df
df = scrape_table("https://en.wikipedia.org/wiki/Timeline_of_natural_history",
                  columns=['Started', 'Period'],
                  rename={'Started': 'date', 'Period': 'event'},
                  date_col='date',
                  to_next_date=True,
                  save='../data/wikipedia_timeline_of_natural_history.csv',
                  parse=True
)

df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13236199466447773, 'fraction_numerical': 0.005335522265544839, 'mean_word_length': 1.9701401584399756, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '650319', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1618, 'n_tokens_neox': 1569, 'n_words': 308, 'path': 'notebooks/scraping wikipedia.ipynb', 'repo_name': 'rjsmethurst/galaxy_soul_trail', 'size': '4873'}