<commit_before># builtin
import os
import collections
# external
import bs4
import yaml


############################################################
# setup
############################################################


base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')


############################################################
# file load functions
############################################################


def read_file(file_name):
    with open(file_name, 'r') as readme_file:
        content = readme_file.read()
    return content


def read_html(table_str):
    soup = bs4.BeautifulSoup(table_str, 'html.parser')
    table_dict = collections.OrderedDict()
    for video in soup.find_all('tr'):
        columns = video.find_all('td')
        name    = columns[4].find('a').contents[0]
        link    = columns[4].find('a').get('href')
        date    = columns[3].contents[0]
        price   = columns[5].contents[0]
        sales   = columns[6].contents[0]
        table_dict[name] = {
            'name': name,
            'date': date,
            'price': price,
            'sales': sales,
            'link': link
        }
    return table_dict
<commit_msg>remove unused function<commit_after># builtin
import os
import collections
# external
import bs4
import yaml


############################################################
# setup
############################################################


base_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..')


############################################################
# file load functions
############################################################


def read_html(table_str):
    soup = bs4.BeautifulSoup(table_str, 'html.parser')
    table_dict = collections.OrderedDict()
    for video in soup.find_all('tr'):
        columns = video.find_all('td')
        name    = columns[4].find('a').contents[0]
        link    = columns[4].find('a').get('href')
        date    = columns[3].contents[0]
        price   = columns[5].contents[0]
        sales   = columns[6].contents[0]
        table_dict[name] = {
            'name': name,
            'date': date,
            'price': price,
            'sales': sales,
            'link': link
        }
    return table_dict

{'fraction_non_alphanumeric': 0.3182016586643387, 'fraction_numerical': 0.00960279353993889, 'mean_word_length': 3.2287822878228782, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1151647', 'n_tokens_mistral': 644, 'n_tokens_neox': 618, 'n_words': 119}