<commit_before>from bs4 import BeautifulSoup
from urllib import request
from itertools import zip_longest

class Cafe(object):

    CAFE_BASE_URL = 'https://eatatstate.com'

    def __init__(self, name, endpoint):
        self.name = name
        self.endpoint = endpoint
        self.restaurants = []
        self._go_download_restaurants()

    def _go_download_restaurants(self):
        page_bytes = request.urlopen(self.CAFE_BASE_URL + self.endpoint)
        soup = BeautifulSoup(page_bytes, 'html.parser')

        """
         here we will get all the cafeterias
        """
        #
        # The target div path:
        # view-content -> views-table
        tables = soup.find_all('table')

        for table in tables:
            rest_anchor = table.find('caption').find('a')
            rest_name = rest_anchor.text
            caf_rest_endpoint = rest_anchor.get('href')

            rest = Restaurant(rest_name, caf_rest_endpoint, self)
            self.restaurants.append(rest)

    def is_closed(self):
        for r in self.restaurants:
            # if there is at least 1 restaurant that is not closed,
            # the caf is not closed
            if not r.is_closed():
                return False
        return True

    def __str__(self):
        output = ""
        output += 'Cafe: ' + self.name

        # each restaurant
        for r in self.restaurants:
            if not r.is_closed():
                output += '\n' + str(r)
        return output

    def __repr__(self):
        return self.__str__()

class Restaurant(object):
    CAFE_BASE_URL = 'https://eatatstate.com'

    def __init__(self, name, endpoint, cafe):
        self.name = name
        self.endpoint = endpoint
        self.cafe = cafe
        self.lunch = []
        self.dinner = []
        self.late_night = []
        self.breakfast = []

        self._go_download_options()

    def _go_download_options(self):
        # lets do lunch only for now
        page_bytes = request.urlopen(self.CAFE_BASE_URL + self.endpoint)
        soup = BeautifulSoup(page_bytes, 'html.parser')

        # map the tags to their lists
        menu_list_tags = {'field-field-lunch-menu': self.lunch, 'field-field-dinner-menu': self.dinner,
        'field-field-late-night': self.late_night, 'field-field-breakfast-menu': self.breakfast}

        # this will collect all the menu items
        for menu_tag in menu_list_tags:
            menu = soup.find('div', attrs={'class': menu_tag})

            # not being served
            if menu == None:
                print('Could not find ', menu_tag)
                continue

            # loop over all the menu options and add them to their respective list
            for div in menu.find_all('div', attrs={'class': 'field-item'}):
                name = div.text.strip()

                # skip these garbage entries
                if name.lower() == 'closed' or '\n' in name:
                    continue

                #print(self.cafe.name, self.name, '----->', "'{}'".format(name))

                menu_list_tags[menu_tag].append(name)

    def is_closed(self):
        return len(self.lunch) == 0 and len(self.dinner) == 0 and len(self.late_night) == 0 \
               and len(self.breakfast) == 0

    def __str__(self):
        output = ""
        output += "{:^180}\n".format(self.name)

        if self.is_closed():
            output += ' CLOSED'
            return output

        # print the menus
        output += "{:^50}{:^50}{:^50}{:^50}\n".format('Breakfast', 'Lunch', 'Dinner', 'Late-Night')
        output += "{:^50}{:^50}{:^50}{:^50}\n".format('-'*10, '-'*10, '-'*10, '-'*10)
        for breakfast,lunch,dinner,late_night in zip_longest(self.breakfast, self.lunch, self.dinner, self.late_night):
            output += "{:^50}{:^50}{:^50}{:^50}\n".format(str(breakfast), str(lunch), str(dinner), str(late_night))

        return output

    def __repr__(self):
        return self.__str__()<commit_msg>Cleaned up some of the code to make it more clear and less repetitive.<commit_after>from bs4 import BeautifulSoup
from urllib import request
from itertools import zip_longest

class Cafe(object):

    CAFE_BASE_URL = 'https://eatatstate.com'

    def __init__(self, name, endpoint):
        self.name = name
        self.endpoint = endpoint
        self.restaurants = []
        self._go_download_restaurants()

        # assume cafe is closed and check if it indeed is by checking the restaurants
        # TODO: Scrape site for hours instead
        self.is_closed = True
        for r in self.restaurants:
            # if there is at least 1 restaurant that is not closed,
            # the caf is not closed
            if not r.is_closed():
                self.is_closed = False

    def _go_download_restaurants(self):
        page_bytes = request.urlopen(self.CAFE_BASE_URL + self.endpoint)
        soup = BeautifulSoup(page_bytes, 'html.parser')

        """
         here we will get all the cafeterias
        """
        #
        # The target div path:
        # view-content -> views-table
        tables = soup.find_all('table')

        for table in tables:
            rest_anchor = table.find('caption').find('a')
            rest_name = rest_anchor.text
            caf_rest_endpoint = rest_anchor.get('href')

            rest = Restaurant(rest_name, caf_rest_endpoint, self)
            self.restaurants.append(rest)

    def __str__(self):
        output = ""
        output += 'Cafe: ' + self.name

        if self.is_closed:
            output += ' CLOSED\n'
            return output

        # each restaurant
        for r in self.restaurants:
            if not r.is_closed():
                output += '\n' + str(r)

        return output

    def __repr__(self):
        return self.__str__()

class Restaurant(object):
    CAFE_BASE_URL = 'https://eatatstate.com'

    def __init__(self, name, endpoint, cafe):
        self.name = name
        self.endpoint = endpoint
        self.cafe = cafe
        self.lunch = []
        self.dinner = []
        self.late_night = []
        self.breakfast = []

        self._go_download_options()

    def _go_download_options(self):
        # lets do lunch only for now
        page_bytes = request.urlopen(self.CAFE_BASE_URL + self.endpoint)
        soup = BeautifulSoup(page_bytes, 'html.parser')

        # map the tags to their lists
        menu_list_tags = {'field-field-lunch-menu': self.lunch, 'field-field-dinner-menu': self.dinner,
        'field-field-late-night': self.late_night, 'field-field-breakfast-menu': self.breakfast}

        # this will collect all the menu items
        for menu_tag in menu_list_tags:
            menu = soup.find('div', attrs={'class': menu_tag})

            # not being served
            if menu == None:
                #print('Could not find ', menu_tag)
                continue

            # loop over all the menu options and add them to their respective list
            for div in menu.find_all('div', attrs={'class': 'field-item'}):
                name = div.text.strip()

                # skip these garbage entries
                # hours is mixed in the same section of the list of their websites
                if name.lower() == 'closed' or '\n' in name or 'hours' in name.lower():
                    continue

                #print(self.cafe.name, self.name, '----->', "'{}'".format(name))

                menu_list_tags[menu_tag].append(name)

    def is_closed(self):
        return len(self.lunch) == 0 and len(self.dinner) == 0 and len(self.late_night) == 0 \
               and len(self.breakfast) == 0

    def __str__(self):
        output = ""
        output += "{:^180}\n".format(self.name)

        # check if is closed first
        if self.is_closed():
            output += ' CLOSED'
            return output

        # print the menus
        output += "{:^50}{:^50}{:^50}{:^50}\n".format('Breakfast', 'Lunch', 'Dinner', 'Late-Night')
        output += "{:^50}{:^50}{:^50}{:^50}\n".format('-'*10, '-'*10, '-'*10, '-'*10)
        for breakfast,lunch,dinner,late_night in zip_longest(self.breakfast, self.lunch, self.dinner, self.late_night):
            output += "{:^50}{:^50}{:^50}{:^50}\n".format(str(breakfast), str(lunch), str(dinner), str(late_night))

        return output

    def __repr__(self):
        return self.__str__()
{'fraction_non_alphanumeric': 0.12417753319775093, 'fraction_numerical': 0.009809785859552578, 'mean_word_length': 2.0466472303206995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6843656', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2466, 'n_words': 628}