<commit_before>#!/usr/bin/python

"""
Box Office Mojo API
Unofficial Python API for Box Office Mojo.

Author: <NAME>
Email: <EMAIL>
License: MIT
"""
import urllib
import urllib2
import re
#import requests
from bs4 import BeautifulSoup
from .constants import BASE_URL, WEEKEND_CHART, DAILY_CHART

def get_soup(page=WEEKEND_CHART):
    #content = requests.get('%s/%s/' % (BASE_URL, page))
    content = urllib2.urlopen('%s/%s' % (BASE_URL, page)).read()
    return BeautifulSoup(content)

class BOM(object):
    """
    The class that parses the BoxOfficeMojo page, and builds up all the movies
    """

    def _get_movie_id(self, movie_id_link):
        """
        Returns the id from movie_id_link
        """
        #return re.sub('\/movies\/\?id\=', '', movie_id_link)
        return movie_id_link.split('id=')[1]


    def weekend_chart(self, limit=10):
        """
        Yields a list of box office movies from the weekend chart of BoxOfficeMojo

        'limit' is the max number of movies to return.
        Default is 10, cannot be more than 25.
        """
        if limit <= 0 or limit > 25:
            limit = 25

        movies_found = 0

        soup = get_soup(WEEKEND_CHART)
        table = soup.findChildren('table')[4]
        movies = table.findChildren('tr')[1:-1]
        
        for m in movies:
            attrs = m.findChildren('td')
            rank = str(attrs[0].string)
            title = str(attrs[2].string)
            movie_id = self._get_movie_id(str(attrs[2].a['href']))
            studio = str(attrs[3].string)
            gross = str(attrs[4].string)

            movie = Movie(movie_id, rank, title, studio, gross)
            yield movie
            movies_found += 1

            if movies_found >= limit:
                return

    def daily_chart(self, limit=10):
        """
        Yields a list of box office movies from the daily chart of BoxOfficeMojo

        'limit' is the max number of movies to return.
        Default is 10, cannot be more than 25.
        """
        if limit <= 0 or limit > 25:
            limit = 25

        movies_found = 0

        soup = get_soup(DAILY_CHART)
        movies = soup.findChildren('tr')[16:-2]

        s = soup.findChildren('b')
        mvs = s[13:len(s)-3]

        it = iter(mvs)

        mvs = zip(it, it)

        assert len(mvs) == len(movies)

        for (i, m) in enumerate(movies):
            attrs = m.findChildren('td')
            rank = str(attrs[0].string)
            movie_id = self._get_movie_id(str(attrs[1].b.a['href']))
            studio = str(attrs[1].small.a.string)

            title = str(mvs[i][0].next_element.string)
            gross = str(mvs[i][1].next_element.string)

            movie = Movie(movie_id, rank, title, studio, gross)
            yield movie
            movies_found += 1

            if movies_found >= limit:
                return


class Movie(object):
    """
    Movie class that represents a movie on BoxOfficeMojo
    """

    def _get_movie_soup(self, page='weekend', view=''):
        params = {
            'page':page,
            'view':view,
            'id':self.movie_id
        }
        encoded_params = urllib.urlencode(params)
        page = '/movies/?' + encoded_params
        return get_soup(page)

    def __init__(self, movie_id, rank, title, studio, gross):
        self.movie_id = movie_id # the movie's ID on BoxOfficeMojo.com
        self.rank = rank # this week's rank
        self.title = title # the title of the movie
        self.studio = studio # the movie's producing studio
        self._gross = gross # movie's gross income (can be either be weekend or daily)
        #TODO: add gross_to_date method
        #self.gross_to_date = _get_domestic_total(self.movie_id)

    def weekend_trend(self):
        """
        Returns a list of tuples of the Movie's weekend trend data
        Return value:
        [(week_number, date, rank, weekend_gross),]
        """
        soup = self._get_movie_soup('weekend')
        table = soup.findChildren('table')[6]

        # sometimes the page will have an extra IMDb advertisement
        # skip to the next table if such element exists
        if 'imdb' in table.a['href']:
            table = soup.findChildren('table')[7]

        rows = table.findChildren('tr')[1:]
        
        weekend = []
        rank = []
        gross = []
        index = []
        for row in rows:
            td = row.findChildren('td')
            weekend.append(td[0].string.encode('raw_unicode_escape').replace('\x96', '-'))
            rank.append(str(td[1].string))
            gross.append(str(td[2].string))
            index.append(str(td[8].string))

        return zip(index, weekend, rank, gross)

    def daily_trend(self):<commit_msg>added Movie._get_domestic_total() functions declaration<commit_after>#!/usr/bin/python

"""
Box Office Mojo API
Unofficial Python API for Box Office Mojo.

Author: <NAME>
Email: <EMAIL>
License: MIT
"""
import urllib
import urllib2
import re
#import requests
from bs4 import BeautifulSoup
from .constants import BASE_URL, WEEKEND_CHART, DAILY_CHART

def get_soup(page=WEEKEND_CHART):
    #content = requests.get('%s/%s/' % (BASE_URL, page))
    content = urllib2.urlopen('%s/%s' % (BASE_URL, page)).read()
    return BeautifulSoup(content)

class BOM(object):
    """
    The class that parses the BoxOfficeMojo page, and builds up all the movies
    """

    def _get_movie_id(self, movie_id_link):
        """
        Returns the id from movie_id_link
        """
        #return re.sub('\/movies\/\?id\=', '', movie_id_link)
        return movie_id_link.split('id=')[1]


    def weekend_chart(self, limit=10):
        """
        Yields a list of box office movies from the weekend chart of BoxOfficeMojo

        'limit' is the max number of movies to return.
        Default is 10, cannot be more than 25.
        """
        if limit <= 0 or limit > 25:
            limit = 25

        movies_found = 0

        soup = get_soup(WEEKEND_CHART)
        table = soup.findChildren('table')[4]
        movies = table.findChildren('tr')[1:-1]
        
        for m in movies:
            attrs = m.findChildren('td')
            rank = str(attrs[0].string)
            title = str(attrs[2].string)
            movie_id = self._get_movie_id(str(attrs[2].a['href']))
            studio = str(attrs[3].string)
            gross = str(attrs[4].string)

            movie = Movie(movie_id, rank, title, studio, gross)
            yield movie
            movies_found += 1

            if movies_found >= limit:
                return

    def daily_chart(self, limit=10):
        """
        Yields a list of box office movies from the daily chart of BoxOfficeMojo

        'limit' is the max number of movies to return.
        Default is 10, cannot be more than 25.
        """
        if limit <= 0 or limit > 25:
            limit = 25

        movies_found = 0

        soup = get_soup(DAILY_CHART)
        movies = soup.findChildren('tr')[16:-2]

        s = soup.findChildren('b')
        mvs = s[13:len(s)-3]

        it = iter(mvs)

        mvs = zip(it, it)

        assert len(mvs) == len(movies)

        for (i, m) in enumerate(movies):
            attrs = m.findChildren('td')
            rank = str(attrs[0].string)
            movie_id = self._get_movie_id(str(attrs[1].b.a['href']))
            studio = str(attrs[1].small.a.string)

            title = str(mvs[i][0].next_element.string)
            gross = str(mvs[i][1].next_element.string)

            movie = Movie(movie_id, rank, title, studio, gross)
            yield movie
            movies_found += 1

            if movies_found >= limit:
                return


class Movie(object):
    """
    Movie class that represents a movie on BoxOfficeMojo
    """

    def _get_movie_soup(self, page='weekend', view=''):
        params = {
            'page':page,
            'view':view,
            'id':self.movie_id
        }
        encoded_params = urllib.urlencode(params)
        page = '/movies/?' + encoded_params
        return get_soup(page)

    def _get_domestic_total(self):
        soup = self._get_movie_soup(page='',view='chart')
        print soup.prettify()


    def __init__(self, movie_id, rank, title, studio, gross):
        self.movie_id = movie_id # the movie's ID on BoxOfficeMojo.com
        self.rank = rank # this week's rank
        self.title = title # the title of the movie
        self.studio = studio # the movie's producing studio
        self._gross = gross # movie's gross income (can be either be weekend or daily)
        #TODO: add gross_to_date method
        #self.gross_to_date = _get_domestic_total()

    def weekend_trend(self):
        """
        Returns a list of tuples of the Movie's weekend trend data
        Return value:
        [(week_number, date, rank, weekend_gross),]
        """
        soup = self._get_movie_soup('weekend')
        table = soup.findChildren('table')[6]

        # sometimes the page will have an extra IMDb advertisement
        # skip to the next table if such element exists
        if 'imdb' in table.a['href']:
            table = soup.findChildren('table')[7]

        rows = table.findChildren('tr')[1:]
        
        weekend = []
        rank = []
        gross = []
        index = []
        for row in rows:
            td = row.findChildren('td')
            weekend.append(td[0].string.encode('raw_unicode_escape').replace('\x96', '-'))
            rank.append(str(td[1].string))
            gross.append(str(td[2].string))
            index.append(str(td[8].string))

        return zip(index, weekend, rank, gross)

    def daily_trend(self):
{'fraction_non_alphanumeric': 0.11711804837709323, 'fraction_numerical': 0.011990903452553235, 'mean_word_length': 2.0763116057233706, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '850579', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2964, 'n_words': 792}