<commit_before>import urllib
import time
from BeautifulSoup import BeautifulSoup
from college.models import *
from utils import update_college_year, populate_head_coaches
from django.template.defaultfilters import slugify
from scrapers.models import NCAAGame

def game_updater(year, teams, week, nostats=False):
    """
    Loads a team's games for a given year, creating new games as it finds them and updating scores. If needed,
    it will create new College records for opponents. The function accepts an optional QuerySet of teams. 
    It can be run in nostats mode, in which case the function updates only the score. If nostats=True, 
    after the game is updated the function calls other functions to update player and drive statistics. After completing,
    this function calls update_college_year for the given year, which updates the win-loss record of each team.
    
    >>> game_updater(2010, teams, 12)
    """
    if not teams:
        teams = CollegeYear.objects.filter(season=year, college__updated=True).order_by('id')
    
    games = []

    starting_time = datetime.datetime.now()
    print "Run Start"
    print starting_time.strftime("%Y-%m-%d %H:%M")

    last_team_end = datetime.datetime.now()
    counter = 1
    total_count = teams.count()
    for team in teams:
        print "%(counter)s out of %(total_count)s " % { 'counter': str(counter), 'total_count': str(total_count) }
        team_start = datetime.datetime.now()
        print team
        print "Team Start"
        print team_start.strftime("%Y-%m-%d %H:%M:%S")
        url = "http://web1.ncaa.org/football/exec/rankingSummary?org=%s&year=%s&week=%s" % (team.college.id, year, week)
        html = urllib.urlopen(url).read()
        soup = BeautifulSoup(html)
        try:
            tables = soup.findAll('table') # list
            t = tables[2] # BS.Tag
            rows = t.findAll('tr')[2:] #list
            for row in rows: # BS.Tag
                game_results = row.findAll('td')
                try:
                    game_file = game_results[0].find('a')['href'].split('game=')[1]
                    stringdate = game_results[0].find('a').contents[0][4:]
                    team1_score, team2_score = [int(x) for x in game_results[2].contents[0].split(' - ')]
                    if len(game_results[3].contents[0].strip().split(' ')) == 2:
                        t1_result, ot = game_results[3].contents[0].strip().split(' ')
                    else:
                        t1_result = game_results[3].contents[0].strip()
                        ot = None

                except:
                    game_file = None
                    stringdate = game_results[0].contents[0][4:]
                    team1_score = None
                    team2_score = None
                    t1_result = None
                    ot = None
                date = datetime.date(*(time.strptime(stringdate, '%m/%d/%Y')[0:3]))
                try:
                    t2 = int(game_results[1].find('a')['href'].split('=')[1].split('&')[0])
                    try:
                        if t2 == 115:   # hack job to cover for ncaa change
                            team2 = CollegeYear.objects.get(college__id=30416, season=year)
                        elif t2 == 357: # another one like the above - Lincoln Univ. PA
                            team2 = CollegeYear.objects.get(college__id=30417, season=year)
                        elif t2 == 30134: # Black Hills St.
                            team2 = CollegeYear.objects.get(college__id=30551, season=year)
                        elif t2 == 30120: # <NAME>
                            team2 = CollegeYear.objects.get(college__id=30642, season=year)
                        else:
                            team2 = CollegeYear.objects.get(college__id=t2, season=year)
                    except:
                        name = game_results[1].contents[0].replace("*","").strip().title()
                        slug = slugify(name)
                        new_college, created = College.objects.get_or_create(name=name, slug=slug)
                        team2 = CollegeYear.objects.get_or_create(college=new_college, season=year)
                except:
                    if len(game_results[1].contents) > 0 and game_results[1].contents[0] != '':
                        name = game_results[1].contents[0].replace("*","").replace("@","").strip().title()
                        slug = slugify(name)
                        new_college, created = College.objects.get_or_create(name=name, slug=slug)
                        team2, created = CollegeYear.objects.get_or_create(college=new_college, season=year)
                    else:
                        continue
                g, new_game = Game.objects.get_or_create(season=year, team1=team, team2=team2, date=date)<commit_msg>special case for <NAME><commit_after>import urllib
import time
from BeautifulSoup import BeautifulSoup
from college.models import *
from utils import update_college_year, populate_head_coaches
from django.template.defaultfilters import slugify
from scrapers.models import NCAAGame

def game_updater(year, teams, week, nostats=False):
    """
    Loads a team's games for a given year, creating new games as it finds them and updating scores. If needed,
    it will create new College records for opponents. The function accepts an optional QuerySet of teams. 
    It can be run in nostats mode, in which case the function updates only the score. If nostats=True, 
    after the game is updated the function calls other functions to update player and drive statistics. After completing,
    this function calls update_college_year for the given year, which updates the win-loss record of each team.
    
    >>> game_updater(2010, teams, 12)
    """
    if not teams:
        teams = CollegeYear.objects.filter(season=year, college__updated=True).order_by('id')
    
    games = []

    starting_time = datetime.datetime.now()
    print "Run Start"
    print starting_time.strftime("%Y-%m-%d %H:%M")

    last_team_end = datetime.datetime.now()
    counter = 1
    total_count = teams.count()
    for team in teams:
        print "%(counter)s out of %(total_count)s " % { 'counter': str(counter), 'total_count': str(total_count) }
        team_start = datetime.datetime.now()
        print team
        print "Team Start"
        print team_start.strftime("%Y-%m-%d %H:%M:%S")
        url = "http://web1.ncaa.org/football/exec/rankingSummary?org=%s&year=%s&week=%s" % (team.college.id, year, week)
        html = urllib.urlopen(url).read()
        soup = BeautifulSoup(html)
        try:
            tables = soup.findAll('table') # list
            t = tables[2] # BS.Tag
            rows = t.findAll('tr')[2:] #list
            for row in rows: # BS.Tag
                game_results = row.findAll('td')
                try:
                    game_file = game_results[0].find('a')['href'].split('game=')[1]
                    stringdate = game_results[0].find('a').contents[0][4:]
                    team1_score, team2_score = [int(x) for x in game_results[2].contents[0].split(' - ')]
                    if len(game_results[3].contents[0].strip().split(' ')) == 2:
                        t1_result, ot = game_results[3].contents[0].strip().split(' ')
                    else:
                        t1_result = game_results[3].contents[0].strip()
                        ot = None

                except:
                    game_file = None
                    stringdate = game_results[0].contents[0][4:]
                    team1_score = None
                    team2_score = None
                    t1_result = None
                    ot = None
                date = datetime.date(*(time.strptime(stringdate, '%m/%d/%Y')[0:3]))
                try:
                    t2 = int(game_results[1].find('a')['href'].split('=')[1].split('&')[0])
                    try:
                        if t2 == 115:   # hack job to cover for ncaa change
                            team2 = CollegeYear.objects.get(college__id=30416, season=year)
                        elif t2 == 357: # another one like the above - Lincoln Univ. PA
                            team2 = CollegeYear.objects.get(college__id=30417, season=year)
                        elif t2 == 30134: # Black Hills St.
                            team2 = CollegeYear.objects.get(college__id=30551, season=year)
                        elif t2 == 30120: # <NAME>
                            team2 = CollegeYear.objects.get(college__id=30642, season=year)
                        elif t2 == 30127: # <NAME>
                            team2 = CollegeYear.objects.get(college__id=30722, season=year)
                        else:
                            team2 = CollegeYear.objects.get(college__id=t2, season=year)
                    except:
                        name = game_results[1].contents[0].replace("*","").strip().title()
                        slug = slugify(name)
                        new_college, created = College.objects.get_or_create(name=name, slug=slug)
                        team2 = CollegeYear.objects.get_or_create(college=new_college, season=year)
                except:
                    if len(game_results[1].contents) > 0 and game_results[1].contents[0] != '':
                        name = game_results[1].contents[0].replace("*","").replace("@","").strip().title()
                        slug = slugify(name)
                        new_college, created = College.objects.get_or_create(name=name, slug=slug)
                        team2, created = CollegeYear.objects.get_or_create(college=new_college, season=year)
                    else:
                        continue
                g, new_game = Game.objects.get_or_create(season=year, team1=team, team2=team2, date=date)
{'fraction_non_alphanumeric': 0.1144425246468137, 'fraction_numerical': 0.02134363248297591, 'mean_word_length': 1.8654630168899242, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7441053', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2701, 'n_words': 645}