<commit_before>import urllib2
from BeautifulSoup import BeautifulSoup
import sys
import re
import json
from config import unit_titles, get_normal_unit
from UserDict import UserDict
import os.path

master_url = "http://www.cs.manchester.ac.uk/undergraduate/programmes/courseunits/syllabus.php?code=%s"

header_re = re.compile(".*: (.*) \(\d{4}\)")


class UnitNames(UserDict):
    """A dictionary-like class, storing unit titles."""
    def __init__(self, file_name = unit_titles):
        UserDict.__init__(self)
        self.file_name = os.path.expanduser(file_name)
        self.read()


    def __del__(self):
        self.write()


    def read(self):
        """Read the unit names from disk."""
        try:
            self.data = json.load(open(self.file_name))
        except:
            pass


    def write(self):
        """Write the unit names to disk."""
        json.dump(self.data, open(self.file_name, 'w'),
                  indent=4)


    def __getitem__(self, unit):
        if unit in self.data:
            return self.data[unit]
        else:
            title = get_title(unit)
            self.data[unit] = title
            return title


def web_unit_codes(unit):
    """Turn a unit code from ARCADE into a list of possible web unit codes."""
    # Try just prepending 'COMP' to the unit.
    yield "COMP%s" % unit
    # Try decreasing the last number.
    if unit[-1].isdigit():
        for i in reversed(range(int(unit[-1]))):
            yield "COMP%s%i" % (unit[:-1], i)
    # Finally tryjust the unit.
    yield unit


def get_web_title(unit):
    """Get the title of a unit from the web."""
    # Silently ignore all network and parse errors.
    # Failure is always an option!
    try:
        # Download and parse the page.
        page = urllib2.urlopen(master_url % unit)
        soup = BeautifulSoup(page)<commit_msg>Use pprint/eval rather than json for compatibility with python 2.4.<commit_after>import urllib2
from BeautifulSoup import BeautifulSoup
import sys
import re
#import json
import pprint
from config import unit_titles, get_normal_unit
from UserDict import UserDict
import os.path

master_url = "http://www.cs.manchester.ac.uk/undergraduate/programmes/courseunits/syllabus.php?code=%s"

header_re = re.compile(".*: (.*) \(\d{4}\)")


class UnitNames(UserDict):
    """A dictionary-like class, storing unit titles."""
    def __init__(self, file_name = unit_titles):
        UserDict.__init__(self)
        self.file_name = os.path.expanduser(file_name)
        self.read()


    def __del__(self):
        self.write()


    def read(self):
        """Read the unit names from disk."""
        try:
            #self.data = json.load(open(self.file_name))
            self.data = eval(open(self.file_name).read())
        except:
            pass


    def write(self):
        """Write the unit names to disk."""
        # json.dump(self.data, open(self.file_name, 'w'),
        #           indent=4)
        pprint.pprint(self.data, open(self.file_name, 'w'),
                      indent=4, width=10)


    def __getitem__(self, unit):
        if unit in self.data:
            return self.data[unit]
        else:
            title = get_title(unit)
            self.data[unit] = title
            return title


def web_unit_codes(unit):
    """Turn a unit code from ARCADE into a list of possible web unit codes."""
    # Try just prepending 'COMP' to the unit.
    yield "COMP%s" % unit
    # Try decreasing the last number.
    if unit[-1].isdigit():
        for i in reversed(range(int(unit[-1]))):
            yield "COMP%s%i" % (unit[:-1], i)
    # Finally tryjust the unit.
    yield unit


def get_web_title(unit):
    """Get the title of a unit from the web."""
    # Silently ignore all network and parse errors.
    # Failure is always an option!
    try:
        # Download and parse the page.
        page = urllib2.urlopen(master_url % unit)
        soup = BeautifulSoup(page)
{'fraction_non_alphanumeric': 0.11651469098277609, 'fraction_numerical': 0.0048125633232016215, 'mean_word_length': 2.4131374243733794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5120762', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1201, 'n_words': 355}