<jupyter_start><jupyter_text>Advanced algorithms        CS-450 / Section IN        Kapralov,Svensson                    EN                              -          -          -                                      4h          3h          -                                                  Data Analytics            Foundations of software            Information Security-SP            Networking and Mobility            Computer Science Theory                                      Summer sessionWritten                    7                    afficher        masquer<jupyter_code># Parse the html code corresponding to a single course
# Returns a dictionary with the following info:
# - full name (ie. "Advanced algorithms")
# - code and section (ie. "CS-450 / Section IN")
# - link to the coursebook page of the given course
# - session (ie. "Summer session")
# - evaluation method (ie. "Written")
# - number of credits
# - a remark, for example whether the course is biennial, usually empty
# - the list of teachers (ie ["Kapralov"])
# - the set of specializations (ie {
#                                   "Data Analytics",
#                                   "Foundations of software",
#                                   "Information Security-SP",
#                                   "Networking and Mobility",
#                                   "Computer Science Theory"
#                                  })
# The group (ie. "Group 1") is not returned by the function.
def parse_course(course):
    row = {}
    
    row["name"] = course.find("div", class_="cours")["data-title"]
    row["code"] = course.find("div", class_="cours-info").string
    row["link"] = BASE + course.find("div", class_="cours-name").find("a")["href"]
    row["session"] = course.find("div", class_="examen").find("b").string
    row["eval"] = course.find("div", class_="examen").find("span").string
    row["credits"] = int(course.find("div", class_="credit-time").string)
    
    # Extract the remark
    remark = course.find("div", class_="cours-name").find("i")
    row["remark"] = ""
    if remark is not None and remark != "":
        if remark.string[0] == "(" and remark.string[-1] == ")":
            row["remark"] = remark.string[1:-1]
        else:
            row["remark"] = remark.string
    
    # Extract the list of teachers
    row["teachers"] = []
    for teacher in course.find("div", class_="enseignement-name").find_all("a"):
        row["teachers"].append(teacher.string)

    # Extract the set of specializations
    row["specializations"] = set()
    specs = course.find("div", attrs={"data-title": "Specialization"})
    if specs is not None:
        for spec in specs.find_all("li"):
            row["specializations"].add(spec.string)
    
    return row

# The html parser
html = BeautifulSoup(content, 'html.parser')

# List of extracted course information
courses = []

# Iterate over groups
for group in html.find_all("div", class_=re.compile("table-like mb-5")):
    groupname = group.find("h4").string

    # Then iterate over the courses
    for course in group.find_all("div", class_="line-down"):
        courseinfo = parse_course(course)
        # Add the group
        courseinfo["group"] = groupname
        courses.append(courseinfo)
# Create the DataFrame
# for easier queries later
df = pd.DataFrame(courses)
df.head(10)
# Out of curiosity see the possible remarks
df[df.remark != ""]["remark"]
# Extract the sorted list of specializations
allspecs = set()
for specs in df["specializations"]:
    allspecs = allspecs.union(specs)
    
allspecs = sorted(allspecs)
allspecs
# Return the courses that have the given specialization.
def find_spec(df, spec):
    return df[df.specializations.apply(lambda specs: spec in specs)]

find_spec(df, "Software Systems")
# Export the courses information into a sheet
def coursebook_sheet(df):
    header = ["Nom", "Période", "Crédits", "Type", "Biennal", ""]
    sheet = Sheet('L', size=(1 + df.shape[0], len(header) + len(allspecs) + 1))
    
    # Print header
    for i, text in enumerate(header):
        sheet[0, i].set_value(text)
    for i, text in enumerate(allspecs):
        sheet[0, len(header) + i].set_value(text)
    sheet[0, len(header)+len(allspecs)].set_value("Exam session")

    # Print one line per course
    for coursenum, (_, course) in enumerate(df.iterrows()):
        # If the name contains parenthesis, just remove them
        # (because it's not convenient in the spreadsheet)
        sheet[coursenum + 1, 0].set_value(course["name"].replace('(', '-').replace(')', '-'))

        # The session is either "W", "S" or "both"
        w = "Winter" in course["session"]
        s = "Summer" in course["session"]
        if w and s:
            sheet[coursenum + 1, 1].set_value("both")
        elif w:
            sheet[coursenum + 1, 1].set_value("W")
        elif s:
            sheet[coursenum + 1, 1].set_value("S")
        else:
            sheet[coursenum + 1, 1].set_value("") # error ?
        
        sheet[coursenum + 1, 2].set_value(course["credits"])
        sheet[coursenum + 1, 3].set_value(course["group"])
        
        # Check if course is biennial, not a 100% sure way
        sheet[coursenum + 1, 4].set_value(1 if re.search("[Pp]as donné|biennal", course["remark"]) else "")
        sheet[coursenum + 1, 5].set_value("")

        for i, spec in enumerate(allspecs):
            sheet[coursenum + 1, 6 + i].set_value(1 if spec in course["specializations"] else "")
        
        # If "Oral" or "Written", the course is evaluated during the exam session
        sheet[coursenum + 1, 6 + len(allspecs)].set_value(1 if course["eval"] in {"Oral", "Written"} else "")
    
    return sheet
# `num` is the number of the semester
# `courseheight` must be at least the number of course.
# `height` is the number of rows allocated for the courses
# Generates a sheet named f"S{num}", which tracks the credits taken
# during a given semester. 
def semester_sheet(num, courseheight, height=15):
    sheet = Sheet(f"S{num}", size=(3 + height, 4 + len(allspecs)))
    
    # Header
    sheet[0, 1].set_value("Credits")
    sheet[0, 2].set_value("Core")
    for i, text in enumerate(allspecs):
        sheet[0, 3 + i].set_value(text)
    sheet[0, 3 + len(allspecs)].set_value("Exam session")

    # Sum of credits
    sheet[1, 0].set_value("Total:")
    sheet[1, 1].formula = f'of:=SUM(B4:B{4+height})+ORG.OPENOFFICE.STYLE(IF(ORG.OPENOFFICE.CURRENT() >= 30; "Good"; "Warning"))'
    for i in range(len(allspecs) + 1):
        sheet[1, 2 + i].formula = f'of:=SUM({chr(ord("C") + i)}4:{chr(ord("C") + i)}{3+height})'
    sheet[1, 3 + len(allspecs)].formula = f'of:=SUM({chr(ord("D") + len(allspecs))}4:{chr(ord("D") + len(allspecs))}{3+height})'
    
    for i in range(height):
        sheet[3 + i, 1].formula = f'of:=IFERROR(INT(IFERROR(VLOOKUP($A{4 + i};$L.$A$2:$L.$Q${courseheight + 1};3;FALSE());"")&T(ORG.OPENOFFICE.STYLE(IFERROR(IF(VLOOKUP($A{4 + i};$L.$A$2:$L.$Q${courseheight + 1};2;FALSE())<>IF(MOD(SHEET();2)=0;"S";"W");"Good";"Error");"Default"))));"")'
        sheet[3 + i, 2].formula = f'of:=IFERROR(IF(VLOOKUP($A{4 + i};$L.$A$2:$L.$Q${courseheight + 1};4;FALSE())="Group 1";$B{4 + i};0);"")'
        for j in range(len(allspecs)):
            sheet[3 + i, 3 + j].formula = f'of:=IFERROR(VLOOKUP($A{4 + i};$L.$A$2:$L.$Q${courseheight + 1};{7 + j};FALSE())*$B{4 + i};"")'
        sheet[3 + i, 3 + len(allspecs)].formula = f'of:=IFERROR(VLOOKUP($A{4 + i};$L.$A$2:$L.$Q${courseheight + 1};{7 + len(allspecs)};FALSE());"")'
            
    return sheet
# Generates a sheet which computes the total credits taken.
def total_sheet(nbsemesters):
    sheet = Sheet("T", size=(2, 3 + len(allspecs)))
    
    def sum_semesters(col):
        return "+".join([f"S{num}.{col}2" for num in range(1, nbsemesters + 1)])
    
    # Header
    sheet[0, 1].set_value("Credits")
    sheet[0, 2].set_value("Core")
    for i, text in enumerate(allspecs):
        sheet[0, 3 + i].set_value(text)

    # Sum of credits
    sheet[1, 0].set_value("Total:")
    sheet[1, 1].formula = f'of:={sum_semesters("B")}+ORG.OPENOFFICE.STYLE(IF(ORG.OPENOFFICE.CURRENT()>=120;"Good";"Error"))'
    sheet[1, 2].formula = f'of:={sum_semesters("C")}+ORG.OPENOFFICE.STYLE(IF(ORG.OPENOFFICE.CURRENT()>=30;"Good";"Error"))'
    for i, spec in enumerate(allspecs):
        sheet[1, 3 + i].formula = f'of:={sum_semesters(chr(ord("D") + i))}+ORG.OPENOFFICE.STYLE(IF(ORG.OPENOFFICE.CURRENT()>=30;"Good";IF(ORG.OPENOFFICE.CURRENT()>=20;"Neutral";"Default")))'
    
    return sheet
nbsemesters = 6

# The actual spreadsheet
spreadsheet = newdoc(doctype='ods', filename='courses.epfl.ods')

# Add the coursebook spreadsheet.
spreadsheet.sheets += coursebook_sheet(df)

# Add the semesters spreadsheets.
for num in range(1, nbsemesters + 1):
    spreadsheet.sheets += semester_sheet(num, df.shape[0])

# Add the final spreadsheet.
spreadsheet.sheets += total_sheet(nbsemesters)
# Exports the spreadsheet into a file.
spreadsheet.save()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16371289322108995, 'fraction_numerical': 0.016282676118741693, 'mean_word_length': 2.868466152527849, 'pattern_counts': {'":': 3, '<': 6, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '616132', 'license': 'MIT', 'n_tokens_mistral': 3100, 'n_tokens_neox': 2906, 'n_words': 712, 'path': 'coursebook-parser.ipynb', 'repo_name': 'pgimalac/epfl-coursebook-parser', 'size': '9028'}