<commit_before>import json
import os
import re
import time
import datetime
from typing import Dict, List, Union

import requests
from bs4 import BeautifulSoup


def parse_page(page: str) -> Dict:
    result = {}
    page = page.decode('utf-8')

    # Fix page first - there are missing opening <tr> tags
    page = re.sub(r'</tr>', '</tr><tr>', page)

    soup = BeautifulSoup(page, "html.parser")
    for course in soup.find_all('tr'):
        children = course.findChildren()
        if not children:
            continue

        try:
            department = int(children[0].text)
        except ValueError:
            continue  # Skip rows that do not contain number as their department

        course_id = children[1].text
        course_name = children[2].text
        enrolled = int(children[3].text)
        finished = int(children[4].text)
        submitted_survey = int(children[5].text)
        percent_finished = finished / enrolled

        result[course_id] = {
            'department': department,
            'course_id': course_id,
            'course_name': course_name,
            'enrolled': enrolled,
            'finished': finished,
            'submitted_survey': submitted_survey,
            'percent_finished': percent_finished
        }

    return result


def fetch_courses():
    page = fetch_page()
    courses = parse_page(page)

    return courses


def fetch_page():
    response = requests.get('https://anketa.cvut.cz/stav/stav_anketa_fit.html')
    response.raise_for_status()

    return response.content


def read_data(fn):
    with open(fn, encoding='utf-8') as f:
        data = json.load(f)

    return data


def save_data(fn, data):
    if not os.path.exists(os.path.dirname(fn)):
        os.mkdir(os.path.dirname(fn))

    with open(f"{fn}.temp", 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=2)

    if os.path.exists(fn):
        os.remove(fn)

    os.rename(f"{fn}.temp", fn)


def combine_courses(courses, fn='courses.json'):
    timestamp = int(time.time())

    if os.path.exists(fn):
        data = read_data(fn)
    else:
        data = {}

    data[timestamp] = courses
    return data


def make_md_table(semester: str, course_semester_data: List[Dict[str, Union[str, int, float]]]):
    """ Make markdown table from data of a single semester-course. """
    if not course_semester_data:
        return ""

    data_item = course_semester_data[0]
    course_id = data_item['course_id']
    course_name = data_item['course_name']

    row_headers = '|                          |'
    row_separator = '|--------------------------|'
    row_completed_interval = "|**Splněno v období**      |"
    row_completed_total = "|**Splněno celkem**        |"
    row_completed_total_percent = "|**Splněno celkem procent**|"

    row_data_headers = []
    row_data_separator = []
    row_data_compl_interval = []
    row_data_compl_total = []
    row_data_compl_total_percent = []
    for datapoint in course_semester_data:
        row_data_headers.append(util.timestamp_to_date_str(datapoint['timestamp']))
        row_data_separator.append("-"*20)
        row_data_compl_interval.append("?")
        row_data_compl_total.append("?")
        row_data_compl_total_percent.append("?")

    md = f"""##{course_name} ({course_id})
####Semestr {semester}

{row_headers}{"|".join(row_data_headers)}|
{row_separator}{"|".join(row_data_separator)}|
{row_completed_interval}{"|".join(row_data_compl_interval)}|
{row_completed_total}{"|".join(row_data_compl_total)}|
{row_completed_total_percent}{"|".join(row_data_compl_total_percent)}|
"""

    return md


def load_semester(semester: str):
    fn = f'data/{semester.upper()}.json'
    if not os.path.exists(fn):
        return {}

    return read_data(fn)


def parse_course_data(course_id: str, data):
    study_programme = course_id.split('-', maxsplit=1)[0]
    if not data:
        return []

    try:
        course_data = data[study_programme]
        course_data = course_data[course_id]
        return course_data
    except KeyError:
        return []<commit_msg>Rendering done<commit_after>import json
import os
import re
import time
import datetime
from typing import Dict, List, Union

import requests
from bs4 import BeautifulSoup


def parse_page(page: str) -> Dict:
    result = {}
    page = page.decode('utf-8')

    # Fix page first - there are missing opening <tr> tags
    page = re.sub(r'</tr>', '</tr><tr>', page)

    soup = BeautifulSoup(page, "html.parser")
    for course in soup.find_all('tr'):
        children = course.findChildren()
        if not children:
            continue

        try:
            department = int(children[0].text)
        except ValueError:
            continue  # Skip rows that do not contain number as their department

        course_id = children[1].text
        course_name = children[2].text
        enrolled = int(children[3].text)
        finished = int(children[4].text)
        submitted_survey = int(children[5].text)
        percent_finished = finished / enrolled

        result[course_id] = {
            'department': department,
            'course_id': course_id,
            'course_name': course_name,
            'enrolled': enrolled,
            'finished': finished,
            'submitted_survey': submitted_survey,
            'percent_finished': percent_finished
        }

    return result


def fetch_courses():
    page = fetch_page()
    courses = parse_page(page)

    return courses


def fetch_page():
    response = requests.get('https://anketa.cvut.cz/stav/stav_anketa_fit.html')
    response.raise_for_status()

    return response.content


def read_data(fn):
    with open(fn, encoding='utf-8') as f:
        data = json.load(f)

    return data


def save_data(fn, data):
    if not os.path.exists(os.path.dirname(fn)):
        os.mkdir(os.path.dirname(fn))

    with open(f"{fn}.temp", 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=2)

    if os.path.exists(fn):
        os.remove(fn)

    os.rename(f"{fn}.temp", fn)


def combine_courses(courses, fn='courses.json'):
    timestamp = int(time.time())

    if os.path.exists(fn):
        data = read_data(fn)
    else:
        data = {}

    data[timestamp] = courses
    return data


def make_md_table(semester: str, course_semester_data: List[Dict[str, Union[str, int, float]]]):
    """ Make markdown table from data of a single semester-course. """
    if not course_semester_data:
        return ""

    data_item = course_semester_data[0]
    course_id = data_item['course_id']
    course_name = data_item['course_name']
    enrolled_total = data_item['enrolled']

    row_headers = '|                          |'
    row_separator = '|--------------------------|'
    row_completed_total = "|**Splněno celkem**        |"
    row_completed_total_percent = "|**Splněno celkem procent**|"

    row_data_headers = []
    row_data_separator = []
    row_data_compl_total = []
    row_data_compl_total_percent = []

    previous_completed = 0
    previous_completed_percent = 0

    for datapoint in course_semester_data:
        row_data_headers.append(util.timestamp_to_date_str(datapoint['timestamp']))
        row_data_separator.append("-"*20)

        new_completed = datapoint['finished']
        new_completed_percent = datapoint['percent_finished']
        completed_delta = new_completed-previous_completed
        completed_percent_delta = new_completed_percent-previous_completed_percent
        previous_completed = new_completed
        previous_completed_percent = new_completed_percent

        row_data_compl_total.append(f'''{new_completed} ({completed_delta:+})''')
        row_data_compl_total_percent.append(f'''{datapoint['percent_finished']*100:.0f}% ({completed_percent_delta*100:+.0f}%)''')

    print("ha")
    md = f"""## {course_name} ({course_id})

**Semestr**: {semester}

**Přihlášeno studentů**: {enrolled_total}

{row_headers}{"|".join(row_data_headers)}|
{row_separator}{"|".join(row_data_separator)}|
{row_completed_total}{"|".join(row_data_compl_total)}|
{row_completed_total_percent}{"|".join(row_data_compl_total_percent)}|
"""

    return md


def load_semester(semester: str):
    fn = f'data/{semester.upper()}.json'
    if not os.path.exists(fn):
        return {}

    return read_data(fn)


def parse_course_data(course_id: str, data):
    study_programme = course_id.split('-', maxsplit=1)[0]
    if not data:
        return []

    try:
        course_data = data[study_programme]
        course_data = course_data[course_id]
        return course_data
    except KeyError:
        return []
{'fraction_non_alphanumeric': 0.13073633105999308, 'fraction_numerical': 0.004854930065888337, 'mean_word_length': 2.954296160877514, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6012578', 'n_tokens_mistral': 2888, 'n_tokens_neox': 2733, 'n_words': 543}