<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Combines all data for the semester into a shared json"""

__authors__    = ["<NAME>"]
__copyright__  = "<NAME>"
__license__    = "MIT"
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.

import os
import sys
from bs4 import BeautifulSoup
from collections import OrderedDict
import json

def dump_to_file(data, filename):
    with open(filename, 'w') as outfile:
        json.dump(data, outfile, indent=4, ensure_ascii=False)

def get_semester_order(start_year, stop_year):
    start_year = int(start_year)
    stop_year = int(stop_year)

    s_order = []
    for i in range(2000,2030):
        s_order.append("V"+str(i))
        s_order.append("H"+str(i))
    return s_order

def get_semesters(path):
    semester_order = get_semester_order(2000,2030)
    semesters = []
    for root, subdirs, files in os.walk(path):
        semesters = subdirs
        break

    indices = [semester_order.index(x) for x in semesters]
    semesters = [x for (y,x) in sorted(zip(indices,semesters))]
    return semesters

def main(dir):
    pass

if __name__ == '__main__':
    semesters = get_semesters("./data")
    courses = OrderedDict()
    for s in semesters:
        p = "./data/"+s+"/semester.json"
        semester = json.load(open(p), object_pairs_hook=OrderedDict)
        for course in semester:
            if course not in courses:
                courses[course] = OrderedDict()
            courses[course][s] = semester[course]
    dump_to_file(courses, "./data/courses.json")
<commit_msg>Bug fix: unused argument in year/semester range<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Combines all data for the semester into a shared json"""

__authors__    = ["<NAME>"]
__copyright__  = "<NAME>"
__license__    = "MIT"
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.

import os
import sys
from bs4 import BeautifulSoup
from collections import OrderedDict
import json

def dump_to_file(data, filename):
    with open(filename, 'w') as outfile:
        json.dump(data, outfile, indent=4, ensure_ascii=False)

def get_semester_order(start_year, stop_year):
    start_year = int(start_year)
    stop_year = int(stop_year)
    s_order = []
    for i in range(start_year,stop_year):
        s_order.append("V"+str(i))
        s_order.append("H"+str(i))
    return s_order

def get_semesters(path):
    semester_order = get_semester_order(2000,2030)
    semesters = []
    for root, subdirs, files in os.walk(path):
        semesters = subdirs
        break

    indices = [semester_order.index(x) for x in semesters]
    semesters = [x for (y,x) in sorted(zip(indices,semesters))]
    return semesters

def main(dir):
    pass

if __name__ == '__main__':
    semesters = get_semesters("./data")
    courses = OrderedDict()
    for s in semesters:
        p = "./data/"+s+"/semester.json"
        semester = json.load(open(p), object_pairs_hook=OrderedDict)
        for course in semester:
            if course not in courses:
                courses[course] = OrderedDict()
            courses[course][s] = semester[course]
    dump_to_file(courses, "./data/courses.json")

{'fraction_non_alphanumeric': 0.11682808716707022, 'fraction_numerical': 0.009685230024213076, 'mean_word_length': 3.2535392535392536, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6111582', 'n_tokens_mistral': 1138, 'n_tokens_neox': 1079, 'n_words': 296}