<commit_before>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import org.apache.http.client.HttpResponseException;
import org.jetbrains.annotations.NotNull;
import org.joda.time.LocalDate;
import org.joda.time.format.DateTimeFormat;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parser for substitution schedules in HTML format created by the <a href="http://untis.de/">Untis</a> software
 * using the "Info-Stundenplan" layout. Only substitution schedule tables (often labelled with "Ver-Kla" in the
 * dropdown menu) are supported, not timetables.
 * <p>
 * Example: <a href="http://www.akg-bensheim.de/akgweb2011/content/Vertretung/default.htm">AKG Bensheim</a>
 * <p>
 * This parser can be accessed using <code>"untis-info"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>baseurl</code> (String, required)</dt>
 * <dd>The base URL under which all the HTML files (e.g. <code>default.htm</code>) are located, wthout a slash at
 * the end.</dd>
 *
 * <dt><code>encoding</code> (String, required)</dt>
 * <dd>The charset of the HTML files. It's probably either UTF-8 or ISO-8859-1.</dd>
 *
 * <dt><code>classes</code> (Array of Strings, optional)</dt>
 * <dd>The list of all classes, as they can appear in the schedule. If this is omitted, classes are automatically
 * determined by parsing JavaScript code in the <code>frames/navbar.htm</code> page.</dd>
 *
 * <dt><code>classSelectRegex</code> (String, optional)</dt>
 * <dd>RegEx to modify the classes parsed from JavaScript code in {@link #getAllClasses()}. The RegEx is matched against
 * the class using {@link Matcher#find()}. If the RegEx contains groups, the concatenation of all group results
 * {@link Matcher#group(int)} is used as the resulting class. Otherwise, {@link Matcher#group()} is used.
 * </dd>
 *
 * <dt><code>removeNonMatchingClasses</code> (Boolean, optional)</dt>
 * <dd>If this is set to <code>true</code>, classes parsed from JavaScript in {@link #getAllClasses()} where
 * <code>classSelectRegex</code> is not found ({@link Matcher#find()} returns <code>false</code>) are discarded from
 * the list. Default: <code>false</code>
 * </dd>
 *
 * <dt><code>singleClasses</code> (Boolean, optional)</dt>
 * <dd>Set this to <code>true</code> if there is no common substitution schedule for all classes, but separate ones
 * for each class selectable in a dropdown instead. This of course drastically increases the number of HTTP
 * requests needed to load the schedule. Default: <code>"false"</code>
 * </dd>
 *
 * <dt><code>wAfterNumber</code> (Boolean, optional)</dt>
 * <dd>Set this to <code>true</code> if the URL of the actual schedules (displayed in a frame) end with
 * <code>36/w/w00000.htm</code> instead of <code>w/36/w00000.htm</code>. Default: <code>"false"</code>
 * </dd>
 *
 * <dt><code>letter</code> (String, optional, Default: <code>w</code>)</dt>
 * <dd>The letter occurring in the URL of the schedule pages. For student schedules, this is almost always a
 * <code>w</code>. Teacher schedules use a <code>v</code>.</dd>
 *
 * <dt><code>scheduleBaseurl</code> (String, optional, Default: same as <code>baseurl</code>)</dt>
 * <dd>The url under which the actual schedule HTML files are hosted. In almost all cases you don't need to set it
 * as it's the same as <code>baseurl</code>.</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules
 * and those specified in {@link UntisCommonParser}.
 */
public class UntisInfoParser extends UntisCommonParser {

    public static final String PARAM_BASEURL = "baseurl";
    private static final String PARAM_ENCODING = "encoding";
    public static final String PARAM_CLASS_SELECT_REGEX = "classSelectRegex";
    public static final String PARAM_REMOVE_NON_MATCHING_CLASSES = "removeNonMatchingClasses";
    private static final String PARAM_SINGLE_CLASSES = "singleClasses";
    public static final String PARAM_W_AFTER_NUMBER = "wAfterNumber";
    private static final String PARAM_LETTER = "letter";
    private static final String PARAM_SCHEDULE_TYPE = "scheduleType";
    private static final String PARAM_SCHEDULE_BASEURL = "scheduleBaseurl";
    private String baseUrl;
    private JSONObject data;
    private String navbarDoc;

    public UntisInfoParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        try {
            data = scheduleData.getData();
            baseUrl = data.getString(PARAM_BASEURL);
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }

    private String getNavbarDoc() throws JSONException, IOException, CredentialInvalidException {
        if (navbarDoc == null) {
            String navbarUrl = baseUrl + "/frames/navbar.htm";
            navbarDoc = httpGet(navbarUrl, data.optString(PARAM_ENCODING, null));
        }
        return navbarDoc;
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule()
            throws IOException, JSONException, CredentialInvalidException {
        new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore);

        Document navbarDoc = Jsoup.parse(getNavbarDoc().replace("&nbsp;", ""));
        Element select = navbarDoc.select("select[name=week]").first();

        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        String info = navbarDoc.select(".description").text();
        String lastChange;
        try {
            lastChange = info.substring(info.indexOf("Stand:") + "Stand:".length()).trim();
        } catch (Exception e) {
            try {
                String infoHtml = httpGet(baseUrl + "/frames/title.htm", data.optString(PARAM_ENCODING, null));
                Document infoDoc = Jsoup.parse(infoHtml);
                String info2 = infoDoc.select(".description").text();
                lastChange = info2.substring(info2.indexOf("Stand:") + "Stand:".length()).trim();
            } catch (Exception e1) {
                lastChange = "";
            }
        }

        final List<String> allClasses = getAllClasses();

        int successfulWeeks = 0;
        HttpResponseException lastException = null;
		for (Element option : select.children()) {
			String week = option.attr("value");
            String weekName = option.text();
			if (data.optBoolean(PARAM_SINGLE_CLASSES,
                    data.optBoolean("single_classes", false)) // backwards compatibility
                    || data.optString(PARAM_SCHEDULE_TYPE, "substitution").equals("timetable")) {
                int classNumber = 1;
                List<String> classesToSelect = parseClasses(getNavbarDoc(), data);
                if (getLetter(data).equals("v")) classesToSelect = parseTeachers(getNavbarDoc(), data);
                for (String klasse : classesToSelect) {
                    String url = getScheduleUrl(week, classNumber, data);
					try {
                        parsePage(v, lastChange, klasse, url, weekName, allClasses);
                    } catch (HttpResponseException e) {
                        if (e.getStatusCode() == 500) {
                            // occurs in Hannover_MMBS
                            classNumber++;<commit_msg>UntisInfoParser: classes might not be there on teacher-only schedules<commit_after>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import org.apache.http.client.HttpResponseException;
import org.jetbrains.annotations.NotNull;
import org.joda.time.LocalDate;
import org.joda.time.format.DateTimeFormat;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parser for substitution schedules in HTML format created by the <a href="http://untis.de/">Untis</a> software
 * using the "Info-Stundenplan" layout. Only substitution schedule tables (often labelled with "Ver-Kla" in the
 * dropdown menu) are supported, not timetables.
 * <p>
 * Example: <a href="http://www.akg-bensheim.de/akgweb2011/content/Vertretung/default.htm">AKG Bensheim</a>
 * <p>
 * This parser can be accessed using <code>"untis-info"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>baseurl</code> (String, required)</dt>
 * <dd>The base URL under which all the HTML files (e.g. <code>default.htm</code>) are located, wthout a slash at
 * the end.</dd>
 *
 * <dt><code>encoding</code> (String, required)</dt>
 * <dd>The charset of the HTML files. It's probably either UTF-8 or ISO-8859-1.</dd>
 *
 * <dt><code>classes</code> (Array of Strings, optional)</dt>
 * <dd>The list of all classes, as they can appear in the schedule. If this is omitted, classes are automatically
 * determined by parsing JavaScript code in the <code>frames/navbar.htm</code> page.</dd>
 *
 * <dt><code>classSelectRegex</code> (String, optional)</dt>
 * <dd>RegEx to modify the classes parsed from JavaScript code in {@link #getAllClasses()}. The RegEx is matched against
 * the class using {@link Matcher#find()}. If the RegEx contains groups, the concatenation of all group results
 * {@link Matcher#group(int)} is used as the resulting class. Otherwise, {@link Matcher#group()} is used.
 * </dd>
 *
 * <dt><code>removeNonMatchingClasses</code> (Boolean, optional)</dt>
 * <dd>If this is set to <code>true</code>, classes parsed from JavaScript in {@link #getAllClasses()} where
 * <code>classSelectRegex</code> is not found ({@link Matcher#find()} returns <code>false</code>) are discarded from
 * the list. Default: <code>false</code>
 * </dd>
 *
 * <dt><code>singleClasses</code> (Boolean, optional)</dt>
 * <dd>Set this to <code>true</code> if there is no common substitution schedule for all classes, but separate ones
 * for each class selectable in a dropdown instead. This of course drastically increases the number of HTTP
 * requests needed to load the schedule. Default: <code>"false"</code>
 * </dd>
 *
 * <dt><code>wAfterNumber</code> (Boolean, optional)</dt>
 * <dd>Set this to <code>true</code> if the URL of the actual schedules (displayed in a frame) end with
 * <code>36/w/w00000.htm</code> instead of <code>w/36/w00000.htm</code>. Default: <code>"false"</code>
 * </dd>
 *
 * <dt><code>letter</code> (String, optional, Default: <code>w</code>)</dt>
 * <dd>The letter occurring in the URL of the schedule pages. For student schedules, this is almost always a
 * <code>w</code>. Teacher schedules use a <code>v</code>.</dd>
 *
 * <dt><code>scheduleBaseurl</code> (String, optional, Default: same as <code>baseurl</code>)</dt>
 * <dd>The url under which the actual schedule HTML files are hosted. In almost all cases you don't need to set it
 * as it's the same as <code>baseurl</code>.</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules
 * and those specified in {@link UntisCommonParser}.
 */
public class UntisInfoParser extends UntisCommonParser {

    public static final String PARAM_BASEURL = "baseurl";
    private static final String PARAM_ENCODING = "encoding";
    public static final String PARAM_CLASS_SELECT_REGEX = "classSelectRegex";
    public static final String PARAM_REMOVE_NON_MATCHING_CLASSES = "removeNonMatchingClasses";
    private static final String PARAM_SINGLE_CLASSES = "singleClasses";
    public static final String PARAM_W_AFTER_NUMBER = "wAfterNumber";
    private static final String PARAM_LETTER = "letter";
    private static final String PARAM_SCHEDULE_TYPE = "scheduleType";
    private static final String PARAM_SCHEDULE_BASEURL = "scheduleBaseurl";
    private String baseUrl;
    private JSONObject data;
    private String navbarDoc;

    public UntisInfoParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        try {
            data = scheduleData.getData();
            baseUrl = data.getString(PARAM_BASEURL);
        } catch (JSONException e) {
            e.printStackTrace();
        }
    }

    private String getNavbarDoc() throws JSONException, IOException, CredentialInvalidException {
        if (navbarDoc == null) {
            String navbarUrl = baseUrl + "/frames/navbar.htm";
            navbarDoc = httpGet(navbarUrl, data.optString(PARAM_ENCODING, null));
        }
        return navbarDoc;
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule()
            throws IOException, JSONException, CredentialInvalidException {
        new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore);

        Document navbarDoc = Jsoup.parse(getNavbarDoc().replace("&nbsp;", ""));
        Element select = navbarDoc.select("select[name=week]").first();

        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        String info = navbarDoc.select(".description").text();
        String lastChange;
        try {
            lastChange = info.substring(info.indexOf("Stand:") + "Stand:".length()).trim();
        } catch (Exception e) {
            try {
                String infoHtml = httpGet(baseUrl + "/frames/title.htm", data.optString(PARAM_ENCODING, null));
                Document infoDoc = Jsoup.parse(infoHtml);
                String info2 = infoDoc.select(".description").text();
                lastChange = info2.substring(info2.indexOf("Stand:") + "Stand:".length()).trim();
            } catch (Exception e1) {
                lastChange = "";
            }
        }

        List<String> allClasses = new ArrayList<>();
        try {
            allClasses = getAllClasses();
        } catch (IOException e) {
            // we only really need this for teacher schedules
            if (!getLetter(data).equals("v")) {
                throw e;
            }
        }

        int successfulWeeks = 0;
        HttpResponseException lastException = null;
		for (Element option : select.children()) {
			String week = option.attr("value");
            String weekName = option.text();
			if (data.optBoolean(PARAM_SINGLE_CLASSES,
                    data.optBoolean("single_classes", false)) // backwards compatibility
                    || data.optString(PARAM_SCHEDULE_TYPE, "substitution").equals("timetable")) {
                int classNumber = 1;
                List<String> classesToSelect;
                if (getLetter(data).equals("v")) {
                    classesToSelect = parseTeachers(getNavbarDoc(), data);
                } else {
                    classesToSelect = parseClasses(getNavbarDoc(), data);
                }
                for (String klasse : classesToSelect) {
                    String url = getScheduleUrl(week, classNumber, data);
					try {
                        parsePage(v, lastChange, klasse, url, weekName, allClasses);
                    } catch (HttpResponseException e) {
                        if (e.getStatusCode() == 500) {
                            // occurs in Hannover_MMBS
                            classNumber++;
{'fraction_non_alphanumeric': 0.11664029072152864, 'fraction_numerical': 0.005040736181935408, 'mean_word_length': 3.4351442682609825, 'pattern_counts': {'":': 0, '<': 210, '<?xml version=': 0, '>': 210, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3020582', 'n_tokens_mistral': 4951, 'n_tokens_neox': 4677, 'n_words': 1399}