<commit_before>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.jetbrains.annotations.NotNull;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parser for substitution schedules in HTML format created by the <a href="http://www.haneke.de/SvPlan.html">svPlan</a>
 * software.
 * <p>
 * Example: <a href="http://www.ratsschule.de/Vplan/PH_heute.htm">Ratsschule Melle</a>
 * <p>
 * This parser can be accessed using <code>"svplan"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>urls</code> (Array of Strings, required)</dt>
 * <dd>The URLs of the HTML files of the schedule. There is one file for each day.</dd>
 *
 * <dt><code>encoding</code> (String, required)</dt>
 * <dd>The charset of the HTML files. It's probably either UTF-8 or ISO-8859-1.</dd>
 *
 * <dt><code>classes</code> (Array of Strings, required)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>classSeparator</code> (String, optional, Default: <code>", "</code>)</dt>
 * <dd>The string with which multiple classes are separated.</dd>
 *
 * <dt><code>excludeTeachers</code> (Boolean, optional, Default: <code>false</code>)</dt>
 * <dd>Don't show teachers on the schedule.</dd>
 *
 * <dt><code>repeatClass</code> (Boolean, optional, Default: <code>true</code>)</dt>
 * <dd>Whether an empty class column means that there is no class associated with this substitution (false) or that
 * the class from the previous row should be used (true).</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules.
 */
public class SVPlanParser extends BaseParser {

    private static final String PARAM_URLS = "urls";
    private static final String PARAM_ENCODING = "encoding";
    private static final String PARAM_CLASS_SEPARATOR = "classSeparator";
    private static final String PARAM_EXCLUDE_TEACHERS = "excludeTeachers";
    private static final String PARAM_REPEAT_CLASS = "repeatClass";
    private JSONObject data;

    public SVPlanParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore); //

        JSONArray urls = data.getJSONArray(PARAM_URLS);
        String encoding = data.optString(PARAM_ENCODING, null);
        List<Document> docs = new ArrayList<>();

        int successfulSchedules = 0;
        IOException lastException = null;
        for (int i = 0; i < urls.length(); i++) {
            String url;
            if (urls.get(i) instanceof JSONObject) {
                // backwards compatibility
                final JSONObject obj = urls.getJSONObject(i);
                url = obj.getString("url");
                if (obj.has("postData")) {
                    JSONObject postParams = obj.getJSONObject("postData");
                    List<NameValuePair> nvps = new ArrayList<>();
                    for (String name : JSONObject.getNames(postParams)) {
                        String value = postParams.getString(name);
                        nvps.add(new BasicNameValuePair(name, value));
                    }
                    try {
                        docs.add(Jsoup.parse(httpPost(url, encoding, nvps).replace("&nbsp;", "")));
                        successfulSchedules++;
                    } catch (IOException e) {
                        lastException = e;
                    }
                } else {
                    try {
                        docs.add(Jsoup.parse(httpGet(url, encoding).replace("&nbsp;", "")));
                        successfulSchedules++;
                    } catch (IOException e) {
                        lastException = e;
                    }
                }
            } else {
                url = urls.getString(i);
                try {
                    docs.add(Jsoup.parse(httpGet(url, encoding).replace("&nbsp;", "")));
                    successfulSchedules++;
                } catch (IOException e) {
                    lastException = e;
                }
            }
        }
        if (successfulSchedules == 0 && lastException != null) {
            throw lastException;
        }

        SubstitutionSchedule v = parseSVPlanSchedule(docs);
        return v;
    }

    @NotNull
    SubstitutionSchedule parseSVPlanSchedule(List<Document> docs) throws IOException, JSONException {
        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        for (Document doc : docs) {
            if (doc.select(".svp").size() > 0) {
                for (Element svp : doc.select(".svp")) {
                    parseSvPlanDay(v, svp, doc);
                }
            } else if (doc.select(".Trennlinie").size() > 0) {
                Element div = new Element(Tag.valueOf("div"), "");
                for (Node node : doc.body().childNodesCopy()) {
                    if (node instanceof Element && ((Element) node).hasClass("Trennlinie")
                            && div.select("table").size() > 0) {
                        parseSvPlanDay(v, div, doc);
                        div = new Element(Tag.valueOf("div"), "");
                    } else {
                        div.appendChild(node);
                    }
                }
                parseSvPlanDay(v, div, doc);
            } else {
                parseSvPlanDay(v, doc, doc);
            }
        }

        v.setClasses(getAllClasses());
        v.setTeachers(getAllTeachers());
        return v;
    }

    private void parseSvPlanDay(SubstitutionSchedule v, Element svp, Document doc) throws IOException, JSONException {
        SubstitutionScheduleDay day = new SubstitutionScheduleDay();
        if ((svp.select(".svp-plandatum-heute, .svp-plandatum-morgen, .Titel").size() > 0 || doc.title()
                .startsWith("Vertretungsplan für "))) {
            setDate(svp, doc, day);
            if (svp.select(".svp-tabelle, table:has(.Klasse)").size() > 0) {

                Elements rows = svp.select(".svp-tabelle tr, table:has(.Klasse) tr");
                String lastLesson = "";
                String lastClass = "";
                for (Element row : rows) {
                    if ((doc.select(".svp-header").size() > 0 && row.hasClass("svp-header"))
                            || row.select("th").size() > 0 || row.text().trim().equals("")) {
                        continue;
                    }

                    Substitution substitution = new Substitution();

                    for (Element column : row.select("td")) {
                        String type = column.className();
                        if (!hasData(column.text())) {
                            if ((type.startsWith("svp-stunde") || type.startsWith("Stunde")) && hasData(lastLesson)) {
                                substitution.setLesson(lastLesson);
                            } else if ((type.startsWith("svp-klasse") || type.startsWith("Klasse"))
                                    && hasData(lastClass) && data.optBoolean(PARAM_REPEAT_CLASS, true)) {
                                substitution.getClasses().addAll(getClasses(lastClass));
                            }
                            continue;
                        }
                        if (type.startsWith("svp-stunde") || type.startsWith("Stunde")) {
                            substitution.setLesson(column.text());
                            lastLesson = column.text();<commit_msg>SVPlan tables fix<commit_after>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import org.apache.http.NameValuePair;
import org.apache.http.message.BasicNameValuePair;
import org.jetbrains.annotations.NotNull;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parser for substitution schedules in HTML format created by the <a href="http://www.haneke.de/SvPlan.html">svPlan</a>
 * software.
 * <p>
 * Example: <a href="http://www.ratsschule.de/Vplan/PH_heute.htm">Ratsschule Melle</a>
 * <p>
 * This parser can be accessed using <code>"svplan"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>urls</code> (Array of Strings, required)</dt>
 * <dd>The URLs of the HTML files of the schedule. There is one file for each day.</dd>
 *
 * <dt><code>encoding</code> (String, required)</dt>
 * <dd>The charset of the HTML files. It's probably either UTF-8 or ISO-8859-1.</dd>
 *
 * <dt><code>classes</code> (Array of Strings, required)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>classSeparator</code> (String, optional, Default: <code>", "</code>)</dt>
 * <dd>The string with which multiple classes are separated.</dd>
 *
 * <dt><code>excludeTeachers</code> (Boolean, optional, Default: <code>false</code>)</dt>
 * <dd>Don't show teachers on the schedule.</dd>
 *
 * <dt><code>repeatClass</code> (Boolean, optional, Default: <code>true</code>)</dt>
 * <dd>Whether an empty class column means that there is no class associated with this substitution (false) or that
 * the class from the previous row should be used (true).</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules.
 */
public class SVPlanParser extends BaseParser {

    private static final String PARAM_URLS = "urls";
    private static final String PARAM_ENCODING = "encoding";
    private static final String PARAM_CLASS_SEPARATOR = "classSeparator";
    private static final String PARAM_EXCLUDE_TEACHERS = "excludeTeachers";
    private static final String PARAM_REPEAT_CLASS = "repeatClass";
    private JSONObject data;

    public SVPlanParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore); //

        JSONArray urls = data.getJSONArray(PARAM_URLS);
        String encoding = data.optString(PARAM_ENCODING, null);
        List<Document> docs = new ArrayList<>();

        int successfulSchedules = 0;
        IOException lastException = null;
        for (int i = 0; i < urls.length(); i++) {
            String url;
            if (urls.get(i) instanceof JSONObject) {
                // backwards compatibility
                final JSONObject obj = urls.getJSONObject(i);
                url = obj.getString("url");
                if (obj.has("postData")) {
                    JSONObject postParams = obj.getJSONObject("postData");
                    List<NameValuePair> nvps = new ArrayList<>();
                    for (String name : JSONObject.getNames(postParams)) {
                        String value = postParams.getString(name);
                        nvps.add(new BasicNameValuePair(name, value));
                    }
                    try {
                        docs.add(Jsoup.parse(httpPost(url, encoding, nvps).replace("&nbsp;", "")));
                        successfulSchedules++;
                    } catch (IOException e) {
                        lastException = e;
                    }
                } else {
                    try {
                        docs.add(Jsoup.parse(httpGet(url, encoding).replace("&nbsp;", "")));
                        successfulSchedules++;
                    } catch (IOException e) {
                        lastException = e;
                    }
                }
            } else {
                url = urls.getString(i);
                try {
                    docs.add(Jsoup.parse(httpGet(url, encoding).replace("&nbsp;", "")));
                    successfulSchedules++;
                } catch (IOException e) {
                    lastException = e;
                }
            }
        }
        if (successfulSchedules == 0 && lastException != null) {
            throw lastException;
        }

        SubstitutionSchedule v = parseSVPlanSchedule(docs);
        return v;
    }

    @NotNull
    SubstitutionSchedule parseSVPlanSchedule(List<Document> docs) throws IOException, JSONException {
        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        for (Document doc : docs) {
            if (doc.select(".svp").size() > 0) {
                for (Element svp : doc.select(".svp")) {
                    parseSvPlanDay(v, svp, doc);
                }
            } else if (doc.select(".Trennlinie").size() > 0) {
                Element div = new Element(Tag.valueOf("div"), "");
                for (Node node : doc.body().childNodesCopy()) {
                    if (node instanceof Element && ((Element) node).hasClass("Trennlinie")
                            && div.select("table").size() > 0) {
                        parseSvPlanDay(v, div, doc);
                        div = new Element(Tag.valueOf("div"), "");
                    } else {
                        div.appendChild(node);
                    }
                }
                parseSvPlanDay(v, div, doc);
            } else {
                parseSvPlanDay(v, doc, doc);
            }
        }

        v.setClasses(getAllClasses());
        v.setTeachers(getAllTeachers());
        return v;
    }

    private void parseSvPlanDay(SubstitutionSchedule v, Element svp, Document doc) throws IOException, JSONException {
        SubstitutionScheduleDay day = new SubstitutionScheduleDay();
        if ((svp.select(".svp-plandatum-heute, .svp-plandatum-morgen, .Titel").size() > 0 || doc.title()
                .startsWith("Vertretungsplan für "))) {
            setDate(svp, doc, day);
            final Elements tables = svp.select(".svp-tabelle, table:has(.Klasse)");
            if (tables.size() > 0) {
                Iterator<Element> iter = tables.iterator();
                while (iter.hasNext()) {
                    Element table = iter.next();
                    if (!table.hasClass("svp-tabelle") && table.select("> tbody > tr > td.Klasse").size() == 0) {
                        iter.remove();
                    }
                }

                Elements rows = tables.select("tr");
                String lastLesson = "";
                String lastClass = "";
                for (Element row : rows) {
                    if ((doc.select(".svp-header").size() > 0 && row.hasClass("svp-header"))
                            || row.select("th").size() > 0 || row.text().trim().equals("")) {
                        continue;
                    }

                    Substitution substitution = new Substitution();

                    for (Element column : row.select("td")) {
                        String type = column.className();
                        if (!hasData(column.text())) {
                            if ((type.startsWith("svp-stunde") || type.startsWith("Stunde")) && hasData(lastLesson)) {
                                substitution.setLesson(lastLesson);
                            } else if ((type.startsWith("svp-klasse") || type.startsWith("Klasse"))
                                    && hasData(lastClass) && data.optBoolean(PARAM_REPEAT_CLASS, true)) {
                                substitution.getClasses().addAll(getClasses(lastClass));
                            }
                            continue;
                        }
                        if (type.startsWith("svp-stunde") || type.startsWith("Stunde")) {
                            substitution.setLesson(column.text());
                            lastLesson = column.text();
{'fraction_non_alphanumeric': 0.12027137042062415, 'fraction_numerical': 0.002876526458616011, 'mean_word_length': 2.0848819688598694, 'pattern_counts': {'":': 0, '<': 126, '<?xml version=': 0, '>': 141, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3019468', 'n_tokens_mistral': 5103, 'n_tokens_neox': 4814, 'n_words': 1168}