<commit_before>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.AdditionalInfo;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import me.vertretungsplan.objects.authentication.SchoolNumberPasswordAuthenticationData;
import me.vertretungsplan.objects.credential.SchoolNumberPasswordCredential;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.http.entity.ContentType;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.ISODateTimeFormat;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Parser for Untis and DaVinci substitution schedules served by
 * <a href="http://www.digitales-schwarzes-brett.de/">DSB</a>mobile
 * (<a href="http://mobile.dsbcontrol.de/">mobile.dsbcontrol.de</a>).
 * <p>
 * This parser can be accessed using <code>"dsbmobile"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>classes</code> (Array of Strings, required)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>type</code> (String, optional)</dt>
 * <dd>Can be set to either <code>"untis"</code> or <code>"davinci"</code> to specify which type of schedule is
 * used. By default, the parser tries to detect this automatically, but this does not always work.</dd>
 *
 * <dt><code>encoding</code> (String, optional)</dt>
 * <dd>The charset of the Untis/DaVinci schedule. DSBmobile itself always uses UTF-8, but the hosted HTML schedule can
 * also be ISO-8859-1. Default: UTF-8</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link UntisCommonParser} (if it is an Untis
 * schedule).
 *
 * DSBmobile schedules always need a login using a school number and a password. You have to set a
 * {@link SchoolNumberPasswordAuthenticationData} which specifies the 6-digit school number.
 */
public class DSBMobileParser extends UntisCommonParser {

    private static final String URL = "https://app.dsbcontrol.de/JsonHandlerWeb.ashx/GetData";
    private static final String ENCODING = "UTF-8";
    private static final String PARAM_ENCODING = "encoding";
    private static final String PARAM_TYPE = "type";
    private JSONObject data;

    public DSBMobileParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    private static String decode(String input) throws IOException {
        byte[] inputBytes = Base64.decodeBase64(input);
        InputStream is = new GZIPInputStream(new ByteArrayInputStream(inputBytes));
        return IOUtils.toString(is, ENCODING);
    }

    private static String encode(String input) throws IOException {
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        OutputStream gzipOs = new GZIPOutputStream(os);
        gzipOs.write(input.getBytes(ENCODING));
        gzipOs.close();
        byte[] outputBytes = os.toByteArray();
        return Base64.encodeBase64String(outputBytes);
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        if (credential == null || !(credential instanceof SchoolNumberPasswordCredential)
                || ((SchoolNumberPasswordCredential) credential).getPassword() == null
                || ((SchoolNumberPasswordCredential) credential).getPassword().isEmpty()) {
            throw new IOException("no login");
        }
        String login = ((SchoolNumberPasswordAuthenticationData) scheduleData.getAuthenticationData()).getSchoolNumber();
        String password = ((SchoolNumberPasswordCredential) credential).getPassword();

        JSONObject data = getDataFromApi(login, password);

        int resultcode = data.getInt("Resultcode");
        if (resultcode == 1) {
            throw new CredentialInvalidException();
        } else if (resultcode != 0) throw new IOException("Resultcode " + resultcode);

        JSONObject timetablePage = getPageByType(data, "timetable");
        if (timetablePage == null) throw new IOException("no timetable page found");
        JSONArray timetableModules = timetablePage.getJSONObject("Root").getJSONArray("Childs" /* sic! */);

        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        List<String> usedUrls = new ArrayList<>();
        int successfulUrls = 0;
        int totalUrls = 0;
        for (int i = 0; i < timetableModules.length(); i++) {
            JSONArray timetableParts = timetableModules.getJSONObject(i).getJSONArray("Childs" /* sic! */);
            for (int j = 0; j < timetableParts.length(); j++) {
                JSONObject timetablePart = timetableParts.getJSONObject(j);<commit_msg>DSBmobile: add optional pageType option<commit_after>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.AdditionalInfo;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import me.vertretungsplan.objects.authentication.SchoolNumberPasswordAuthenticationData;
import me.vertretungsplan.objects.credential.SchoolNumberPasswordCredential;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.IOUtils;
import org.apache.http.entity.ContentType;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.ISODateTimeFormat;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Parser for Untis and DaVinci substitution schedules served by
 * <a href="http://www.digitales-schwarzes-brett.de/">DSB</a>mobile
 * (<a href="http://mobile.dsbcontrol.de/">mobile.dsbcontrol.de</a>).
 * <p>
 * This parser can be accessed using <code>"dsbmobile"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>classes</code> (Array of Strings, required)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>type</code> (String, optional)</dt>
 * <dd>Can be set to either <code>"untis"</code> or <code>"davinci"</code> to specify which type of schedule is
 * used. By default, the parser tries to detect this automatically, but this does not always work.</dd>
 *
 * <dt><code>encoding</code> (String, optional)</dt>
 * <dd>The charset of the Untis/DaVinci schedule. DSBmobile itself always uses UTF-8, but the hosted HTML schedule can
 * also be ISO-8859-1. Default: UTF-8</dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link UntisCommonParser} (if it is an Untis
 * schedule).
 *
 * DSBmobile schedules always need a login using a school number and a password. You have to set a
 * {@link SchoolNumberPasswordAuthenticationData} which specifies the 6-digit school number.
 */
public class DSBMobileParser extends UntisCommonParser {

    private static final String URL = "https://app.dsbcontrol.de/JsonHandlerWeb.ashx/GetData";
    private static final String ENCODING = "UTF-8";
    private static final String PARAM_ENCODING = "encoding";
    private static final String PARAM_TYPE = "type";
    private JSONObject data;

    public DSBMobileParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    private static String decode(String input) throws IOException {
        byte[] inputBytes = Base64.decodeBase64(input);
        InputStream is = new GZIPInputStream(new ByteArrayInputStream(inputBytes));
        return IOUtils.toString(is, ENCODING);
    }

    private static String encode(String input) throws IOException {
        ByteArrayOutputStream os = new ByteArrayOutputStream();
        OutputStream gzipOs = new GZIPOutputStream(os);
        gzipOs.write(input.getBytes(ENCODING));
        gzipOs.close();
        byte[] outputBytes = os.toByteArray();
        return Base64.encodeBase64String(outputBytes);
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        if (credential == null || !(credential instanceof SchoolNumberPasswordCredential)
                || ((SchoolNumberPasswordCredential) credential).getPassword() == null
                || ((SchoolNumberPasswordCredential) credential).getPassword().isEmpty()) {
            throw new IOException("no login");
        }
        String login = ((SchoolNumberPasswordAuthenticationData) scheduleData.getAuthenticationData()).getSchoolNumber();
        String password = ((SchoolNumberPasswordCredential) credential).getPassword();

        JSONObject data = getDataFromApi(login, password);

        int resultcode = data.getInt("Resultcode");
        if (resultcode == 1) {
            throw new CredentialInvalidException();
        } else if (resultcode != 0) throw new IOException("Resultcode " + resultcode);

        JSONObject timetablePage = getPageByType(data, scheduleData.getData().optString("pageType", "timetable"));
        if (timetablePage == null) throw new IOException("no timetable page found");
        JSONArray timetableModules = timetablePage.getJSONObject("Root").getJSONArray("Childs" /* sic! */);

        SubstitutionSchedule v = SubstitutionSchedule.fromData(scheduleData);

        List<String> usedUrls = new ArrayList<>();
        int successfulUrls = 0;
        int totalUrls = 0;
        for (int i = 0; i < timetableModules.length(); i++) {
            JSONArray timetableParts = timetableModules.getJSONObject(i).getJSONArray("Childs" /* sic! */);
            for (int j = 0; j < timetableParts.length(); j++) {
                JSONObject timetablePart = timetableParts.getJSONObject(j);
{'fraction_non_alphanumeric': 0.1073078897826459, 'fraction_numerical': 0.005990073592332706, 'mean_word_length': 4.15299823633157, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 75, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '938313', 'n_tokens_mistral': 3358, 'n_tokens_neox': 3172, 'n_words': 886}