<commit_before>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import com.github.sardine.Sardine;
import com.github.sardine.impl.SardineImpl;
import com.mifmif.common.regex.Generex;
import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.networking.MultiTrustManager;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.credential.Credential;
import me.vertretungsplan.objects.credential.UserPasswordCredential;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.fluent.Executor;
import org.apache.http.client.fluent.Request;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.joda.time.LocalDateTime;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.universalchardet.UniversalDetector;

import javax.net.ssl.*;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Base class for {@link SubstitutionScheduleParser} implementations.
 */
public abstract class BaseParser implements SubstitutionScheduleParser {
    public static final String PARAM_CLASS_REGEX = "classRegex";
    private static final String PARAM_SSL_HOSTNAME = "sslHostname";
    private static final String PARAM_SSL_VERIFY_HOSTNAME = "sslVerifyHostname";
    static final String PARAM_CLASS_RANGES = "classRanges";
    static final String PARAM_HEADERS = "headers";
    static final String CLASS_RANGES_CLASS_REGEX = "classRegex";
    static final String CLASS_RANGES_GRADE_REGEX = "gradeRegex";
    static final String CLASS_RANGES_RANGE_FORMAT = "rangeFormat";
    static final String CLASS_RANGES_SINGLE_FORMAT = "singleFormat";

    protected SubstitutionScheduleData scheduleData;
    protected Executor executor;
    protected Credential credential;
    protected CookieStore cookieStore;
    protected ColorProvider colorProvider;
    protected CookieProvider cookieProvider;
    protected UniversalDetector encodingDetector;
    protected DebuggingDataHandler debuggingDataHandler;
    protected Sardine sardine;
    private Path localSource;

    BaseParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        this.scheduleData = scheduleData;
        this.cookieProvider = cookieProvider;
        this.cookieStore = new BasicCookieStore();
        this.colorProvider = new ColorProvider(scheduleData);
        this.encodingDetector = new UniversalDetector(null);
        this.debuggingDataHandler = new NoOpDebuggingDataHandler();
        this.sardine = null;

        try {
            SSLConnectionSocketFactory sslsf = getSslConnectionSocketFactory(scheduleData);

            CloseableHttpClient httpclient = HttpClients.custom()
                    .setSSLSocketFactory(sslsf)
                    .setRedirectStrategy(new LaxRedirectStrategy())
                    .setDefaultRequestConfig(RequestConfig.custom()
                            .setCookieSpec(CookieSpecs.STANDARD).build())
                    .setUserAgent(
                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36")
                    .build();
            this.executor = Executor.newInstance(httpclient).use(cookieStore);
        } catch (GeneralSecurityException | JSONException | IOException e) {
            throw new RuntimeException(e);
        }
    }

    @NotNull private SSLConnectionSocketFactory getSslConnectionSocketFactory(SubstitutionScheduleData scheduleData)
            throws IOException, GeneralSecurityException, JSONException {
        KeyStore ks = loadKeyStore();
        MultiTrustManager multiTrustManager = new MultiTrustManager();
        multiTrustManager.addTrustManager(getDefaultTrustManager());
        multiTrustManager.addTrustManager(trustManagerFromKeystore(ks));

        TrustManager[] trustManagers = new TrustManager[]{multiTrustManager};
        SSLContext sslContext = SSLContext.getInstance("TLS");
        sslContext.init(null, trustManagers, null);
        final HostnameVerifier hostnameVerifier;

        if (scheduleData.getData() != null && scheduleData.getData().has(PARAM_SSL_HOSTNAME)) {
            hostnameVerifier = new CustomHostnameVerifier(scheduleData.getData().getString(PARAM_SSL_HOSTNAME));
        } else if (scheduleData.getData() != null &&
                !scheduleData.getData().optBoolean(PARAM_SSL_VERIFY_HOSTNAME, true)) {
            hostnameVerifier = new NoopHostnameVerifier();
        } else {
            hostnameVerifier = new DefaultHostnameVerifier();
        }
        return new SSLConnectionSocketFactory(
                sslContext,
                new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"},
                null,
                hostnameVerifier);
    }

    /**
     * Create an appropriate parser for a given school. Automatically uses the appropriate subclass depending on
     * {@link SubstitutionScheduleData#getApi()}.
     *
     * @param data a {@link SubstitutionScheduleData} object containing information about the substitution schedule
     * @return a {@link BaseParser} subclass able to parse the given schedule.
     */
    public static BaseParser getInstance(SubstitutionScheduleData data, @Nullable CookieProvider cookieProvider) {
        BaseParser parser = null;
        switch (data.getApi()) {
            case "untis-monitor":
                parser = new UntisMonitorParser(data, cookieProvider);
                break;
            case "untis-info":
                parser = new UntisInfoParser(data, cookieProvider);
                break;
            case "untis-info-headless":
                parser = new UntisInfoHeadlessParser(data, cookieProvider);
                break;
            case "untis-subst":
                parser = new UntisSubstitutionParser(data, cookieProvider);
                break;
            case "dsbmobile":
                parser = new NotCompatibleParser(data, cookieProvider);
                break;
            case "dsblight":
                parser = new NotCompatibleParser(data, cookieProvider);
                break;
            case "svplan":
                parser = new SVPlanParser(data, cookieProvider);
                break;
            case "davinci":
                parser = new DaVinciParser(data, cookieProvider);
                break;
            case "eschool":
                parser = new ESchoolParser(data, cookieProvider);
                break;
            case "turbovertretung":
                parser = new TurboVertretungParser(data, cookieProvider);
                break;
            case "csv":
                parser = new CSVParser(data, cookieProvider);
                break;
            case "legionboard":
                parser = new LegionBoardParser(data, cookieProvider);
                break;
            case "iphis":<commit_msg>add not-compatible Parser<commit_after>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import com.github.sardine.Sardine;
import com.github.sardine.impl.SardineImpl;
import com.mifmif.common.regex.Generex;
import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.networking.MultiTrustManager;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.credential.Credential;
import me.vertretungsplan.objects.credential.UserPasswordCredential;
import org.apache.http.NameValuePair;
import org.apache.http.client.CookieStore;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.fluent.Executor;
import org.apache.http.client.fluent.Request;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.LaxRedirectStrategy;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.joda.time.LocalDateTime;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.universalchardet.UniversalDetector;

import javax.net.ssl.*;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.*;
import java.security.cert.CertificateException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Base class for {@link SubstitutionScheduleParser} implementations.
 */
public abstract class BaseParser implements SubstitutionScheduleParser {
    public static final String PARAM_CLASS_REGEX = "classRegex";
    private static final String PARAM_SSL_HOSTNAME = "sslHostname";
    private static final String PARAM_SSL_VERIFY_HOSTNAME = "sslVerifyHostname";
    static final String PARAM_CLASS_RANGES = "classRanges";
    static final String PARAM_HEADERS = "headers";
    static final String CLASS_RANGES_CLASS_REGEX = "classRegex";
    static final String CLASS_RANGES_GRADE_REGEX = "gradeRegex";
    static final String CLASS_RANGES_RANGE_FORMAT = "rangeFormat";
    static final String CLASS_RANGES_SINGLE_FORMAT = "singleFormat";

    protected SubstitutionScheduleData scheduleData;
    protected Executor executor;
    protected Credential credential;
    protected CookieStore cookieStore;
    protected ColorProvider colorProvider;
    protected CookieProvider cookieProvider;
    protected UniversalDetector encodingDetector;
    protected DebuggingDataHandler debuggingDataHandler;
    protected Sardine sardine;
    private Path localSource;

    BaseParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        this.scheduleData = scheduleData;
        this.cookieProvider = cookieProvider;
        this.cookieStore = new BasicCookieStore();
        this.colorProvider = new ColorProvider(scheduleData);
        this.encodingDetector = new UniversalDetector(null);
        this.debuggingDataHandler = new NoOpDebuggingDataHandler();
        this.sardine = null;

        try {
            SSLConnectionSocketFactory sslsf = getSslConnectionSocketFactory(scheduleData);

            CloseableHttpClient httpclient = HttpClients.custom()
                    .setSSLSocketFactory(sslsf)
                    .setRedirectStrategy(new LaxRedirectStrategy())
                    .setDefaultRequestConfig(RequestConfig.custom()
                            .setCookieSpec(CookieSpecs.STANDARD).build())
                    .setUserAgent(
                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36")
                    .build();
            this.executor = Executor.newInstance(httpclient).use(cookieStore);
        } catch (GeneralSecurityException | JSONException | IOException e) {
            throw new RuntimeException(e);
        }
    }

    @NotNull private SSLConnectionSocketFactory getSslConnectionSocketFactory(SubstitutionScheduleData scheduleData)
            throws IOException, GeneralSecurityException, JSONException {
        KeyStore ks = loadKeyStore();
        MultiTrustManager multiTrustManager = new MultiTrustManager();
        multiTrustManager.addTrustManager(getDefaultTrustManager());
        multiTrustManager.addTrustManager(trustManagerFromKeystore(ks));

        TrustManager[] trustManagers = new TrustManager[]{multiTrustManager};
        SSLContext sslContext = SSLContext.getInstance("TLS");
        sslContext.init(null, trustManagers, null);
        final HostnameVerifier hostnameVerifier;

        if (scheduleData.getData() != null && scheduleData.getData().has(PARAM_SSL_HOSTNAME)) {
            hostnameVerifier = new CustomHostnameVerifier(scheduleData.getData().getString(PARAM_SSL_HOSTNAME));
        } else if (scheduleData.getData() != null &&
                !scheduleData.getData().optBoolean(PARAM_SSL_VERIFY_HOSTNAME, true)) {
            hostnameVerifier = new NoopHostnameVerifier();
        } else {
            hostnameVerifier = new DefaultHostnameVerifier();
        }
        return new SSLConnectionSocketFactory(
                sslContext,
                new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"},
                null,
                hostnameVerifier);
    }

    /**
     * Create an appropriate parser for a given school. Automatically uses the appropriate subclass depending on
     * {@link SubstitutionScheduleData#getApi()}.
     *
     * @param data a {@link SubstitutionScheduleData} object containing information about the substitution schedule
     * @return a {@link BaseParser} subclass able to parse the given schedule.
     */
    public static BaseParser getInstance(SubstitutionScheduleData data, @Nullable CookieProvider cookieProvider) {
        BaseParser parser = null;
        switch (data.getApi()) {
            case "untis-monitor":
                parser = new UntisMonitorParser(data, cookieProvider);
                break;
            case "untis-info":
                parser = new UntisInfoParser(data, cookieProvider);
                break;
            case "untis-info-headless":
                parser = new UntisInfoHeadlessParser(data, cookieProvider);
                break;
            case "untis-subst":
                parser = new UntisSubstitutionParser(data, cookieProvider);
                break;
            case "dsbmobile":
                parser = new NotCompatibleParser(data, cookieProvider);
                break;
            case "dsblight":
                parser = new NotCompatibleParser(data, cookieProvider);
                break;
            case "not-compatible":
                parser = new NotCompatibleParser(data, cookieProvider);
                break;
            case "svplan":
                parser = new SVPlanParser(data, cookieProvider);
                break;
            case "davinci":
                parser = new DaVinciParser(data, cookieProvider);
                break;
            case "eschool":
                parser = new ESchoolParser(data, cookieProvider);
                break;
            case "turbovertretung":
                parser = new TurboVertretungParser(data, cookieProvider);
                break;
            case "csv":
                parser = new CSVParser(data, cookieProvider);
                break;
            case "legionboard":
                parser = new LegionBoardParser(data, cookieProvider);
                break;
            case "iphis":
{'fraction_non_alphanumeric': 0.08362099750029758, 'fraction_numerical': 0.004999404832758005, 'mean_word_length': 3.079388201019665, 'pattern_counts': {'":': 27, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4440338', 'n_tokens_mistral': 4551, 'n_tokens_neox': 4314, 'n_words': 990}