<commit_before>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import com.github.sardine.DavResource;
import com.github.sardine.Sardine;
import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import me.vertretungsplan.objects.credential.UserPasswordCredential;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.fluent.Request;
import org.jetbrains.annotations.NotNull;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;

/**
 * Generic parser for substitution schedules in CSV format.
 * <p>
 * This parser can be accessed using <code>"csv"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>url</code> (String, required)</dt>
 * <dd>The url of the CSV file to be fetched</dd>
 *
 * <dt><code>separator</code> (String, required)</dt>
 * <dd>The separator used in the CSV file (such as <code>","</code>, <code>";"</code> or <code>"\t"</code>)</dd>
 *
 * <dt><code>columns</code> (Array of Strings, required)</dt>
 * <dd>The order of columns used in the CSV file. Entries can be: <code>"lesson", "subject",
 * "previousSubject", "type", "type-entfall", "room", "previousRoom", "teacher", "previousTeacher", desc",
 * "desc-type", "class", "day", "stand", "ignore"</code></dd>
 *
 * <dt><code>classes</code> (Array of Strings, required if <code>classesUrl</code> not specified)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>website</code> (String, recommended)</dt>
 * <dd>The URL of a website where the substitution schedule can be seen online</dd>
 *
 * <dt><code>skipLines</code> (Integer, optional)</dt>
 * <dd>The number of lines to skip at the beginning of the CSV file. Default: <code>0</code></dd>
 *
 * <dt><code>classesUrl</code> (String, optional)</dt>
 * <dd>The URL of an additional CSV file containing the classes, one per line</dd>
 *
 * <dt><code>classRegex</code> (String, optional)</dt>
 * <dd>RegEx to modify the classes set on the schedule (in {@link #getSubstitutionSchedule()}, not
 * {@link #getAllClasses()}. The RegEx is matched against the class using {@link Matcher#find()}. If the RegEx
 * contains a group, the content of the first group {@link Matcher#group(int)} is used as the resulting class.
 * Otherwise, {@link Matcher#group()} is used. If the RegEx cannot be matched ({@link Matcher#find()} returns
 * <code>false</code>), the class is set to an empty string.
 * </dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules.
 */
public class CSVParser extends BaseParser {

    private static final String PARAM_SEPARATOR = "separator";
    private static final String PARAM_SKIP_LINES = "skipLines";
    private static final String PARAM_COLUMNS = "columns";
    private static final String PARAM_WEBSITE = "website";
    private static final String PARAM_CLASSES_URL = "classesUrl";
    private static final String PARAM_CLASSES = "classes";
    private static final String PARAM_URL = "url";
    private JSONObject data;

    public CSVParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        String url = data.getString(PARAM_URL);
        SubstitutionSchedule schedule = SubstitutionSchedule.fromData(scheduleData);

        if (url.startsWith("webdav")) {
            UserPasswordCredential credential = (UserPasswordCredential) this.credential;
            try {
                Sardine client = getWebdavClient(credential);
                String httpUrl = url.replaceFirst("webdav", "http");
                List<DavResource> files = client.list(httpUrl);
                for (DavResource file : files) {
                    if (!file.isDirectory()) {
                        InputStream stream = client.get(new URI(httpUrl).resolve(file.getHref()).toString());
                        parseCSV(IOUtils.toString(stream, "UTF-8"), schedule);
                    }
                }
                return schedule;
            } catch (GeneralSecurityException | URISyntaxException e) {
                throw new IOException(e);
            }
        } else {
            new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore);
            String response = httpGet(url);
            return parseCSV(response, schedule);
        }
    }

    @NotNull
    SubstitutionSchedule parseCSV(String response, SubstitutionSchedule schedule)
            throws JSONException, IOException, CredentialInvalidException {
        String[] lines = response.split("\n");

        String separator = data.getString(PARAM_SEPARATOR);
        for (int i = data.optInt(PARAM_SKIP_LINES, 0); i < lines.length; i++) {
            String[] columns = lines[i].split(separator);
            Substitution v = new Substitution();
            String dayName = null;
            String stand = "";
            int j = 0;
            for (String column:columns) {
                String type = data.getJSONArray(PARAM_COLUMNS)
                        .getString(j);
                switch (type) {
                    case "lesson":
                        v.setLesson(column);
                        break;
                    case "subject":
                        v.setSubject(column);
                        break;
                    case "previousSubject":
                        v.setPreviousSubject(column);
                        break;
                    case "type":
                        v.setType(column);
                        v.setColor(colorProvider.getColor(column));
                        break;
                    case "type-entfall":
                        if (column.equals("x")) {
                            v.setType("Entfall");
                            v.setColor(colorProvider.getColor("Entfall"));
                        } else {
                            v.setType("Vertretung");
                            v.setColor(colorProvider.getColor("Vertretung"));
                        }
                        break;
                    case "room":
                        v.setRoom(column);
                        break;
                    case "teacher":
                        v.setTeacher(column);
                        break;
                    case "previousTeacher":
                        v.setPreviousTeacher(column);
                        break;
                    case "desc":
                        v.setDesc(column);
                        break;
                    case "desc-type":
                        v.setDesc(column);
                        String recognizedType = recognizeType(column);
                        v.setType(recognizedType);
                        v.setColor(colorProvider.getColor(recognizedType));
                        break;
                    case "previousRoom":
                        v.setPreviousRoom(column);
                        break;
                    case "class":
                        UntisCommonParser.handleClasses(data, v, column, getAllClasses());
                        break;
                    case "day":
                        dayName = column;
                        break;
                    case "stand":
                        stand = column;
                        break;
                    case "ignore":
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown column type: " + type);
                }
                j++;
            }
            if (v.getType() == null) {
                v.setType("Vertretung");
                v.setColor(colorProvider.getColor("Vertretung"));
            }

            if (dayName != null && !dayName.isEmpty()) {
                SubstitutionScheduleDay day = new SubstitutionScheduleDay();
                day.setDateString(dayName);
                day.setDate(ParserUtils.parseDate(dayName));
                day.setLastChangeString(stand);
                day.setLastChange(ParserUtils.parseDateTime(stand));
                day.addSubstitution(v);
                schedule.addDay(day);
            }
        }

        if (scheduleData.getData().has(PARAM_WEBSITE)) {
            schedule.setWebsite(scheduleData.getData().getString(PARAM_WEBSITE));
        }

        schedule.setClasses(getAllClasses());
        schedule.setTeachers(getAllTeachers());

        return schedule;
    }

    @Override
    public List<String> getAllClasses() throws IOException, JSONException {
        if (data.has(PARAM_CLASSES_URL)) {
            String url = data.getString(PARAM_CLASSES_URL);
            String response = executor.execute(Request.Get(url)).returnContent().asString();
            List<String> classes = new ArrayList<>();
            for (String string:response.split("\n")) {
                classes.add(string.trim());
            }
            return classes;
        } else {
            return getClassesFromJson();
        }
    }

    @Override
    public List<String> getAllTeachers() {
        return null;
    }
}
<commit_msg>CSVParser: include last modified date from webdav server<commit_after>/*
 * substitution-schedule-parser - Java library for parsing schools' substitution schedules
 * Copyright (c) 2016 <NAME>
 *
 * This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0.
 * If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
 */

package me.vertretungsplan.parser;

import com.github.sardine.DavResource;
import com.github.sardine.Sardine;
import me.vertretungsplan.exception.CredentialInvalidException;
import me.vertretungsplan.objects.Substitution;
import me.vertretungsplan.objects.SubstitutionSchedule;
import me.vertretungsplan.objects.SubstitutionScheduleData;
import me.vertretungsplan.objects.SubstitutionScheduleDay;
import me.vertretungsplan.objects.credential.UserPasswordCredential;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.fluent.Request;
import org.jetbrains.annotations.NotNull;
import org.joda.time.LocalDateTime;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.GeneralSecurityException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;

/**
 * Generic parser for substitution schedules in CSV format.
 * <p>
 * This parser can be accessed using <code>"csv"</code> for {@link SubstitutionScheduleData#setApi(String)}.
 *
 * <h4>Configuration parameters</h4>
 * These parameters can be supplied in {@link SubstitutionScheduleData#setData(JSONObject)} to configure the parser:
 *
 * <dl>
 * <dt><code>url</code> (String, required)</dt>
 * <dd>The url of the CSV file to be fetched</dd>
 *
 * <dt><code>separator</code> (String, required)</dt>
 * <dd>The separator used in the CSV file (such as <code>","</code>, <code>";"</code> or <code>"\t"</code>)</dd>
 *
 * <dt><code>columns</code> (Array of Strings, required)</dt>
 * <dd>The order of columns used in the CSV file. Entries can be: <code>"lesson", "subject",
 * "previousSubject", "type", "type-entfall", "room", "previousRoom", "teacher", "previousTeacher", desc",
 * "desc-type", "class", "day", "stand", "ignore"</code></dd>
 *
 * <dt><code>classes</code> (Array of Strings, required if <code>classesUrl</code> not specified)</dt>
 * <dd>The list of all classes, as they can appear in the schedule</dd>
 *
 * <dt><code>website</code> (String, recommended)</dt>
 * <dd>The URL of a website where the substitution schedule can be seen online</dd>
 *
 * <dt><code>skipLines</code> (Integer, optional)</dt>
 * <dd>The number of lines to skip at the beginning of the CSV file. Default: <code>0</code></dd>
 *
 * <dt><code>classesUrl</code> (String, optional)</dt>
 * <dd>The URL of an additional CSV file containing the classes, one per line</dd>
 *
 * <dt><code>classRegex</code> (String, optional)</dt>
 * <dd>RegEx to modify the classes set on the schedule (in {@link #getSubstitutionSchedule()}, not
 * {@link #getAllClasses()}. The RegEx is matched against the class using {@link Matcher#find()}. If the RegEx
 * contains a group, the content of the first group {@link Matcher#group(int)} is used as the resulting class.
 * Otherwise, {@link Matcher#group()} is used. If the RegEx cannot be matched ({@link Matcher#find()} returns
 * <code>false</code>), the class is set to an empty string.
 * </dd>
 * </dl>
 *
 * Additionally, this parser supports the parameters specified in {@link LoginHandler} for login-protected schedules.
 */
public class CSVParser extends BaseParser {

    private static final String PARAM_SEPARATOR = "separator";
    private static final String PARAM_SKIP_LINES = "skipLines";
    private static final String PARAM_COLUMNS = "columns";
    private static final String PARAM_WEBSITE = "website";
    private static final String PARAM_CLASSES_URL = "classesUrl";
    private static final String PARAM_CLASSES = "classes";
    private static final String PARAM_URL = "url";
    private JSONObject data;

    public CSVParser(SubstitutionScheduleData scheduleData, CookieProvider cookieProvider) {
        super(scheduleData, cookieProvider);
        data = scheduleData.getData();
    }

    @Override
    public SubstitutionSchedule getSubstitutionSchedule() throws IOException, JSONException,
            CredentialInvalidException {
        String url = data.getString(PARAM_URL);
        SubstitutionSchedule schedule = SubstitutionSchedule.fromData(scheduleData);

        if (url.startsWith("webdav")) {
            UserPasswordCredential credential = (UserPasswordCredential) this.credential;
            try {
                Sardine client = getWebdavClient(credential);
                String httpUrl = url.replaceFirst("webdav", "http");
                List<DavResource> files = client.list(httpUrl);
                for (DavResource file : files) {
                    if (!file.isDirectory()) {
                        LocalDateTime modified = new LocalDateTime(file.getModified());
                        if (schedule.getLastChange() == null || schedule.getLastChange().isBefore(modified)) {
                            schedule.setLastChange(modified);
                        }
                        InputStream stream = client.get(new URI(httpUrl).resolve(file.getHref()).toString());
                        parseCSV(IOUtils.toString(stream, "UTF-8"), schedule);
                    }
                }
                return schedule;
            } catch (GeneralSecurityException | URISyntaxException e) {
                throw new IOException(e);
            }
        } else {
            new LoginHandler(scheduleData, credential, cookieProvider).handleLogin(executor, cookieStore);
            String response = httpGet(url);
            return parseCSV(response, schedule);
        }
    }

    @NotNull
    SubstitutionSchedule parseCSV(String response, SubstitutionSchedule schedule)
            throws JSONException, IOException, CredentialInvalidException {
        String[] lines = response.split("\n");

        String separator = data.getString(PARAM_SEPARATOR);
        for (int i = data.optInt(PARAM_SKIP_LINES, 0); i < lines.length; i++) {
            String[] columns = lines[i].split(separator);
            Substitution v = new Substitution();
            String dayName = null;
            String stand = "";
            int j = 0;
            for (String column:columns) {
                String type = data.getJSONArray(PARAM_COLUMNS)
                        .getString(j);
                switch (type) {
                    case "lesson":
                        v.setLesson(column);
                        break;
                    case "subject":
                        v.setSubject(column);
                        break;
                    case "previousSubject":
                        v.setPreviousSubject(column);
                        break;
                    case "type":
                        v.setType(column);
                        v.setColor(colorProvider.getColor(column));
                        break;
                    case "type-entfall":
                        if (column.equals("x")) {
                            v.setType("Entfall");
                            v.setColor(colorProvider.getColor("Entfall"));
                        } else {
                            v.setType("Vertretung");
                            v.setColor(colorProvider.getColor("Vertretung"));
                        }
                        break;
                    case "room":
                        v.setRoom(column);
                        break;
                    case "teacher":
                        v.setTeacher(column);
                        break;
                    case "previousTeacher":
                        v.setPreviousTeacher(column);
                        break;
                    case "desc":
                        v.setDesc(column);
                        break;
                    case "desc-type":
                        v.setDesc(column);
                        String recognizedType = recognizeType(column);
                        v.setType(recognizedType);
                        v.setColor(colorProvider.getColor(recognizedType));
                        break;
                    case "previousRoom":
                        v.setPreviousRoom(column);
                        break;
                    case "class":
                        UntisCommonParser.handleClasses(data, v, column, getAllClasses());
                        break;
                    case "day":
                        dayName = column;
                        break;
                    case "stand":
                        stand = column;
                        break;
                    case "ignore":
                        break;
                    default:
                        throw new IllegalArgumentException("Unknown column type: " + type);
                }
                j++;
            }
            if (v.getType() == null) {
                v.setType("Vertretung");
                v.setColor(colorProvider.getColor("Vertretung"));
            }

            if (dayName != null && !dayName.isEmpty()) {
                SubstitutionScheduleDay day = new SubstitutionScheduleDay();
                day.setDateString(dayName);
                day.setDate(ParserUtils.parseDate(dayName));
                day.setLastChangeString(stand);
                day.setLastChange(ParserUtils.parseDateTime(stand));
                day.addSubstitution(v);
                schedule.addDay(day);
            }
        }

        if (scheduleData.getData().has(PARAM_WEBSITE)) {
            schedule.setWebsite(scheduleData.getData().getString(PARAM_WEBSITE));
        }

        schedule.setClasses(getAllClasses());
        schedule.setTeachers(getAllTeachers());

        return schedule;
    }

    @Override
    public List<String> getAllClasses() throws IOException, JSONException {
        if (data.has(PARAM_CLASSES_URL)) {
            String url = data.getString(PARAM_CLASSES_URL);
            String response = executor.execute(Request.Get(url)).returnContent().asString();
            List<String> classes = new ArrayList<>();
            for (String string:response.split("\n")) {
                classes.add(string.trim());
            }
            return classes;
        } else {
            return getClassesFromJson();
        }
    }

    @Override
    public List<String> getAllTeachers() {
        return null;
    }
}

{'fraction_non_alphanumeric': 0.10698898994734322, 'fraction_numerical': 0.0013403542364767832, 'mean_word_length': 2.02155047729245, 'pattern_counts': {'":': 30, '<': 155, '<?xml version=': 0, '>': 153, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2274667', 'n_tokens_mistral': 5502, 'n_tokens_neox': 5169, 'n_words': 1336}