<commit_before>/*
 *    Copyright 2017 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 *
 */

package eu.fthevenet.binjr.sources.jrds.adapters;

import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import eu.fthevenet.binjr.data.adapters.DataAdapter;
import eu.fthevenet.binjr.data.adapters.HttpDataAdapterBase;
import eu.fthevenet.binjr.data.adapters.TimeSeriesBinding;
import eu.fthevenet.binjr.data.codec.CsvDecoder;
import eu.fthevenet.binjr.data.exceptions.*;
import eu.fthevenet.binjr.data.timeseries.DoubleTimeSeriesProcessor;
import eu.fthevenet.binjr.dialogs.Dialogs;
import eu.fthevenet.binjr.sources.jrds.adapters.json.JsonJrdsItem;
import eu.fthevenet.binjr.sources.jrds.adapters.json.JsonJrdsTree;
import eu.fthevenet.util.xml.XmlUtils;
import javafx.beans.value.ChangeListener;
import javafx.beans.value.ObservableValue;
import javafx.scene.control.TreeItem;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.AbstractResponseHandler;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import javax.xml.bind.JAXB;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * This class provides an implementation of {@link DataAdapter} for JRDS.
 *
 * @author <NAME>
 */
@XmlAccessorType(XmlAccessType.FIELD)
public class JrdsDataAdapter extends HttpDataAdapterBase<Double, CsvDecoder<Double>> {
    private static final Logger logger = LogManager.getLogger(JrdsDataAdapter.class);
    private static final char DELIMITER = ',';
    public static final String JRDS_FILTER = "filter";
    public static final String JRDS_TREE = "tree";

    protected static final String ENCODING_PARAM_NAME = "encoding";
    protected static final String ZONE_ID_PARAM_NAME = "zoneId";
    protected static final String TREE_VIEW_TAB_PARAM_NAME = "treeViewTab";
    private final JrdsSeriesBindingFactory bindingFactory = new JrdsSeriesBindingFactory();
    private final static Pattern uriSchemePattern = Pattern.compile("^[a-zA-Z]*://");
    private String filter;
    private ZoneId zoneId;
    private String encoding;
    private JrdsTreeViewTab treeViewTab;

    /**
     * Default constructor
     */
    public JrdsDataAdapter() throws DataAdapterException {
        super();
    }

    /**
     * Initializes a new instance of the {@link JrdsDataAdapter} class.
     *
     * @param zoneId      the id of the time zone used to record dates.
     * @param encoding    the encoding used by the download servlet.
     * @param treeViewTab the filter to apply to the tree view
     */
    public JrdsDataAdapter(URL baseURL, ZoneId zoneId, String encoding, JrdsTreeViewTab treeViewTab, String filter) throws DataAdapterException {
        super(baseURL);
        this.zoneId = zoneId;
        this.encoding = encoding;
        this.treeViewTab = treeViewTab;
        this.filter = filter;
    }

    /**
     * Builds a new instance of the {@link JrdsDataAdapter} class from the provided parameters.
     *
     * @param address the URL to the JRDS webapp.
     * @param zoneId  the id of the time zone used to record dates.
     * @return a new instance of the {@link JrdsDataAdapter} class.
     */
    public static JrdsDataAdapter fromUrl(String address, ZoneId zoneId, JrdsTreeViewTab treeViewTab, String filter) throws DataAdapterException {
        try {
            // Detect if URL protocol is present. If not, assume http.
            if (!uriSchemePattern.matcher(address).find()) {
                address = "http://" + address;
            }
            URL url = new URL(address);
            if (url.getHost().trim().isEmpty()) {
                throw new CannotInitializeDataAdapterException("Malformed URL: no host");
            }
            return new JrdsDataAdapter(url, zoneId, "utf-8", treeViewTab, filter);
        } catch (MalformedURLException e) {
            throw new CannotInitializeDataAdapterException("Malformed URL: " + e.getMessage(), e);
        }
    }

    //region [DataAdapter Members]

    @Override
    public TreeItem<TimeSeriesBinding<Double>> getBindingTree() throws DataAdapterException {
        Gson gson = new Gson();
        try {
            JsonJrdsTree t = gson.fromJson(getJsonTree(treeViewTab.getCommand(), treeViewTab.getArgument(), filter), JsonJrdsTree.class);
            Map<String, JsonJrdsItem> m = Arrays.stream(t.items).collect(Collectors.toMap(o -> o.id, (o -> o)));
            TreeItem<TimeSeriesBinding<Double>> tree = new TreeItem<>(bindingFactory.of("", getSourceName(), "/", this));
            for (JsonJrdsItem branch : Arrays.stream(t.items).filter(jsonJrdsItem -> JRDS_TREE.equals(jsonJrdsItem.type) || JRDS_FILTER.equals(jsonJrdsItem.type)).collect(Collectors.toList())) {
                attachNode(tree, branch.id, m);
            }
            return tree;
        } catch (JsonParseException e) {
            throw new DataAdapterException("An error occurred while parsing the json response to getBindingTree request", e);
        } catch (URISyntaxException e) {
            throw new SourceCommunicationException("Error building URI for request", e);
        }
    }

    @Override
    protected URI craftFetchUri(String path, Instant begin, Instant end) throws DataAdapterException {<commit_msg>Fixed JRDS adapter fails to connect to source if url contains a trailing slash<commit_after>/*
 *    Copyright 2017 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 *
 */

package eu.fthevenet.binjr.sources.jrds.adapters;

import com.google.gson.Gson;
import com.google.gson.JsonParseException;
import eu.fthevenet.binjr.data.adapters.DataAdapter;
import eu.fthevenet.binjr.data.adapters.HttpDataAdapterBase;
import eu.fthevenet.binjr.data.adapters.TimeSeriesBinding;
import eu.fthevenet.binjr.data.codec.CsvDecoder;
import eu.fthevenet.binjr.data.exceptions.*;
import eu.fthevenet.binjr.data.timeseries.DoubleTimeSeriesProcessor;
import eu.fthevenet.binjr.dialogs.Dialogs;
import eu.fthevenet.binjr.sources.jrds.adapters.json.JsonJrdsItem;
import eu.fthevenet.binjr.sources.jrds.adapters.json.JsonJrdsTree;
import eu.fthevenet.util.xml.XmlUtils;
import javafx.beans.value.ChangeListener;
import javafx.beans.value.ObservableValue;
import javafx.scene.control.TreeItem;
import org.apache.http.HttpEntity;
import org.apache.http.NameValuePair;
import org.apache.http.StatusLine;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.AbstractResponseHandler;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import javax.xml.bind.JAXB;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * This class provides an implementation of {@link DataAdapter} for JRDS.
 *
 * @author <NAME>
 */
@XmlAccessorType(XmlAccessType.FIELD)
public class JrdsDataAdapter extends HttpDataAdapterBase<Double, CsvDecoder<Double>> {
    private static final Logger logger = LogManager.getLogger(JrdsDataAdapter.class);
    private static final char DELIMITER = ',';
    public static final String JRDS_FILTER = "filter";
    public static final String JRDS_TREE = "tree";

    protected static final String ENCODING_PARAM_NAME = "encoding";
    protected static final String ZONE_ID_PARAM_NAME = "zoneId";
    protected static final String TREE_VIEW_TAB_PARAM_NAME = "treeViewTab";
    private final JrdsSeriesBindingFactory bindingFactory = new JrdsSeriesBindingFactory();
    private final static Pattern uriSchemePattern = Pattern.compile("^[a-zA-Z]*://");
    private String filter;
    private ZoneId zoneId;
    private String encoding;
    private JrdsTreeViewTab treeViewTab;

    /**
     * Default constructor
     */
    public JrdsDataAdapter() throws DataAdapterException {
        super();
    }

    /**
     * Initializes a new instance of the {@link JrdsDataAdapter} class.
     *
     * @param zoneId      the id of the time zone used to record dates.
     * @param encoding    the encoding used by the download servlet.
     * @param treeViewTab the filter to apply to the tree view
     */
    public JrdsDataAdapter(URL baseURL, ZoneId zoneId, String encoding, JrdsTreeViewTab treeViewTab, String filter) throws DataAdapterException {
        super(baseURL);
        this.zoneId = zoneId;
        this.encoding = encoding;
        this.treeViewTab = treeViewTab;
        this.filter = filter;
    }

    /**
     * Builds a new instance of the {@link JrdsDataAdapter} class from the provided parameters.
     *
     * @param address the URL to the JRDS webapp.
     * @param zoneId  the id of the time zone used to record dates.
     * @return a new instance of the {@link JrdsDataAdapter} class.
     */
    public static JrdsDataAdapter fromUrl(String address, ZoneId zoneId, JrdsTreeViewTab treeViewTab, String filter) throws DataAdapterException {
        try {
            // Detect if URL protocol is present. If not, assume http.
            if (!uriSchemePattern.matcher(address).find()) {
                address = "http://" + address;
            }
            URL url = new URL(address.replaceAll("/$", ""));
            if (url.getHost().trim().isEmpty()) {
                throw new CannotInitializeDataAdapterException("Malformed URL: no host");
            }
            return new JrdsDataAdapter(url, zoneId, "utf-8", treeViewTab, filter);
        } catch (MalformedURLException e) {
            throw new CannotInitializeDataAdapterException("Malformed URL: " + e.getMessage(), e);
        }
    }

    //region [DataAdapter Members]

    @Override
    public TreeItem<TimeSeriesBinding<Double>> getBindingTree() throws DataAdapterException {
        Gson gson = new Gson();
        try {
            JsonJrdsTree t = gson.fromJson(getJsonTree(treeViewTab.getCommand(), treeViewTab.getArgument(), filter), JsonJrdsTree.class);
            Map<String, JsonJrdsItem> m = Arrays.stream(t.items).collect(Collectors.toMap(o -> o.id, (o -> o)));
            TreeItem<TimeSeriesBinding<Double>> tree = new TreeItem<>(bindingFactory.of("", getSourceName(), "/", this));
            for (JsonJrdsItem branch : Arrays.stream(t.items).filter(jsonJrdsItem -> JRDS_TREE.equals(jsonJrdsItem.type) || JRDS_FILTER.equals(jsonJrdsItem.type)).collect(Collectors.toList())) {
                attachNode(tree, branch.id, m);
            }
            return tree;
        } catch (JsonParseException e) {
            throw new DataAdapterException("An error occurred while parsing the json response to getBindingTree request", e);
        } catch (URISyntaxException e) {
            throw new SourceCommunicationException("Error building URI for request", e);
        }
    }

    @Override
    protected URI craftFetchUri(String path, Instant begin, Instant end) throws DataAdapterException {
{'fraction_non_alphanumeric': 0.09645817633760362, 'fraction_numerical': 0.0016578749058025622, 'mean_word_length': 3.568330464716007, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6990857', 'n_tokens_mistral': 3856, 'n_tokens_neox': 3801, 'n_words': 1023}