<commit_before>package fi.nls.oskari.statistics.eurostat;

import fi.nls.oskari.control.statistics.plugins.*;
import fi.nls.oskari.util.IOHelper;
import fi.nls.oskari.util.JSONHelper;
import org.json.JSONArray;
import org.json.JSONObject;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.HashMap;
import java.util.Map;

public class EurostatStatisticalIndicatorLayer implements StatisticalIndicatorLayer {

    private long id;
    private String indicatorId;
    private String baseUrl;
    private String regionKey;

    public EurostatStatisticalIndicatorLayer(long id, String indicatorId, String baseUrl, String regionKey) {
        this.id = id;
        this.indicatorId = indicatorId;
        this.baseUrl = baseUrl;
        this.regionKey = regionKey;
    }

    @Override
    public long getOskariLayerId() {
        return id;
    }

    @Override
    public IndicatorValueType getIndicatorValueType() {
        return null;
    }

    /*
    "query": [
   {
     "code": "Alue",
     "selection": {
       "filter": "item",
       "values": [
         "0910000000",
         "0911000000",
         "0911101000"
       ]
     }
   },
   {
     "code": "Käyttötarkoitus",
     "selection": {
       "filter": "item",
       "values": [
         "all",
         "01",
         "02"
       ]
     }
   },
   {
     "code": "Toimenpide",
     "selection": {
       "filter": "item",
       "values": [
         "all",
         "1"
       ]
     }
   },
   {
     "code": "Yksikkö",
     "selection": {
       "filter": "item",
       "values": [
         "1",
         "2"
       ]
     }
   },
   {
     "code": "Vuosi",
     "selection": {
       "filter": "item",
       "values": [
         "0",
         "1",
         "2"
       ]
     }
   }
 ],
 "response": {
   "format": "csv"
 }
}
     */
    @Override
    public Map<String, IndicatorValue> getIndicatorValues(StatisticalIndicatorSelectors selectors) {
        Map<String, IndicatorValue> values = new HashMap<>();
        String url = baseUrl + "/" + indicatorId;
        JSONArray query = new JSONArray();
        JSONObject payload = JSONHelper.createJSONObject("query", query);
        for (StatisticalIndicatorSelector selector : selectors.getSelectors()) {
            if (regionKey.equalsIgnoreCase(selector.getId())) {
                // skip Alue
                continue;
            }
            JSONObject param = new JSONObject();
            JSONHelper.putValue(param, "code", selector.getId());
            JSONObject selection = new JSONObject();
            JSONHelper.putValue(selection, "filter", "item");
            JSONArray paramValues = new JSONArray();
            paramValues.put(selector.getValue());
            JSONHelper.putValue(selection, "values", paramValues);

            JSONHelper.putValue(param, "selection", selection);
            query.put(param);
        }
        JSONHelper.putValue(payload, "response", JSONHelper.createJSONObject("format", "json-stat"));

        try {
            final HttpURLConnection con = IOHelper.getConnection(url);
            IOHelper.writeHeader(con, IOHelper.HEADER_CONTENTTYPE, IOHelper.CONTENT_TYPE_JSON + ";  charset=utf-8");
            IOHelper.writeToConnection(con, payload.toString().getBytes("UTF-8"));
            final String data = IOHelper.readString(con);
            JSONObject json = JSONHelper.createJSONObject(data);
            //dataset.dimension.Alue.category.index -> key==region id & value == index pointer to dataset.value
            JSONObject stats = json.optJSONObject("dataset").optJSONObject("dimension").optJSONObject(regionKey).optJSONObject("category").optJSONObject("index");
            JSONArray responseValues = json.optJSONObject("dataset").optJSONArray("value");
            JSONArray names = stats.names();
            for (int i = 0; i < names.length(); ++i) {
                String region = names.optString(i);
                Double val = responseValues.optDouble(stats.optInt(region));
                if (val.isNaN()) {
                    continue;
                }
                IndicatorValue indicatorValue = new IndicatorValueFloat(val);
                values.put(region, indicatorValue);
            }
        } catch (IOException e) {
            throw new APIException("Couldn't get data from service/parsing failed", e);
        }

        return values;
    }
}<commit_msg>Modified data-fetching for eurostat<commit_after>package fi.nls.oskari.statistics.eurostat;

import fi.nls.oskari.control.statistics.plugins.*;
import fi.nls.oskari.util.IOHelper;
import fi.nls.oskari.util.JSONHelper;
import org.json.JSONArray;
import org.json.JSONObject;

import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.HashMap;
import java.util.Map;

public class EurostatStatisticalIndicatorLayer implements StatisticalIndicatorLayer {

    private long id;
    private String indicatorId;
    private String baseUrl;
    private String regionKey;

    public EurostatStatisticalIndicatorLayer(long id, String indicatorId, String baseUrl, String regionKey) {
        this.id = id;
        this.indicatorId = indicatorId;
        this.baseUrl = baseUrl;
        this.regionKey = regionKey;
    }

    @Override
    public long getOskariLayerId() {
        return id;
    }

    @Override
    public IndicatorValueType getIndicatorValueType() {
        return null;
    }

    /*
    "query": [
   {
     "code": "Alue",
     "selection": {
       "filter": "item",
       "values": [
         "0910000000",
         "0911000000",
         "0911101000"
       ]
     }
   },
   {
     "code": "Käyttötarkoitus",
     "selection": {
       "filter": "item",
       "values": [
         "all",
         "01",
         "02"
       ]
     }
   },
   {
     "code": "Toimenpide",
     "selection": {
       "filter": "item",
       "values": [
         "all",
         "1"
       ]
     }
   },
   {
     "code": "Yksikkö",
     "selection": {
       "filter": "item",
       "values": [
         "1",
         "2"
       ]
     }
   },
   {
     "code": "Vuosi",
     "selection": {
       "filter": "item",
       "values": [
         "0",
         "1",
         "2"
       ]
     }
   }
 ],
 "response": {
   "format": "csv"
 }
}
     */
    @Override
    public Map<String, IndicatorValue> getIndicatorValues(StatisticalIndicatorSelectors selectors) {
        Map<String, String> params = new HashMap<>();
        for (StatisticalIndicatorSelector selector : selectors.getSelectors()) {
            if (regionKey.equalsIgnoreCase(selector.getId())) {
                // skip Alue
                continue;
            }
            params.put(selector.getId(), selector.getValue());
        }
        String url = IOHelper.constructUrl(baseUrl +"wdds/rest/data/v2.1/json/en/" + indicatorId, params);


        Map<String, IndicatorValue> values = new HashMap<>();
        try {
            final String data = IOHelper.getURL(url);
            // TODO: parsing
            JSONObject json = JSONHelper.createJSONObject(data);
            JSONObject stats = json.optJSONObject("dataset").optJSONObject("dimension").optJSONObject(regionKey).optJSONObject("category").optJSONObject("index");
            JSONArray responseValues = json.optJSONObject("dataset").optJSONArray("value");
            JSONArray names = stats.names();
            for (int i = 0; i < names.length(); ++i) {
                String region = names.optString(i);
                Double val = responseValues.optDouble(stats.optInt(region));
                if (val.isNaN()) {
                    continue;
                }
                IndicatorValue indicatorValue = new IndicatorValueFloat(val);
                values.put(region, indicatorValue);
            }
        } catch (IOException e) {
            throw new APIException("Couldn't get data from service/parsing failed", e);
        }

        return values;
    }
}
{'fraction_non_alphanumeric': 0.1259287243420224, 'fraction_numerical': 0.010829870293413927, 'mean_word_length': 2.069965210668728, 'pattern_counts': {'":': 46, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6895187', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2264, 'n_words': 450}