<commit_before>package org.helioviewer.jhv.plugins.swek.sources.hek;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.helioviewer.jhv.base.JSONUtils;
import org.helioviewer.jhv.base.Pair;
import org.helioviewer.jhv.base.conversion.FlareConversion;
import org.helioviewer.jhv.base.logging.Log;
import org.helioviewer.jhv.base.time.TimeUtils;
import org.helioviewer.jhv.data.datatype.event.JHVEventType;
import org.helioviewer.jhv.data.datatype.event.SWEKDownloader;
import org.helioviewer.jhv.data.datatype.event.SWEKEventType;
import org.helioviewer.jhv.data.datatype.event.SWEKParam;
import org.helioviewer.jhv.database.EventDatabase;
import org.helioviewer.jhv.database.JHVDatabaseParam;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

@SuppressWarnings({ "rawtypes", "unchecked" })
public class HEKDownloader extends SWEKDownloader {
    private static final String _baseURL = "http://www.lmsal.com/hek/her?";

    private void patch_event(JSONObject result, JHVEventType type) {
        boolean c1 = type.getEventType().getEventName().equals("Flare");
        boolean c2 = type.getSupplier().getSupplierName().equals("SWPC");
        boolean c = c1 && c2;
        if (c && result.has("fl_goescls")) {
            result.put("fl_val", FlareConversion.getFloatValue(result.getString("fl_goescls")));
        }
    }

    @Override
    protected boolean parseEvents(JSONObject eventJSON, JHVEventType type) {
        JSONArray results = eventJSON.getJSONArray("result");
        ArrayList<EventDatabase.Event2Db> event2db_list = new ArrayList<EventDatabase.Event2Db>();

        for (int i = 0; i < results.length(); i++) {
            JSONObject result = results.getJSONObject(i);
            patch_event(result, type);
            String uid;
            long start;
            long end;
            long archiv;
            ArrayList<JHVDatabaseParam> paramList = new ArrayList<JHVDatabaseParam>();
            try {
                start = TimeUtils.utcDateFormat.parse(result.getString("event_starttime")).getTime();
                end = TimeUtils.utcDateFormat.parse(result.getString("event_endtime")).getTime();
                archiv = TimeUtils.utcDateFormat.parse(result.getString("kb_archivdate")).getTime();
                uid = result.getString("kb_archivid");

                HashMap<String, String> dbFields = type.getEventType().getAllDatabaseFields();
                for (Map.Entry<String, String> entry : dbFields.entrySet()) {
                    String dbType = entry.getValue();
                    String fieldName = entry.getKey();
                    String lfieldName = fieldName.toLowerCase(Locale.ENGLISH);
                    if (!result.isNull(lfieldName)) {
                        if (dbType.equals(JHVDatabaseParam.DBINTTYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBINTTYPE, result.getInt(lfieldName), fieldName));
                        } else if (dbType.equals(JHVDatabaseParam.DBSTRINGTYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBSTRINGTYPE, result.getString(lfieldName), fieldName));
                        } else if (dbType.equals(JHVDatabaseParam.DBDOUBLETYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBDOUBLETYPE, result.getDouble(lfieldName), fieldName));
                        }
                    }
                }
            } catch (JSONException e) {
                return false;
            } catch (ParseException e) {
                return false;
            }

            byte[] compressed;
            try {
                compressed = JSONUtils.compressJSON(result);
            } catch (IOException e) {
                Log.error("compression error");
                return false;
            }

            event2db_list.add(new EventDatabase.Event2Db(compressed, start, end, archiv, uid, paramList));
        }

        int[] ids = EventDatabase.dump_event2db(event2db_list, type);

        return true;
    }

    @Override
    protected boolean parseAssociations(JSONObject eventJSON) {
        JSONArray associations = eventJSON.getJSONArray("association");
        int len = associations.length();
        Pair<String, String>[] assocs = new Pair[len];
        for (int i = 0; i < len; i++) {
            JSONObject asobj = associations.getJSONObject(i);
            assocs[i] = new Pair<String, String>(asobj.getString("first_ivorn"), asobj.getString("second_ivorn"));<commit_msg>keep only original data in compressed son<commit_after>package org.helioviewer.jhv.plugins.swek.sources.hek;

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;

import org.helioviewer.jhv.base.JSONUtils;
import org.helioviewer.jhv.base.Pair;
import org.helioviewer.jhv.base.conversion.FlareConversion;
import org.helioviewer.jhv.base.logging.Log;
import org.helioviewer.jhv.base.time.TimeUtils;
import org.helioviewer.jhv.data.datatype.event.JHVEventType;
import org.helioviewer.jhv.data.datatype.event.SWEKDownloader;
import org.helioviewer.jhv.data.datatype.event.SWEKEventType;
import org.helioviewer.jhv.data.datatype.event.SWEKParam;
import org.helioviewer.jhv.database.EventDatabase;
import org.helioviewer.jhv.database.JHVDatabaseParam;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

@SuppressWarnings({ "rawtypes", "unchecked" })
public class HEKDownloader extends SWEKDownloader {
    private static final String _baseURL = "http://www.lmsal.com/hek/her?";

    private void patch_event(JSONObject result, JHVEventType type) {
        boolean c1 = type.getEventType().getEventName().equals("Flare");
        boolean c2 = type.getSupplier().getSupplierName().equals("SWPC");
        boolean c = c1 && c2;
        if (c && result.has("fl_goescls")) {
            result.put("fl_val", FlareConversion.getFloatValue(result.getString("fl_goescls")));
        }
    }

    @Override
    protected boolean parseEvents(JSONObject eventJSON, JHVEventType type) {
        JSONArray results = eventJSON.getJSONArray("result");
        ArrayList<EventDatabase.Event2Db> event2db_list = new ArrayList<EventDatabase.Event2Db>();

        for (int i = 0; i < results.length(); i++) {
            JSONObject result = results.getJSONObject(i);
            byte[] compressed;
            try {
                compressed = JSONUtils.compressJSON(result);
            } catch (IOException e) {
                Log.error("compression error");
                return false;
            }
            patch_event(result, type);
            String uid;
            long start;
            long end;
            long archiv;
            ArrayList<JHVDatabaseParam> paramList = new ArrayList<JHVDatabaseParam>();
            try {
                start = TimeUtils.utcDateFormat.parse(result.getString("event_starttime")).getTime();
                end = TimeUtils.utcDateFormat.parse(result.getString("event_endtime")).getTime();
                archiv = TimeUtils.utcDateFormat.parse(result.getString("kb_archivdate")).getTime();
                uid = result.getString("kb_archivid");

                HashMap<String, String> dbFields = type.getEventType().getAllDatabaseFields();
                for (Map.Entry<String, String> entry : dbFields.entrySet()) {
                    String dbType = entry.getValue();
                    String fieldName = entry.getKey();
                    String lfieldName = fieldName.toLowerCase(Locale.ENGLISH);
                    if (!result.isNull(lfieldName)) {
                        if (dbType.equals(JHVDatabaseParam.DBINTTYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBINTTYPE, result.getInt(lfieldName), fieldName));
                        } else if (dbType.equals(JHVDatabaseParam.DBSTRINGTYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBSTRINGTYPE, result.getString(lfieldName), fieldName));
                        } else if (dbType.equals(JHVDatabaseParam.DBDOUBLETYPE)) {
                            paramList.add(new JHVDatabaseParam(JHVDatabaseParam.DBDOUBLETYPE, result.getDouble(lfieldName), fieldName));
                        }
                    }
                }
            } catch (JSONException e) {
                return false;
            } catch (ParseException e) {
                return false;
            }

            event2db_list.add(new EventDatabase.Event2Db(compressed, start, end, archiv, uid, paramList));
        }

        EventDatabase.dump_event2db(event2db_list, type);

        return true;
    }

    @Override
    protected boolean parseAssociations(JSONObject eventJSON) {
        JSONArray associations = eventJSON.getJSONArray("association");
        int len = associations.length();
        Pair<String, String>[] assocs = new Pair[len];
        for (int i = 0; i < len; i++) {
            JSONObject asobj = associations.getJSONObject(i);
            assocs[i] = new Pair<String, String>(asobj.getString("first_ivorn"), asobj.getString("second_ivorn"));
{'fraction_non_alphanumeric': 0.10933725449007457, 'fraction_numerical': 0.0027308055876483564, 'mean_word_length': 2.6191562143671607, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6350605', 'n_tokens_mistral': 2831, 'n_tokens_neox': 2770, 'n_words': 475}