<commit_before>package elasticparser;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.Random;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.json.simple.JSONObject;

/**
 * Trida pro zpracovani dat - index CSFD a MALL
 * Urceno pro studenty VŠE v Praze - Kompetencniho centra pro nestrukturovana data
 * 
 * @author <NAME>
 */
class Ingest {

    //kde mate spusten ES
    private String hostES = "es.vse.cz";
    private String portES = "9200";
    // upravte si cestu k datovym souborum
    private String pathToData = "/Users/ivanjelinek/Dropbox/The Analytical Company/data";

    public Ingest() {
        //vytvori indexy a priradi jim analyzery
        createIndex("csfd");
        createIndex("mall");

        //vytvori mapovani pro defaultni analyzery
        prepareMapping("csfd", "negative");
        prepareMapping("csfd", "neutral");
        prepareMapping("csfd", "positive");
        prepareMapping("mall", "negative");
        prepareMapping("mall", "neutral");
        prepareMapping("mall", "positive");

        //zpracuje data
        loadFile(pathToData + "/csfd/negative.txt", "csfd", "negative");
        loadFile(pathToData + "/csfd/positive.txt", "csfd", "positive");
        loadFile(pathToData + "/csfd/neutral.txt", "csfd", "neutral");

        loadFile(pathToData + "/mallcz/negative.txt", "mall", "negative");
        loadFile(pathToData + "/mallcz/positive.txt", "mall", "positive");
        loadFile(pathToData + "/mallcz/neutral.txt", "mall", "neutral");
    }

    /**
     * Metoda nacte soubory a radek po radku je rozparsuje do zadaneho indexu a typu
     * 
     * @param soubor cesta k souboru
     * @param index index v ES
     * @param type typ v ES
     */
    private void loadFile(String soubor, String index, String type) {
        try {
            BufferedReader br = new BufferedReader(new FileReader(soubor));
            String line = br.readLine();
            int i = 1;
            
            Random randomGenerator = new Random();
            
            while (line != null) {
               // line = line.replaceAll("\\\\", "");
               // line = "{ \"body\" : \"" + line.replaceAll("\"", "") + "\"}";
                JSONObject lineJS = new JSONObject();
                lineJS.put("body", line);
                lineJS.put("user", randomGenerator.nextInt(5000));
                if (i <= 1000){
                    lineJS.put("segment", "A");
                    lineJS.put("country", "RU");
                }
                if (i <= 3000 && i > 1000){
                    lineJS.put("segment", "B");
                    lineJS.put("country", "FR");
                } else {
                    lineJS.put("segment", "C");
                    lineJS.put("country", "CZ");
                }
                double minLat = -90.00;
    double maxLat = 90.00;      
    double latitude = minLat + (double)(Math.random() * ((maxLat - minLat) + 1));
    double minLon = 0.00;
    double maxLon = 180.00;     
    double longitude = minLon + (double)(Math.random() * ((maxLon - minLon) + 1));
    DecimalFormat df = new DecimalFormat("#.##");
                lineJS.put("location", df.format(longitude).replace(",", ".") +","+df.format(latitude).replace(",", "."));
                URL url = new URL("http://" + hostES + ":" + portES + "/" + index + "/" + type + "/" + i);
                System.out.println(sendRQ(url, "PUT", lineJS.toString()));
                line = br.readLine();
                i++;

            }

        } catch (FileNotFoundException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }

    }

    /**
     * Metoda vytvori index v ES
     * 
     * @param index nazev indexu, ktery ma byt vytvoren
     */
    private void createIndex(String index) {
        try {
            String analyzer = "{\"settings\": {\"analysis\": {\"filter\": {\"czech_stop\": {\"type\":       \"stop\",\"stopwords\":  \"_czech_\"},\"czech_keywords\": {\"type\":       \"keyword_marker\",\"keywords\":   [\"x\"]}, \"czech_stemmer\": { \"type\":       \"stemmer\", \"language\":   \"czech\"}},\"analyzer\": {\"czech\": {\"tokenizer\":  \"standard\",\"filter\": [ \"lowercase\",\"czech_stop\", \"czech_keywords\", \"czech_stemmer\"]}}}}}";
            URL url = new URL("http://" + hostES + ":" + portES + "/" + index + "/");
            System.out.println(sendRQ(url, "PUT", analyzer));
        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        } 
    }

    /**
     * Metoda vytvori mapovani pro analyzer k indexu a typu, ktery je specifikovan
     * 
     * @param index mapovani, ktere ma byt vytvoren
     * @param typ k namapovani
     */
    private void prepareMapping(String index, String typ) {
        //String mappingBody = "{\"properties\": {\"body\": {\"type\":\"string\",\"analyzer\": \"czech\"}}}";
        JSONObject child = new JSONObject();
        child.put("type", "string");
        child.put("analyzer", "czech");
        JSONObject czech = new JSONObject();
        czech.put("body", child);
        JSONObject geotype = new JSONObject();
        geotype.put("type", "geo_point");
        czech.put("location", geotype);
        JSONObject mappingBody = new JSONObject();
        mappingBody.put("properties", czech);
        //System.out.println(mappingBody);
        try {
            System.out.println(sendRQ(new URL("http://" + hostES + ":" + portES+"/" + index + "/_mapping/" + typ), "POST", mappingBody.toString()));
        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * Metoda odesle RQ skrze REST API
     * 
     * @param hostES IP adresa ES
     * @param portES port ES
     * @param index index, kam se ma zprava poslat
     * @param typ typ kam se zprava posle
     * @param method GET, POST, PUT, DELETE
     * @param message JSON zprava
     * @return String odpoved ES nebo null pri chybe
     */
    private String sendRQ(String hostES, String portES, String index, String typ, String method, String message) {

        String urlString = "";
        try {
            if (typ == null) {
                urlString = "http://" + hostES + ":" + portES + "/" + index + "/";
            } else {
                urlString = "http://" + hostES + ":" + portES + "/" + index + "/" + typ;
            }
            URL url = new URL(urlString);
            return sendRQ(url, method, message);

        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        } 
        return null;
    }

    /**
     * Pretizena verze metody pro odelsani pozadavku
     * 
     * @param url URL ES - cely string
     * @param method GET, POST, PUT, DELETE
     * @return String ES odpoved
     */
    private String sendRQ(URL url, String method) {
        return sendRQ(url, method, "");
    }

    /**
     * Pretizena verze metody, ktera skutecne odesle REST RQ
     * 
     * @param url URL string pro pripojeni k ES
     * @param method GET POST PUT DELETE
     * @param message JSON zprava
     * @return String odpoved ES null pri chybe
     */
    private String sendRQ(URL url, String method, String message) {
        try {
            HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
            httpCon.setDoOutput(true);
            httpCon.setRequestMethod(method);

            if (!method.equals("GET")) {
                OutputStreamWriter out = new OutputStreamWriter(
                        httpCon.getOutputStream());
                out.write(message);
                out.close();
            }

            BufferedReader in = new BufferedReader(
                    new InputStreamReader(httpCon.getInputStream()));
            String inputLine;
            StringBuffer response = new StringBuffer();

            while ((inputLine = in.readLine()) != null) {
                response.append(inputLine);
            }
            in.close();

            //print result
            return response.toString();<commit_msg>Geo dokončeno, přidáno časové pole?<commit_after>package elasticparser;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Random;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;

/**
 * Trida pro zpracovani dat - index CSFD a MALL Urceno pro studenty VŠE v Praze
 * - Kompetencniho centra pro nestrukturovana data
 *
 * @author <NAME>
 */
class Ingest {

    //kde mate spusten ES
    private String hostES = "es.vse.cz";
    private String portES = "9200";
    // upravte si cestu k datovym souborum
    private String pathToData = "C:\\Users\\jelineiv\\Dropbox\\The Analytical Company\\data";

    public Ingest() {
        //vytvori indexy a priradi jim analyzery
        createIndex("csfd");
        createIndex("mall");

        //vytvori mapovani pro defaultni analyzery
        prepareMapping("csfd", "negative");
        prepareMapping("csfd", "neutral");
        prepareMapping("csfd", "positive");
        prepareMapping("mall", "negative");
        prepareMapping("mall", "neutral");
        prepareMapping("mall", "positive");

        //zpracuje data
        loadFile(pathToData + "/csfd/negative.txt", "csfd", "negative");
        loadFile(pathToData + "/csfd/positive.txt", "csfd", "positive");
        loadFile(pathToData + "/csfd/neutral.txt", "csfd", "neutral");

        loadFile(pathToData + "/mallcz/negative.txt", "mall", "negative");
        loadFile(pathToData + "/mallcz/positive.txt", "mall", "positive");
        loadFile(pathToData + "/mallcz/neutral.txt", "mall", "neutral");
    }

    /**
     * Metoda nacte soubory a radek po radku je rozparsuje do zadaneho indexu a
     * typu
     *
     * @param soubor cesta k souboru
     * @param index index v ES
     * @param type typ v ES
     */
    private void loadFile(String soubor, String index, String type) {
        try {
            BufferedReader br = new BufferedReader(new FileReader(soubor));
            String line = br.readLine();
            int i = 1;

            Random randomGenerator = new Random();

            while (line != null) {
               // line = line.replaceAll("\\\\", "");
                // line = "{ \"body\" : \"" + line.replaceAll("\"", "") + "\"}";
                JSONObject lineJS = new JSONObject();
                lineJS.put("body", line);
                lineJS.put("user", randomGenerator.nextInt(5000));
                if (i <= 1000) {
                    lineJS.put("segment", "A");
                    lineJS.put("country", "RU");
                }
                if (i <= 3000 && i > 1000) {
                    lineJS.put("segment", "B");
                    lineJS.put("country", "FR");
                } else {
                    lineJS.put("segment", "C");
                    lineJS.put("country", "CZ");
                }
                double minLat = -90.00;
                double maxLat = 90.00;
                double latitude = minLat + (double) (Math.random() * ((maxLat - minLat) + 1));
                double minLon = -180.00;
                double maxLon = 180.00;
                double longitude = minLon + (double) (Math.random() * ((maxLon - minLon) + 1));
                DecimalFormat df = new DecimalFormat("#.##");
                JSONArray lonlatArr = new JSONArray();
                lonlatArr.add(Double.parseDouble(df.format(longitude).replace(",", ".")));
                lonlatArr.add(Double.parseDouble(df.format(latitude).replace(",", ".")));
                //lineJS.put("location", df.format(longitude).replace(",", ".") +","+df.format(latitude).replace(",", "."));
                lineJS.put("location", lonlatArr);
                lineJS.put("timestamp", new SimpleDateFormat("HH24:mm:ss dd.mm.yyyy")+"");
                System.out.println(lineJS.toString());
                URL url = new URL("http://" + hostES + ":" + portES + "/" + index + "/" + type + "/" + i);
                System.out.println(sendRQ(url, "PUT", lineJS.toString()));
                line = br.readLine();
                i++;

            }

        } catch (FileNotFoundException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        } catch (IOException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }

    }

    /**
     * Metoda vytvori index v ES
     *
     * @param index nazev indexu, ktery ma byt vytvoren
     */
    private void createIndex(String index) {
        try {
            String analyzer = "{\"settings\": {\"analysis\": {\"filter\": {\"czech_stop\": {\"type\":       \"stop\",\"stopwords\":  \"_czech_\"},\"czech_keywords\": {\"type\":       \"keyword_marker\",\"keywords\":   [\"x\"]}, \"czech_stemmer\": { \"type\":       \"stemmer\", \"language\":   \"czech\"}},\"analyzer\": {\"czech\": {\"tokenizer\":  \"standard\",\"filter\": [ \"lowercase\",\"czech_stop\", \"czech_keywords\", \"czech_stemmer\"]}}}}}";
            URL url = new URL("http://" + hostES + ":" + portES + "/" + index + "/");
            System.out.println(sendRQ(url, "PUT", analyzer));
        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * Metoda vytvori mapovani pro analyzer k indexu a typu, ktery je
     * specifikovan
     *
     * @param index mapovani, ktere ma byt vytvoren
     * @param typ k namapovani
     */
    private void prepareMapping(String index, String typ) {
        //String mappingBody = "{\"properties\": {\"body\": {\"type\":\"string\",\"analyzer\": \"czech\"}}}";
        JSONObject child = new JSONObject();
        child.put("type", "string");
        child.put("analyzer", "czech");
        JSONObject czech = new JSONObject();
        czech.put("body", child);
        JSONObject geotype = new JSONObject();
        geotype.put("type", "geo_point");
        //JSONObject validate = new JSONObject();
        geotype.put("validate", true);
        geotype.put("lat_lon", true);
        //validate.put("validate", true);
        czech.put("location", geotype);
        JSONObject timestamp = new JSONObject();
        timestamp.put("type", "date");
        czech.put("timestamp", timestamp);
        JSONObject mappingBody = new JSONObject();
        mappingBody.put("properties", czech);
        System.out.println(mappingBody);
        try {
            System.out.println(sendRQ(new URL("http://" + hostES + ":" + portES + "/" + index + "/_mapping/" + typ), "POST", mappingBody.toString()));
        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }
    }

    /**
     * Metoda odesle RQ skrze REST API
     *
     * @param hostES IP adresa ES
     * @param portES port ES
     * @param index index, kam se ma zprava poslat
     * @param typ typ kam se zprava posle
     * @param method GET, POST, PUT, DELETE
     * @param message JSON zprava
     * @return String odpoved ES nebo null pri chybe
     */
    private String sendRQ(String hostES, String portES, String index, String typ, String method, String message) {

        String urlString = "";
        try {
            if (typ == null) {
                urlString = "http://" + hostES + ":" + portES + "/" + index + "/";
            } else {
                urlString = "http://" + hostES + ":" + portES + "/" + index + "/" + typ;
            }
            URL url = new URL(urlString);
            return sendRQ(url, method, message);

        } catch (MalformedURLException ex) {
            Logger.getLogger(Ingest.class.getName()).log(Level.SEVERE, null, ex);
        }
        return null;
    }

    /**
     * Pretizena verze metody pro odelsani pozadavku
     *
     * @param url URL ES - cely string
     * @param method GET, POST, PUT, DELETE
     * @return String ES odpoved
     */
    private String sendRQ(URL url, String method) {
        return sendRQ(url, method, "");
    }

    /**
     * Pretizena verze metody, ktera skutecne odesle REST RQ
     *
     * @param url URL string pro pripojeni k ES
     * @param method GET POST PUT DELETE
     * @param message JSON zprava
     * @return String odpoved ES null pri chybe
     */
    private String sendRQ(URL url, String method, String message) {
        try {
            HttpURLConnection httpCon = (HttpURLConnection) url.openConnection();
            httpCon.setDoOutput(true);
            httpCon.setRequestMethod(method);

            if (!method.equals("GET")) {
                OutputStreamWriter out = new OutputStreamWriter(
                        httpCon.getOutputStream());
                out.write(message);
                out.close();
            }

            BufferedReader in = new BufferedReader(
                    new InputStreamReader(httpCon.getInputStream()));
            String inputLine;
            StringBuffer response = new StringBuffer();

            while ((inputLine = in.readLine()) != null) {
                response.append(inputLine);
            }
            in.close();

            //print result
            return response.toString();
{'fraction_non_alphanumeric': 0.14961587354225392, 'fraction_numerical': 0.004532139501464655, 'mean_word_length': 2.073551894003737, 'pattern_counts': {'":': 50, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5903489', 'n_tokens_mistral': 5529, 'n_tokens_neox': 5305, 'n_words': 1288}