<commit_before>package de.tlongo.serveranalytics.services.logfileservice;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Service;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

import static de.tlongo.serveranalytics.services.logfileservice.JsonBuilder.jsonDocument;
import static spark.Spark.get;

/**
 * Created by tomas on 16.09.14.
 */

/**
 * Extracts relevant information from nginx log files and stores it in a database.
 */

/*
    Paths:
        - /entries/currentday --> Gets all log entries from today
        - /entries/{date}     --> Gets log entries of a specific date
 */
@Service
public class LogService {
    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);

    Logger logger = LoggerFactory.getLogger(LogService.class);
    Properties properties;

    List<LogEntry> latestProcessingCache = new ArrayList<>();

    Gson gson;

    @Autowired
    LogEntryRepository dao;

    public static void main(String[] args) throws IOException {
        ApplicationContext springContext = new ClassPathXmlApplicationContext("/springdataconfig.xml");

        LogService service = springContext.getBean(LogService.class);
        service.startService();
    }

    private void startService() throws IOException {
        logger.info("Starting LogService");
        logger.info("Init Spring Data");

        if (dao == null) {
            logger.error("Could not create repository for db access");
            throw new RuntimeException("Could not create repository for db access");
        }
        logger.info("Spring is up and running");

        initProperties();

        if (properties.getProperty("logfileservice.prettyjson").equals("true")) {
           gson = new GsonBuilder().setPrettyPrinting().create();
        } else {
            gson = new Gson();
        }

        initSpark();

        initScheduledPersistence();
    }

    private void initScheduledPersistence() {
        int scanIntervall = Integer.parseInt(properties.getProperty("logfileservice.scanintervall"));
        int startDelay = Integer.parseInt(properties.getProperty("logfileservice.startdelay"));
        logger.info("Service will scan directory every {} hours", scanIntervall);
        final ScheduledFuture<?> persistorHandle = scheduler.scheduleAtFixedRate(() -> persistLogEntries(), startDelay, scanIntervall, TimeUnit.HOURS);
    }

    private void initSpark() {
        get("/health", (request, response) -> {
            response.status(200);
            return gson.toJson(jsonDocument().
                                    property("message", "LogService Alive!").
                                    property("code", 200).
                                    create());
        });

        /**
         * Returns the entries that were processed by the last regular persistence turn.
         *
         * The entries have been cached so that fetching doesnt require a roundtrip to
         * the db.
         */
        get("/entries/latest", (request, response) -> {
            final JsonObject json = new JsonObject();
            final JsonArray entryArray = new JsonArray();
            latestProcessingCache.forEach(entry -> {
                entryArray.add(entryToJson(entry));
            });

            json.add("entries", entryArray);
            json.addProperty("count", latestProcessingCache.size());
            json.addProperty("status", 200);
<commit_msg>Restricted access to REST-API to localhost only.<commit_after>package de.tlongo.serveranalytics.services.logfileservice;

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.stereotype.Service;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;

import static de.tlongo.serveranalytics.services.logfileservice.JsonBuilder.jsonDocument;
import static spark.Spark.get;
import static spark.SparkBase.setIpAddress;

/**
 * Created by tomas on 16.09.14.
 */

/**
 * Extracts relevant information from nginx log files and stores it in a database.
 */

/*
    Paths:
        - /entries/currentday --> Gets all log entries from today
        - /entries/{date}     --> Gets log entries of a specific date
 */
@Service
public class LogService {
    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);

    Logger logger = LoggerFactory.getLogger(LogService.class);
    Properties properties;

    List<LogEntry> latestProcessingCache = new ArrayList<>();

    Gson gson;

    @Autowired
    LogEntryRepository dao;

    public static void main(String[] args) throws IOException {
        ApplicationContext springContext = new ClassPathXmlApplicationContext("/springdataconfig.xml");

        LogService service = springContext.getBean(LogService.class);
        service.startService();
    }

    private void startService() throws IOException {
        logger.info("Starting LogService");
        logger.info("Init Spring Data");

        if (dao == null) {
            logger.error("Could not create repository for db access");
            throw new RuntimeException("Could not create repository for db access");
        }
        logger.info("Spring is up and running");

        initProperties();

        if (properties.getProperty("logfileservice.prettyjson").equals("true")) {
           gson = new GsonBuilder().setPrettyPrinting().create();
        } else {
            gson = new Gson();
        }

        initSpark();

        initScheduledPersistence();
    }

    private void initScheduledPersistence() {
        int scanIntervall = Integer.parseInt(properties.getProperty("logfileservice.scanintervall"));
        int startDelay = Integer.parseInt(properties.getProperty("logfileservice.startdelay"));
        logger.info("Service will scan directory every {} hours", scanIntervall);
        final ScheduledFuture<?> persistorHandle = scheduler.scheduleAtFixedRate(() -> persistLogEntries(), startDelay, scanIntervall, TimeUnit.HOURS);
    }

    private void initSpark() {
        setIpAddress("127.0.0.1");
        
        get("/health", (request, response) -> {
            response.status(200);
            return gson.toJson(jsonDocument().
                                    property("message", "LogService Alive!").
                                    property("code", 200).
                                    create());
        });

        /**
         * Returns the entries that were processed by the last regular persistence turn.
         *
         * The entries have been cached so that fetching doesnt require a roundtrip to
         * the db.
         */
        get("/entries/latest", (request, response) -> {
            final JsonObject json = new JsonObject();
            final JsonArray entryArray = new JsonArray();
            latestProcessingCache.forEach(entry -> {
                entryArray.add(entryToJson(entry));
            });

            json.add("entries", entryArray);
            json.addProperty("count", latestProcessingCache.size());
            json.addProperty("status", 200);

{'fraction_non_alphanumeric': 0.10432276657060519, 'fraction_numerical': 0.00484149855907781, 'mean_word_length': 2.9598356914650843, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1467150', 'n_tokens_mistral': 2286, 'n_tokens_neox': 2202, 'n_words': 540}