<commit_before>/**
 * Copyright 2016 52°North Initiative for Geospatial Open Source
 * Software GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.envirocar.aggregation;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.envirocar.analyse.AggregationAlgorithm;
import org.envirocar.analyse.util.PointViaJsonMapIterator;
import org.envirocar.analyse.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Singleton;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;

@Singleton
public class ReceiveTracksServlet extends HttpServlet {
    
    /**
     *
     */
    private static final long serialVersionUID = 4589872023160154399L;
    private static final Logger logger = LoggerFactory
            .getLogger(ReceiveTracksServlet.class);
    private static final String PRODUCERS_FILE = "/allowed_producers.cfg";
    private ExecutorService executor;
    
    private Set<String> allowedProducers;
    private List<AggregationAlgorithm> algorithms = new ArrayList<>();
    
    public ReceiveTracksServlet() throws IOException {
        this.executor = Executors.newSingleThreadExecutor();
        logger.info("Servlet created");
    }
    
    @Override
    public void init() throws ServletException {
        super.init();
        try {
            instantiateAlgorithmInstances();
            this.allowedProducers = FileUtil
                    .readConfigFilePerLine(PRODUCERS_FILE);
        } catch (IOException e) {
            throw new ServletException(e);
        }
        
    }
    
    @Override
    public void destroy() {
        super.destroy();
        this.executor.shutdown();
        
        try {
            for (AggregationAlgorithm algorithm : algorithms) {
                algorithm.shutdown();
            }
        } catch (IOException e) {
            logger.warn(e.getMessage(), e);
        }
    }
    
    
    @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        final String contentType = req.getHeader("Content-Type");
        
        final Path tempFile = Util.writeToTempFile(req.getInputStream());
        
        if (!(contentType != null && contentType.startsWith("application/json"))) {
            throw new IllegalArgumentException("Invalid ContentType");
        }
        
        if (verifyRemoteHost(req.getRemoteHost())) {
            this.executor.submit(new Runnable() {
                public void run() {
                    for (AggregationAlgorithm algorithm : algorithms) {
                        PointViaJsonMapIterator it;
                        try {
                            InputStream stream = Files.newInputStream(tempFile);
                            final Map<?, ?> json = Utils.parseJsonStream(stream);
                            it = new PointViaJsonMapIterator(json);<commit_msg>remove temp files after processing<commit_after>/**
 * Copyright 2016 52°North Initiative for Geospatial Open Source
 * Software GmbH
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.envirocar.aggregation;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.envirocar.analyse.AggregationAlgorithm;
import org.envirocar.analyse.util.PointViaJsonMapIterator;
import org.envirocar.analyse.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Module;
import com.google.inject.Singleton;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import java.nio.channels.Channels;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;

@Singleton
public class ReceiveTracksServlet extends HttpServlet {
    
    /**
     *
     */
    private static final long serialVersionUID = 4589872023160154399L;
    private static final Logger logger = LoggerFactory
            .getLogger(ReceiveTracksServlet.class);
    private static final String PRODUCERS_FILE = "/allowed_producers.cfg";
    private ExecutorService executor;
    
    private Set<String> allowedProducers;
    private List<AggregationAlgorithm> algorithms = new ArrayList<>();
    
    public ReceiveTracksServlet() throws IOException {
        this.executor = Executors.newSingleThreadExecutor();
        logger.info("Servlet created");
    }
    
    @Override
    public void init() throws ServletException {
        super.init();
        try {
            instantiateAlgorithmInstances();
            this.allowedProducers = FileUtil
                    .readConfigFilePerLine(PRODUCERS_FILE);
        } catch (IOException e) {
            throw new ServletException(e);
        }
        
    }
    
    @Override
    public void destroy() {
        super.destroy();
        this.executor.shutdown();
        
        try {
            for (AggregationAlgorithm algorithm : algorithms) {
                algorithm.shutdown();
            }
        } catch (IOException e) {
            logger.warn(e.getMessage(), e);
        }
    }
    
    
    @Override
    protected void doPost(HttpServletRequest req, HttpServletResponse resp)
            throws ServletException, IOException {
        final String contentType = req.getHeader("Content-Type");
        
        final Path tempFile = Util.writeToTempFile(req.getInputStream());
        
        if (!(contentType != null && contentType.startsWith("application/json"))) {
            throw new IllegalArgumentException("Invalid ContentType");
        }
        
        if (verifyRemoteHost(req.getRemoteHost())) {
            this.executor.submit(new Runnable() {
                public void run() {
                    for (AggregationAlgorithm algorithm : algorithms) {
                        PointViaJsonMapIterator it;
                        try {
                            InputStream stream = Files.newInputStream(tempFile);
                            final Map<?, ?> json = Utils.parseJsonStream(stream);
                            Files.delete(tempFile);
                            it = new PointViaJsonMapIterator(json);
{'fraction_non_alphanumeric': 0.0865488640461594, 'fraction_numerical': 0.007452818848419281, 'mean_word_length': 2.7460603331832507, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1465451', 'n_tokens_mistral': 2185, 'n_tokens_neox': 2134, 'n_words': 560}