<commit_before>package com.imcode.imcms.servlet;


import com.fasterxml.jackson.databind.ObjectMapper;
import imcode.server.Imcms;
import imcode.util.Utility;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.*;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.apache.http.ssl.SSLContextBuilder;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Stream;


import static java.util.Optional.ofNullable;
import static org.apache.commons.lang3.StringUtils.defaultString;

public class InternalError extends HttpServlet {

    private final String ERROR_500_VIEW_URL = "/imcms/500.jsp";

    private final String DEFAULT_RESPONSE = "N/A";

    private final static Logger LOGGER = Logger.getLogger(InternalError.class);

    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

        String attributeName = "javax.servlet.error.exception";
        Object exception = request.getAttribute(attributeName);
        request.setAttribute(attributeName, null);

        String userId = Utility.getLoggedOnUser(request).getId() + "";

        try {
            String responseJson = sendError((Throwable) exception, request, userId);
            String errorId = logError(responseJson, userId);
            request.setAttribute("errorId", errorId);
        } catch (Exception e) {
            request.setAttribute("errorId", 0);
        }

        request.getRequestDispatcher(ERROR_500_VIEW_URL).forward(request, response);
    }

    private String sendError(Throwable exception, HttpServletRequest request, String userId) throws Exception {
        Properties serverProperties = Imcms.getServerProperties();

        Map<String, String> exceptionInfo = parse(exception);

        Map<String, String> headersInfo = parse(request);

        String errorLoggerUrl = ofNullable(serverProperties.getProperty("ErrorLoggerUrl"))
                                    .map(url -> !url.isEmpty() ? url : Imcms.ERROR_LOGGER_URL)
                                    .orElse(Imcms.ERROR_LOGGER_URL);

        HttpClient httpClient = createHttpClient();

        List<NameValuePair> params = Form.form()

                .add("hash", exceptionInfo.get("hash"))
                .add("message", exceptionInfo.get("message"))
                .add("cause", exceptionInfo.get("cause"))
                .add("stack-trace", exceptionInfo.get("stackTrace"))

                .add("error-url", defaultString(headersInfo.get("errorUrl"), DEFAULT_RESPONSE))
                .add("user-agent", headersInfo.get("userAgent"))
                .add("header-accept", headersInfo.get("headerAccept"))
                .add("header-accept-encoding", headersInfo.get("headerAcceptEncoding"))
                .add("header-accept-language", headersInfo.get("headerAcceptLanguage"))

                .add("server-name", defaultString(serverProperties.getProperty("ServerName"), DEFAULT_RESPONSE))
                .add("database-name", defaultString(serverProperties.getProperty("DBName"), DEFAULT_RESPONSE))
                .add("imcms-version", Version.getImcmsVersion(getServletContext()))
                .add("database-version", defaultString(Version.getRequiredDbVersion(), DEFAULT_RESPONSE))

                .add("user-id", userId)

                .build();

        HttpPost post = new HttpPost(errorLoggerUrl);

        post.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));

        return EntityUtils.toString(httpClient.execute(post).getEntity());
    }

    private Map<String, String> parse(Throwable exception) {
        Map<String, String> exceptionInfo = new HashMap<>();

        String cause = ofNullable(exception.getCause())
                .map(ExceptionUtils::getRootCauseMessage)
                .orElse(exception.getClass().getName());
        exceptionInfo.put("cause", cause);

        String message = ofNullable(exception.getMessage())
                .map(String::new)
                .orElse(exception.getClass().getSimpleName());
        exceptionInfo.put("message", message);

        String stackTrace = ExceptionUtils.getStackTrace(exception);
        exceptionInfo.put("stackTrace", stackTrace);

        Long hash = Stream.of(cause, message, stackTrace)
                .mapToLong(String::hashCode)
                .sum();
        exceptionInfo.put("hash", hash.toString());

        return exceptionInfo;
    }

    private Map<String, String> parse(HttpServletRequest request) {
        Map<String, String> headersInfo = new HashMap<>();

        headersInfo.put("errorUrl", request.getHeader("referer"));
        headersInfo.put("userAgent", request.getHeader("user-agent"));
        headersInfo.put("headerAccept", request.getHeader("accept"));
        headersInfo.put("headerAcceptEncoding", request.getHeader("accept-encoding"));
        headersInfo.put("headerAcceptLanguage", request.getHeader("accept-language"));

        return headersInfo;
    }

    private String logError(String response, String userId) throws IOException {
        HashMap<String, Object> responseMap = new ObjectMapper().readValue(response, HashMap.class);
        String errorId = (String) responseMap.get("error_id");
        String state = (String) responseMap.get("state");
        if (state.equals("new")) {
            LOGGER.error("Internal error has occurred: {errorId =" + errorId + "; " + " userId =" + userId + "};");
        } else {
            LOGGER.info("Error with id " + errorId + " is already reported");
        }
        return errorId;
    }

    private HttpClient createHttpClient() throws Exception {

        HttpClientBuilder builder = HttpClientBuilder.create();

        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (x509Certificates, str) -> true).build();
        builder.setSslcontext(sslContext);

        HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

        SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.getSocketFactory())
                .register("https", sslSocketFactory)
                .build();

        PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
        builder.setConnectionManager(connMgr);

        return builder.build();

    }

    private static class Form {

        private static Form form;
        private static List<NameValuePair> params;

        private Form() {
        }

        public static Form form() {
            params = new LinkedList<>();
            form = new Form();
            return form;
        }

        public Form add(String name, String vlaue) {
            params.add(new BasicNameValuePair(name, vlaue));
            return form;
        }

        public List<NameValuePair> build() {
            return params;
        }

    }

}
<commit_msg>IMCMS-154 	Fixed possible Concurrent Modification Exception.<commit_after>package com.imcode.imcms.servlet;

import com.fasterxml.jackson.databind.ObjectMapper;
import imcode.server.Imcms;
import imcode.util.Utility;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.*;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import org.apache.http.ssl.SSLContextBuilder;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Stream;

import static java.util.Optional.ofNullable;
import static org.apache.commons.lang3.StringUtils.defaultString;

public class InternalError extends HttpServlet {

    private final String ERROR_500_VIEW_URL = "/imcms/500.jsp";

    private final String DEFAULT_RESPONSE = "N/A";

    private final static Logger LOGGER = Logger.getLogger(InternalError.class);

    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        String attributeName = "javax.servlet.error.exception";
        Object exception = request.getAttribute(attributeName);
        request.setAttribute(attributeName, null);

        String userId = Utility.getLoggedOnUser(request).getId() + "";

        try {
            String responseJson = sendError((Throwable) exception, request, userId);
            String errorId = logError(responseJson, userId);
            request.setAttribute("errorId", errorId);
        } catch (Exception e) {
            request.setAttribute("errorId", 0);
        }

        request.getRequestDispatcher(ERROR_500_VIEW_URL).forward(request, response);
    }

    private String sendError(Throwable exception, HttpServletRequest request, String userId) throws Exception {
        Properties serverProperties = Imcms.getServerProperties();

        Map<String, String> exceptionInfo = parse(exception);

        Map<String, String> headersInfo = parse(request);

        String errorLoggerUrl = ofNullable(serverProperties.getProperty("ErrorLoggerUrl"))
                                    .map(url -> !url.isEmpty() ? url : Imcms.ERROR_LOGGER_URL)
                                    .orElse(Imcms.ERROR_LOGGER_URL);

        HttpClient httpClient = createHttpClient();

        List<NameValuePair> params = Form.form()

                .add("hash", exceptionInfo.get("hash"))
                .add("message", exceptionInfo.get("message"))
                .add("cause", exceptionInfo.get("cause"))
                .add("stack-trace", exceptionInfo.get("stackTrace"))

                .add("error-url", defaultString(headersInfo.get("errorUrl"), DEFAULT_RESPONSE))
                .add("user-agent", headersInfo.get("userAgent"))
                .add("header-accept", headersInfo.get("headerAccept"))
                .add("header-accept-encoding", headersInfo.get("headerAcceptEncoding"))
                .add("header-accept-language", headersInfo.get("headerAcceptLanguage"))

                .add("server-name", defaultString(serverProperties.getProperty("ServerName"), DEFAULT_RESPONSE))
                .add("database-name", defaultString(serverProperties.getProperty("DBName"), DEFAULT_RESPONSE))
                .add("imcms-version", Version.getImcmsVersion(getServletContext()))
                .add("database-version", defaultString(Version.getRequiredDbVersion(), DEFAULT_RESPONSE))

                .add("user-id", userId)

                .build();

        HttpPost post = new HttpPost(errorLoggerUrl);

        post.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));

        return EntityUtils.toString(httpClient.execute(post).getEntity());
    }

    private Map<String, String> parse(Throwable exception) {
        Map<String, String> exceptionInfo = new HashMap<>();

        String cause = ofNullable(exception.getCause())
                .map(ExceptionUtils::getRootCauseMessage)
                .orElse(exception.getClass().getName());
        exceptionInfo.put("cause", cause);

        String message = ofNullable(exception.getMessage())
                .map(String::new)
                .orElse(exception.getClass().getSimpleName());
        exceptionInfo.put("message", message);

        String stackTrace = ExceptionUtils.getStackTrace(exception);
        exceptionInfo.put("stackTrace", stackTrace);

        Long hash = Stream.of(cause, message, stackTrace)
                .mapToLong(String::hashCode)
                .sum();
        exceptionInfo.put("hash", hash.toString());

        return exceptionInfo;
    }

    private Map<String, String> parse(HttpServletRequest request) {
        Map<String, String> headersInfo = new HashMap<>();

        headersInfo.put("errorUrl", request.getHeader("referer"));
        headersInfo.put("userAgent", request.getHeader("user-agent"));
        headersInfo.put("headerAccept", request.getHeader("accept"));
        headersInfo.put("headerAcceptEncoding", request.getHeader("accept-encoding"));
        headersInfo.put("headerAcceptLanguage", request.getHeader("accept-language"));

        return headersInfo;
    }

    private String logError(String response, String userId) throws IOException {
        HashMap<String, Object> responseMap = new ObjectMapper().readValue(response, HashMap.class);
        String errorId = (String) responseMap.get("error_id");
        String state = (String) responseMap.get("state");
        if (state.equals("new")) {
            LOGGER.error("Internal error has occurred: {errorId =" + errorId + "; " + " userId =" + userId + "};");
        } else {
            LOGGER.info("Error with id " + errorId + " is already reported");
        }
        return errorId;
    }

    private HttpClient createHttpClient() throws Exception {

        HttpClientBuilder builder = HttpClientBuilder.create();

        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, (x509Certificates, str) -> true).build();
        builder.setSslcontext(sslContext);

        HostnameVerifier hostnameVerifier = SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER;

        SSLConnectionSocketFactory sslSocketFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory>create()
                .register("http", PlainConnectionSocketFactory.getSocketFactory())
                .register("https", sslSocketFactory)
                .build();

        PoolingHttpClientConnectionManager connMgr = new PoolingHttpClientConnectionManager(socketFactoryRegistry);
        builder.setConnectionManager(connMgr);

        return builder.build();

    }

    private static class Form {

        private List<NameValuePair> params = new LinkedList<>();

        private Form() {
        }

        public static Form form() {
            return new Form();
        }

        public Form add(String name, String value) {
            params.add(new BasicNameValuePair(name, value));
            return this;
        }

        public List<NameValuePair> build() {
            return params;
        }

    }

}

{'fraction_non_alphanumeric': 0.11197007481296758, 'fraction_numerical': 0.0021820448877805485, 'mean_word_length': 3.2113415594644263, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5448666', 'n_tokens_mistral': 4221, 'n_tokens_neox': 4066, 'n_words': 777}