<commit_before>package lighthouse.server;

import com.google.common.base.Charsets;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.io.ByteStreams;
import com.google.protobuf.ByteString;
import com.googlecode.protobuf.format.HtmlFormat;
import com.googlecode.protobuf.format.JsonFormat;
import com.googlecode.protobuf.format.XmlFormat;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import javafx.collections.ObservableMap;
import javafx.collections.ObservableSet;
import lighthouse.LighthouseBackend;
import lighthouse.protocol.LHProtos;
import lighthouse.protocol.LHUtils;
import lighthouse.protocol.Project;
import lighthouse.threading.AffinityExecutor;
import org.bitcoinj.core.Sha256Hash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLDecoder;
import java.security.SignatureException;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;

import static com.google.common.base.Preconditions.checkState;
import static java.net.HttpURLConnection.*;

/**
 * Handler that manages all inbound requests.
 */
public class ProjectHandler implements HttpHandler {
    private static final Logger log = LoggerFactory.getLogger(ProjectHandler.class);

    // Refuse to read >1mb of data.
    private static final long MAX_REQUEST_SIZE_BYTES = 1024 * 1024;

    private final LighthouseBackend backend;
    private final AffinityExecutor executor;
    private final ObservableMap<String, LighthouseBackend.ProjectStateInfo> projectStates;

    // This ends up being mostly the same as LighthouseBackend.pledges and exists in case we want to run the web
    // server in a different thread to the backend in future.
    public static class PledgeGroup {
        public final ObservableSet<LHProtos.Pledge> open, claimed;

        public PledgeGroup(ObservableSet<LHProtos.Pledge> open, ObservableSet<LHProtos.Pledge> claimed) {
            this.open = open;
            this.claimed = claimed;
        }
    }
    private final Map<Project, PledgeGroup> pledges = new HashMap<>();

    enum DownloadFormat {
        PBUF,
        JSON,
        HTML,
        XML,
        LIGHTHOUSE_PROJECT   // protobuf of the project file itself
    }

    public ProjectHandler(LighthouseBackend backend) {
        this.backend = backend;
        // This might change in future so alias it to keep assertions simple.
        this.executor = backend.executor;
        this.projectStates = backend.mirrorProjectStates(executor);
    }

    public void sendError(HttpExchange exchange, int code) {
        try {
            log.warn("Returning HTTP error {}", code);
            exchange.sendResponseHeaders(code, -1);
            exchange.close();
        } catch (IOException e) {
            Throwables.propagate(e);
        }
    }

    public void sendSuccess(HttpExchange exchange) {
        try {
            exchange.sendResponseHeaders(200, -1);
            exchange.close();
        } catch (IOException e) {
            Throwables.propagate(e);
        }
    }

    @Override
    public void handle(HttpExchange httpExchange) throws IOException {
        // TODO: Use a SimpleTimeLimiter here to implement request timeouts.
        try {
            realHandle(httpExchange);
        } catch (Exception e) {
            log.error("Error handling request from {}: {}", httpExchange.getRemoteAddress().getAddress().getHostAddress(), httpExchange.getRequestURI());
            log.error("Took exception", e);
            sendError(httpExchange, HTTP_INTERNAL_ERROR);
        }
    }

    private void realHandle(HttpExchange httpExchange) throws Exception {
        executor.checkOnThread();
        // Simulate a slow server.
        // Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);

        final String method = httpExchange.getRequestMethod();
        URI uri = httpExchange.getRequestURI();
        String path = uri.toString();
        log.info("{}: REQ: {} {}", httpExchange.getRemoteAddress().getAddress(), method, path);
        if (!path.startsWith(LHUtils.HTTP_PATH_PREFIX + LHUtils.HTTP_PROJECT_PATH)) {
            sendError(httpExchange, HTTP_NOT_FOUND);
            return;
        }
        DownloadFormat format = DownloadFormat.PBUF;
        if (method.equals("GET") && pathEndsWithFormat(path)) {
            try {
                format = DownloadFormat.valueOf(path.substring(path.lastIndexOf(".") + 1).toUpperCase().replace('-', '_'));
            } catch (IllegalArgumentException e) {
                log.error("Could not figure out format from extension, defaulting to protobuf");
            }
            path = path.substring(0, path.lastIndexOf("."));
            uri = new URI(uri.getScheme(), uri.getHost(), path, uri.getRawFragment());
        }
        Project project = backend.getProjectFromURL(uri);
        if (project == null) {
            log.warn("Project URL did not match any known project", uri);
            sendError(httpExchange, HTTP_NOT_FOUND);
            return;
        }
        switch (method) {
            case "POST": pledgeUpload(httpExchange, project); break;
            case "GET": download(httpExchange, project, format); break;
            default: sendError(httpExchange, HTTP_BAD_METHOD); break;
        }
    }

    private boolean pathEndsWithFormat(String path) {
        return (path.endsWith(".json") || path.endsWith(".xml") || path.endsWith(".html") || path.endsWith(".lighthouse-project"));
    }
<commit_msg>More server logging improvements<commit_after>package lighthouse.server;

import com.google.common.base.Charsets;
import com.google.common.base.Splitter;
import com.google.common.base.Throwables;
import com.google.common.io.ByteStreams;
import com.google.protobuf.ByteString;
import com.googlecode.protobuf.format.HtmlFormat;
import com.googlecode.protobuf.format.JsonFormat;
import com.googlecode.protobuf.format.XmlFormat;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
import javafx.collections.ObservableMap;
import javafx.collections.ObservableSet;
import lighthouse.LighthouseBackend;
import lighthouse.protocol.LHProtos;
import lighthouse.protocol.LHUtils;
import lighthouse.protocol.Project;
import lighthouse.threading.AffinityExecutor;
import org.bitcoinj.core.Sha256Hash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URLDecoder;
import java.security.SignatureException;
import java.time.Instant;
import java.util.HashMap;
import java.util.Map;

import static com.google.common.base.Preconditions.checkState;
import static java.net.HttpURLConnection.*;

/**
 * Handler that manages all inbound requests.
 */
public class ProjectHandler implements HttpHandler {
    private static final Logger log = LoggerFactory.getLogger(ProjectHandler.class);

    // Refuse to read >1mb of data.
    private static final long MAX_REQUEST_SIZE_BYTES = 1024 * 1024;

    private final LighthouseBackend backend;
    private final AffinityExecutor executor;
    private final ObservableMap<String, LighthouseBackend.ProjectStateInfo> projectStates;

    // This ends up being mostly the same as LighthouseBackend.pledges and exists in case we want to run the web
    // server in a different thread to the backend in future.
    public static class PledgeGroup {
        public final ObservableSet<LHProtos.Pledge> open, claimed;

        public PledgeGroup(ObservableSet<LHProtos.Pledge> open, ObservableSet<LHProtos.Pledge> claimed) {
            this.open = open;
            this.claimed = claimed;
        }
    }
    private final Map<Project, PledgeGroup> pledges = new HashMap<>();

    enum DownloadFormat {
        PBUF,
        JSON,
        HTML,
        XML,
        LIGHTHOUSE_PROJECT   // protobuf of the project file itself
    }

    public ProjectHandler(LighthouseBackend backend) {
        this.backend = backend;
        // This might change in future so alias it to keep assertions simple.
        this.executor = backend.executor;
        this.projectStates = backend.mirrorProjectStates(executor);
    }

    public void sendError(HttpExchange exchange, int code) {
        try {
            log.warn("Returning HTTP error {}", code);
            exchange.sendResponseHeaders(code, -1);
            exchange.close();
        } catch (IOException e) {
            Throwables.propagate(e);
        }
    }

    public void sendSuccess(HttpExchange exchange) {
        try {
            exchange.sendResponseHeaders(200, -1);
            exchange.close();
        } catch (IOException e) {
            Throwables.propagate(e);
        }
    }

    @Override
    public void handle(HttpExchange httpExchange) throws IOException {
        // TODO: Use a SimpleTimeLimiter here to implement request timeouts.
        try {
            realHandle(httpExchange);
        } catch (Exception e) {
            log.error("Error handling request from {}: {}", httpExchange.getRemoteAddress().getAddress().getHostAddress(), httpExchange.getRequestURI());
            log.error("Took exception", e);
            sendError(httpExchange, HTTP_INTERNAL_ERROR);
        }
    }

    private void realHandle(HttpExchange httpExchange) throws Exception {
        executor.checkOnThread();
        // Simulate a slow server.
        // Uninterruptibles.sleepUninterruptibly(3, TimeUnit.SECONDS);

        final String method = httpExchange.getRequestMethod();
        URI uri = httpExchange.getRequestURI();
        String path = uri.toString();
        log.info("{}: REQ: {} {}", httpExchange.getRemoteAddress().getAddress().getHostAddress(), method, path);
        if (!path.startsWith(LHUtils.HTTP_PATH_PREFIX + LHUtils.HTTP_PROJECT_PATH)) {
            sendError(httpExchange, HTTP_NOT_FOUND);
            return;
        }
        DownloadFormat format = DownloadFormat.PBUF;
        if (method.equals("GET") && pathEndsWithFormat(path)) {
            try {
                format = DownloadFormat.valueOf(path.substring(path.lastIndexOf(".") + 1).toUpperCase().replace('-', '_'));
            } catch (IllegalArgumentException e) {
                log.error("Could not figure out format from extension, defaulting to protobuf");
            }
            path = path.substring(0, path.lastIndexOf("."));
            uri = new URI(uri.getScheme(), uri.getHost(), path, uri.getRawFragment());
        }
        Project project = backend.getProjectFromURL(uri);
        if (project == null) {
            log.warn("Project URL did not match any known project", uri);
            sendError(httpExchange, HTTP_NOT_FOUND);
            return;
        }
        switch (method) {
            case "POST": pledgeUpload(httpExchange, project); break;
            case "GET": download(httpExchange, project, format); break;
            default: sendError(httpExchange, HTTP_BAD_METHOD); break;
        }
    }

    private boolean pathEndsWithFormat(String path) {
        return (path.endsWith(".json") || path.endsWith(".xml") || path.endsWith(".html") || path.endsWith(".lighthouse-project"));
    }

{'fraction_non_alphanumeric': 0.10181397830339677, 'fraction_numerical': 0.003912502223012627, 'mean_word_length': 2.967195767195767, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6437478', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2995, 'n_words': 737}