<commit_before>package org.jetbrains.io.fastCgi;

import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.util.text.StringUtilRt;
import com.intellij.util.containers.ConcurrentIntObjectMap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.http.*;
import org.jetbrains.io.Responses;
import org.jetbrains.io.SimpleChannelInboundHandlerAdapter;

import static org.jetbrains.io.fastCgi.FastCgiService.LOG;

@ChannelHandler.Sharable
public class FastCgiChannelHandler extends SimpleChannelInboundHandlerAdapter<FastCgiResponse> {
  private final ConcurrentIntObjectMap<Channel> requestToChannel;

  public FastCgiChannelHandler(ConcurrentIntObjectMap<Channel> channel) {
    requestToChannel = channel;
  }

  @Override
  protected void messageReceived(ChannelHandlerContext context, FastCgiResponse response) throws Exception {
    ByteBuf buffer = response.getData();
    Channel channel = requestToChannel.remove(response.getId());
    if (channel == null || !channel.isActive()) {
      if (buffer != null) {
        buffer.release();
      }
      return;
    }

    if (buffer == null) {
      Responses.sendStatus(HttpResponseStatus.BAD_GATEWAY, channel);
      return;
    }

    HttpResponse httpResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buffer);
    try {
      parseHeaders(httpResponse, buffer);
      Responses.addServer(httpResponse);
      if (!HttpHeaders.isContentLengthSet(httpResponse)) {
        HttpHeaders.setContentLength(httpResponse, buffer.readableBytes());
      }
    }
    catch (Throwable e) {
      buffer.release();
      Responses.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR, channel);
      LOG.error(e);
    }
    channel.writeAndFlush(httpResponse);
  }

  private static void parseHeaders(HttpResponse response, ByteBuf buffer) {
    StringBuilder builder = new StringBuilder();
    while (buffer.isReadable()) {
      builder.setLength(0);

      String key = null;
      boolean valueExpected = true;
      while (true) {
        int b = buffer.readByte();
        if (b < 0 || b == '\n') {
          break;
        }

        if (b != '\r') {
          if (valueExpected && b == ':') {
            valueExpected = false;

            key = builder.toString();
            builder.setLength(0);
            skipWhitespace(buffer);
          }
          else {
            builder.append((char)b);
          }
        }
      }

      if (builder.length() == 0) {
        // end of headers
        return;
      }

      // skip standard headers
      if (StringUtil.isEmpty(key) || StringUtilRt.startsWithIgnoreCase(key, "http") || StringUtilRt.startsWithIgnoreCase(key, "X-Accel-")) {
        continue;
      }

      String value = builder.toString();
      if (key.equalsIgnoreCase("status")) {
        response.setStatus(HttpResponseStatus.valueOf(Integer.parseInt(value.substring(0, value.indexOf(' ')))));
      }
      else if (!(key.startsWith("http") || key.startsWith("HTTP"))) {
        response.headers().add(key, value);
      }<commit_msg>add logging — we cannot parse status<commit_after>package org.jetbrains.io.fastCgi;

import com.intellij.openapi.util.text.StringUtil;
import com.intellij.openapi.util.text.StringUtilRt;
import com.intellij.util.containers.ConcurrentIntObjectMap;
import io.netty.buffer.ByteBuf;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.handler.codec.http.*;
import org.jetbrains.io.Responses;
import org.jetbrains.io.SimpleChannelInboundHandlerAdapter;

import static org.jetbrains.io.fastCgi.FastCgiService.LOG;

@ChannelHandler.Sharable
public class FastCgiChannelHandler extends SimpleChannelInboundHandlerAdapter<FastCgiResponse> {
  private final ConcurrentIntObjectMap<Channel> requestToChannel;

  public FastCgiChannelHandler(ConcurrentIntObjectMap<Channel> channel) {
    requestToChannel = channel;
  }

  @Override
  protected void messageReceived(ChannelHandlerContext context, FastCgiResponse response) throws Exception {
    ByteBuf buffer = response.getData();
    Channel channel = requestToChannel.remove(response.getId());
    if (channel == null || !channel.isActive()) {
      if (buffer != null) {
        buffer.release();
      }
      return;
    }

    if (buffer == null) {
      Responses.sendStatus(HttpResponseStatus.BAD_GATEWAY, channel);
      return;
    }

    HttpResponse httpResponse = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, buffer);
    try {
      parseHeaders(httpResponse, buffer);
      Responses.addServer(httpResponse);
      if (!HttpHeaders.isContentLengthSet(httpResponse)) {
        HttpHeaders.setContentLength(httpResponse, buffer.readableBytes());
      }
    }
    catch (Throwable e) {
      buffer.release();
      Responses.sendStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR, channel);
      LOG.error(e);
    }
    channel.writeAndFlush(httpResponse);
  }

  private static void parseHeaders(HttpResponse response, ByteBuf buffer) {
    StringBuilder builder = new StringBuilder();
    while (buffer.isReadable()) {
      builder.setLength(0);

      String key = null;
      boolean valueExpected = true;
      while (true) {
        int b = buffer.readByte();
        if (b < 0 || b == '\n') {
          break;
        }

        if (b != '\r') {
          if (valueExpected && b == ':') {
            valueExpected = false;

            key = builder.toString();
            builder.setLength(0);
            skipWhitespace(buffer);
          }
          else {
            builder.append((char)b);
          }
        }
      }

      if (builder.length() == 0) {
        // end of headers
        return;
      }

      // skip standard headers
      if (StringUtil.isEmpty(key) || StringUtilRt.startsWithIgnoreCase(key, "http") || StringUtilRt.startsWithIgnoreCase(key, "X-Accel-")) {
        continue;
      }

      String value = builder.toString();
      if (key.equalsIgnoreCase("status")) {
        int index = value.indexOf(' ');
        if (index == -1) {
          LOG.warn("Cannot parse status: " + value);
          response.setStatus(HttpResponseStatus.OK);
        }
        else {
          response.setStatus(HttpResponseStatus.valueOf(Integer.parseInt(value.substring(0, index))));
        }
      }
      else if (!(key.startsWith("http") || key.startsWith("HTTP"))) {
        response.headers().add(key, value);
      }
{'fraction_non_alphanumeric': 0.11705027256208358, 'fraction_numerical': 0.0022713506965475467, 'mean_word_length': 2.990936555891239, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5395683', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1897, 'n_words': 353}