<commit_before>/*
 * Copyright (c) 2014 AsyncHttpClient Project. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at
 *     http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package org.asynchttpclient.netty.handler;

import static io.netty.handler.codec.http.HttpHeaders.Names.*;
import static io.netty.handler.codec.http.HttpResponseStatus.*;
import static org.asynchttpclient.util.Assertions.assertNotNull;
import static org.asynchttpclient.util.HttpUtils.*;
import io.netty.channel.Channel;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponse;

import java.util.HashSet;
import java.util.Set;

import org.asynchttpclient.AsyncHandler;
import org.asynchttpclient.AsyncHttpClientConfig;
import org.asynchttpclient.HttpResponseHeaders;
import org.asynchttpclient.HttpResponseStatus;
import org.asynchttpclient.Realm;
import org.asynchttpclient.Realm.AuthScheme;
import org.asynchttpclient.Request;
import org.asynchttpclient.RequestBuilder;
import org.asynchttpclient.cookie.Cookie;
import org.asynchttpclient.cookie.CookieDecoder;
import org.asynchttpclient.filter.FilterContext;
import org.asynchttpclient.filter.FilterException;
import org.asynchttpclient.filter.ResponseFilter;
import org.asynchttpclient.handler.MaxRedirectException;
import org.asynchttpclient.netty.NettyResponseFuture;
import org.asynchttpclient.netty.channel.ChannelManager;
import org.asynchttpclient.netty.request.NettyRequestSender;
import org.asynchttpclient.uri.Uri;
import org.asynchttpclient.util.MiscUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class Protocol {

    protected final Logger logger = LoggerFactory.getLogger(getClass());

    protected final ChannelManager channelManager;
    protected final AsyncHttpClientConfig config;
    protected final NettyRequestSender requestSender;

    private final boolean hasResponseFilters;
    protected final boolean hasIOExceptionFilters;
    private final MaxRedirectException maxRedirectException;

    public static final Set<Integer> REDIRECT_STATUSES = new HashSet<>();
    static {
        REDIRECT_STATUSES.add(MOVED_PERMANENTLY.code());
        REDIRECT_STATUSES.add(FOUND.code());
        REDIRECT_STATUSES.add(SEE_OTHER.code());
        REDIRECT_STATUSES.add(TEMPORARY_REDIRECT.code());
    }

    public Protocol(ChannelManager channelManager, AsyncHttpClientConfig config, NettyRequestSender requestSender) {
        this.channelManager = channelManager;
        this.config = config;
        this.requestSender = requestSender;

        hasResponseFilters = !config.getResponseFilters().isEmpty();
        hasIOExceptionFilters = !config.getIoExceptionFilters().isEmpty();
        maxRedirectException = new MaxRedirectException("Maximum redirect reached: " + config.getMaxRedirects());
    }

    public abstract void handle(Channel channel, NettyResponseFuture<?> future, Object message) throws Exception;

    public abstract void onError(NettyResponseFuture<?> future, Throwable error);

    public abstract void onClose(NettyResponseFuture<?> future);

    private HttpHeaders propagatedHeaders(Request request, Realm realm, boolean keepBody) {

        HttpHeaders headers = request.getHeaders()//
                .remove(HttpHeaders.Names.HOST)//
                .remove(HttpHeaders.Names.CONTENT_LENGTH);

        if (!keepBody) {
            headers.remove(HttpHeaders.Names.CONTENT_TYPE);
        }

        if (realm != null && realm.getScheme() == AuthScheme.NTLM) {
            headers.remove(AUTHORIZATION)//
                    .remove(PROXY_AUTHORIZATION);
        }
        return headers;
    }

    protected boolean exitAfterHandlingRedirect(//
            Channel channel,//
            NettyResponseFuture<?> future,//
            HttpResponse response,//
            Request request,//
            int statusCode,//
            Realm realm) throws Exception {

        if (followRedirect(config, request)) {
            if (future.incrementAndGetCurrentRedirectCount() >= config.getMaxRedirects()) {
                throw maxRedirectException;

            } else {
                // We must allow auth handling again.
                future.getInAuth().set(false);
                future.getInProxyAuth().set(false);

                String originalMethod = request.getMethod();
                boolean switchToGet = !originalMethod.equals(HttpMethod.GET.name()) && (statusCode == 301 || statusCode == 303 || (statusCode == 302 && !config.isStrict302Handling()));
                boolean keepBody = statusCode == 307 || (statusCode == 302 && config.isStrict302Handling());

                final RequestBuilder requestBuilder = new RequestBuilder(switchToGet ? HttpMethod.GET.name() : originalMethod)//
                        .setCookies(request.getCookies())//
                        .setConnectionPoolPartitioning(request.getConnectionPoolPartitioning())//
                        .setFollowRedirect(true)//
                        .setLocalAddress(request.getLocalAddress())//
                        .setNameResolver(request.getNameResolver())//
                        .setProxyServer(request.getProxyServer())//
                        .setRealm(request.getRealm())//
                        .setRequestTimeout(request.getRequestTimeout());

                if (keepBody) {
                    requestBuilder.setCharset(request.getCharset());
                    if (MiscUtils.isNonEmpty(request.getFormParams()))
                        requestBuilder.setFormParams(request.getFormParams());
                    else if (request.getStringData() != null)
                        requestBuilder.setBody(request.getStringData());
                    else if (request.getByteData() != null)
                        requestBuilder.setBody(request.getByteData());
                    else if (request.getByteBufferData() != null)
                        requestBuilder.setBody(request.getByteBufferData());
                    else if (request.getBodyGenerator() != null)
                        requestBuilder.setBody(request.getBodyGenerator());
                }

                requestBuilder.setHeaders(propagatedHeaders(request, realm, keepBody));

                // in case of a redirect from HTTP to HTTPS, future
                // attributes might change
                final boolean initialConnectionKeepAlive = future.isKeepAlive();
                final Object initialPartitionKey = future.getPartitionKey();

                HttpHeaders responseHeaders = response.headers();
                String location = responseHeaders.get(HttpHeaders.Names.LOCATION);
                Uri newUri = Uri.create(future.getUri(), location);

                logger.debug("Redirecting to {}", newUri);

                for (String cookieStr : responseHeaders.getAll(HttpHeaders.Names.SET_COOKIE)) {
                    Cookie c = CookieDecoder.decode(cookieStr);
                    if (c != null)
                        requestBuilder.addOrReplaceCookie(c);
                }

                requestBuilder.setHeaders(propagatedHeaders(future.getCurrentRequest(), realm, keepBody));

                boolean sameBase = isSameBase(request.getUri(), newUri);

                if (sameBase) {
                    // we can only assume the virtual host is still valid if the baseUrl is the same
                    requestBuilder.setVirtualHost(request.getVirtualHost());
                }

                final Request nextRequest = requestBuilder.setUri(newUri).build();
                future.setTargetRequest(nextRequest);

                logger.debug("Sending redirect to {}", newUri);
<commit_msg>Don't set headers twice on redirect<commit_after>/*
 * Copyright (c) 2014 AsyncHttpClient Project. All rights reserved.
 *
 * This program is licensed to you under the Apache License Version 2.0,
 * and you may not use this file except in compliance with the Apache License Version 2.0.
 * You may obtain a copy of the Apache License Version 2.0 at
 *     http://www.apache.org/licenses/LICENSE-2.0.
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the Apache License Version 2.0 is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
 */
package org.asynchttpclient.netty.handler;

import static io.netty.handler.codec.http.HttpHeaders.Names.*;
import static io.netty.handler.codec.http.HttpResponseStatus.*;
import static org.asynchttpclient.util.Assertions.assertNotNull;
import static org.asynchttpclient.util.HttpUtils.*;
import io.netty.channel.Channel;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpMethod;
import io.netty.handler.codec.http.HttpResponse;

import java.util.HashSet;
import java.util.Set;

import org.asynchttpclient.AsyncHandler;
import org.asynchttpclient.AsyncHttpClientConfig;
import org.asynchttpclient.HttpResponseHeaders;
import org.asynchttpclient.HttpResponseStatus;
import org.asynchttpclient.Realm;
import org.asynchttpclient.Realm.AuthScheme;
import org.asynchttpclient.Request;
import org.asynchttpclient.RequestBuilder;
import org.asynchttpclient.cookie.Cookie;
import org.asynchttpclient.cookie.CookieDecoder;
import org.asynchttpclient.filter.FilterContext;
import org.asynchttpclient.filter.FilterException;
import org.asynchttpclient.filter.ResponseFilter;
import org.asynchttpclient.handler.MaxRedirectException;
import org.asynchttpclient.netty.NettyResponseFuture;
import org.asynchttpclient.netty.channel.ChannelManager;
import org.asynchttpclient.netty.request.NettyRequestSender;
import org.asynchttpclient.uri.Uri;
import org.asynchttpclient.util.MiscUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class Protocol {

    protected final Logger logger = LoggerFactory.getLogger(getClass());

    protected final ChannelManager channelManager;
    protected final AsyncHttpClientConfig config;
    protected final NettyRequestSender requestSender;

    private final boolean hasResponseFilters;
    protected final boolean hasIOExceptionFilters;
    private final MaxRedirectException maxRedirectException;

    public static final Set<Integer> REDIRECT_STATUSES = new HashSet<>();
    static {
        REDIRECT_STATUSES.add(MOVED_PERMANENTLY.code());
        REDIRECT_STATUSES.add(FOUND.code());
        REDIRECT_STATUSES.add(SEE_OTHER.code());
        REDIRECT_STATUSES.add(TEMPORARY_REDIRECT.code());
    }

    public Protocol(ChannelManager channelManager, AsyncHttpClientConfig config, NettyRequestSender requestSender) {
        this.channelManager = channelManager;
        this.config = config;
        this.requestSender = requestSender;

        hasResponseFilters = !config.getResponseFilters().isEmpty();
        hasIOExceptionFilters = !config.getIoExceptionFilters().isEmpty();
        maxRedirectException = new MaxRedirectException("Maximum redirect reached: " + config.getMaxRedirects());
    }

    public abstract void handle(Channel channel, NettyResponseFuture<?> future, Object message) throws Exception;

    public abstract void onError(NettyResponseFuture<?> future, Throwable error);

    public abstract void onClose(NettyResponseFuture<?> future);

    private HttpHeaders propagatedHeaders(Request request, Realm realm, boolean keepBody) {

        HttpHeaders headers = request.getHeaders()//
                .remove(HttpHeaders.Names.HOST)//
                .remove(HttpHeaders.Names.CONTENT_LENGTH);

        if (!keepBody) {
            headers.remove(HttpHeaders.Names.CONTENT_TYPE);
        }

        if (realm != null && realm.getScheme() == AuthScheme.NTLM) {
            headers.remove(AUTHORIZATION)//
                    .remove(PROXY_AUTHORIZATION);
        }
        return headers;
    }

    protected boolean exitAfterHandlingRedirect(//
            Channel channel,//
            NettyResponseFuture<?> future,//
            HttpResponse response,//
            Request request,//
            int statusCode,//
            Realm realm) throws Exception {

        if (followRedirect(config, request)) {
            if (future.incrementAndGetCurrentRedirectCount() >= config.getMaxRedirects()) {
                throw maxRedirectException;

            } else {
                // We must allow auth handling again.
                future.getInAuth().set(false);
                future.getInProxyAuth().set(false);

                String originalMethod = request.getMethod();
                boolean switchToGet = !originalMethod.equals(HttpMethod.GET.name()) && (statusCode == 301 || statusCode == 303 || (statusCode == 302 && !config.isStrict302Handling()));
                boolean keepBody = statusCode == 307 || (statusCode == 302 && config.isStrict302Handling());

                final RequestBuilder requestBuilder = new RequestBuilder(switchToGet ? HttpMethod.GET.name() : originalMethod)//
                        .setCookies(request.getCookies())//
                        .setConnectionPoolPartitioning(request.getConnectionPoolPartitioning())//
                        .setFollowRedirect(true)//
                        .setLocalAddress(request.getLocalAddress())//
                        .setNameResolver(request.getNameResolver())//
                        .setProxyServer(request.getProxyServer())//
                        .setRealm(request.getRealm())//
                        .setRequestTimeout(request.getRequestTimeout());

                if (keepBody) {
                    requestBuilder.setCharset(request.getCharset());
                    if (MiscUtils.isNonEmpty(request.getFormParams()))
                        requestBuilder.setFormParams(request.getFormParams());
                    else if (request.getStringData() != null)
                        requestBuilder.setBody(request.getStringData());
                    else if (request.getByteData() != null)
                        requestBuilder.setBody(request.getByteData());
                    else if (request.getByteBufferData() != null)
                        requestBuilder.setBody(request.getByteBufferData());
                    else if (request.getBodyGenerator() != null)
                        requestBuilder.setBody(request.getBodyGenerator());
                }

                requestBuilder.setHeaders(propagatedHeaders(request, realm, keepBody));

                // in case of a redirect from HTTP to HTTPS, future
                // attributes might change
                final boolean initialConnectionKeepAlive = future.isKeepAlive();
                final Object initialPartitionKey = future.getPartitionKey();

                HttpHeaders responseHeaders = response.headers();
                String location = responseHeaders.get(HttpHeaders.Names.LOCATION);
                Uri newUri = Uri.create(future.getUri(), location);

                logger.debug("Redirecting to {}", newUri);

                for (String cookieStr : responseHeaders.getAll(HttpHeaders.Names.SET_COOKIE)) {
                    Cookie c = CookieDecoder.decode(cookieStr);
                    if (c != null)
                        requestBuilder.addOrReplaceCookie(c);
                }

                boolean sameBase = isSameBase(request.getUri(), newUri);

                if (sameBase) {
                    // we can only assume the virtual host is still valid if the baseUrl is the same
                    requestBuilder.setVirtualHost(request.getVirtualHost());
                }

                final Request nextRequest = requestBuilder.setUri(newUri).build();
                future.setTargetRequest(nextRequest);

                logger.debug("Sending redirect to {}", newUri);

{'fraction_non_alphanumeric': 0.09118578037496956, 'fraction_numerical': 0.004747991234477721, 'mean_word_length': 2.6952316689158793, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5452277', 'n_tokens_mistral': 4243, 'n_tokens_neox': 4011, 'n_words': 930}