<commit_before>/**
 * Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.commonjava.indy.httprox.handler;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ClassUtils;
import org.apache.http.HttpRequest;
import org.apache.http.RequestLine;
import org.commonjava.indy.IndyWorkflowException;
import org.commonjava.indy.audit.ChangeSummary;
import org.commonjava.indy.bind.jaxrs.MDCManager;
import org.commonjava.indy.core.ctl.ContentController;
import org.commonjava.indy.data.ArtifactStoreQuery;
import org.commonjava.indy.data.IndyDataException;
import org.commonjava.indy.data.StoreDataManager;
import org.commonjava.indy.folo.ctl.FoloConstants;
import org.commonjava.indy.folo.model.TrackingKey;
import org.commonjava.indy.httprox.conf.HttproxConfig;
import org.commonjava.indy.httprox.conf.TrackingType;
import org.commonjava.indy.httprox.keycloak.KeycloakProxyAuthenticator;
import org.commonjava.indy.httprox.util.HttpConduitWrapper;
import org.commonjava.indy.metrics.conf.IndyMetricsConfig;
import org.commonjava.indy.model.core.AccessChannel;
import org.commonjava.indy.model.core.ArtifactStore;
import org.commonjava.indy.model.core.Group;
import org.commonjava.indy.model.core.HostedRepository;
import org.commonjava.indy.model.core.RemoteRepository;
import org.commonjava.indy.model.core.StoreType;
import org.commonjava.indy.subsys.http.HttpWrapper;
import org.commonjava.indy.subsys.http.util.UserPass;
import org.commonjava.indy.subsys.infinispan.CacheHandle;
import org.commonjava.indy.subsys.infinispan.CacheProducer;
import org.commonjava.indy.util.ApplicationHeader;
import org.commonjava.indy.util.ApplicationStatus;
import org.commonjava.indy.util.UrlInfo;
import org.commonjava.maven.galley.TransferException;
import org.commonjava.maven.galley.event.EventMetadata;
import org.commonjava.maven.galley.model.Transfer;
import org.commonjava.maven.galley.spi.cache.CacheProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xnio.ChannelListener;
import org.xnio.StreamConnection;
import org.xnio.conduits.ConduitStreamSinkChannel;

import java.io.IOException;
import java.net.SocketAddress;
import java.net.URL;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static com.codahale.metrics.MetricRegistry.name;
import static org.apache.commons.codec.digest.DigestUtils.md5Hex;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.ALLOW_HEADER_VALUE;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.GET_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.HEAD_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.OPTIONS_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.PROXY_AUTHENTICATE_FORMAT;
import static org.commonjava.indy.model.core.ArtifactStore.TRACKING_ID;
import static org.commonjava.indy.model.core.GenericPackageTypeDescriptor.GENERIC_PKG_KEY;
import static org.commonjava.indy.util.ApplicationHeader.proxy_authenticate;
import static org.commonjava.indy.util.ApplicationStatus.PROXY_AUTHENTICATION_REQUIRED;
import static org.commonjava.maven.galley.io.SpecialPathConstants.PKG_TYPE_GENERIC_HTTP;

public final class ProxyResponseWriter
                implements ChannelListener<ConduitStreamSinkChannel>
{

    private static final String TRACKED_USER_SUFFIX = "+tracking";

    private static final String HTTP_PROXY_AUTH_CACHE = "httproxy-auth-cache";

    private static final int DEFAULT_AUTH_CACHE_EXPIRATION_HOURS = 3;

    private final Logger logger = LoggerFactory.getLogger( getClass() );

    private final Logger restLogger = LoggerFactory.getLogger( "org.commonjava.topic.httprox.inbound" );

    private final SocketAddress peerAddress;

    private final CacheHandle<String, Boolean> proxyAuthCache;

    private Throwable error;

    private final HttproxConfig config;

    private final ContentController contentController;

    private final StoreDataManager storeManager;

    private KeycloakProxyAuthenticator proxyAuthenticator;

    private CacheProvider cacheProvider;

    private ProxyRepositoryCreator repoCreator;

    private boolean transferred;

    private HttpRequest httpRequest;

    private MDCManager mdcManager;

    private MetricRegistry metricRegistry;

    private IndyMetricsConfig metricsConfig;

    private String cls; // short class name for metrics

    public ProxyResponseWriter( final HttproxConfig config, final StoreDataManager storeManager,
                                final ContentController contentController,
                                final KeycloakProxyAuthenticator proxyAuthenticator, final CacheProvider cacheProvider,
                                final MDCManager mdcManager,
                                final ProxyRepositoryCreator repoCreator, final StreamConnection streamConnection,
                                final IndyMetricsConfig metricsConfig, final MetricRegistry metricRegistry,
                                final CacheProducer cacheProducer )
    {
        this.config = config;
        this.contentController = contentController;
        this.storeManager = storeManager;
        this.proxyAuthenticator = proxyAuthenticator;
        this.cacheProvider = cacheProvider;
        this.mdcManager = mdcManager;
        this.repoCreator = repoCreator;
        this.peerAddress = streamConnection.getPeerAddress();
        this.metricsConfig = metricsConfig;
        this.metricRegistry = metricRegistry;
        this.cls = ClassUtils.getAbbreviatedName( getClass().getName(), 1 ); // e.g., foo.bar.ClassA -> f.b.ClassA
        this.proxyAuthCache = cacheProducer.getCache( HTTP_PROXY_AUTH_CACHE, String.class, Boolean.class );
    }

    @Override
    public void handleEvent( final ConduitStreamSinkChannel channel )
    {
        if ( metricsConfig == null || metricRegistry == null )
        {
            doHandleEvent( channel );
            return;
        }

        Timer timer = metricRegistry.timer( name( metricsConfig.getNodePrefix(), cls, "handleEvent" ) );
        Timer.Context timerContext = timer.time();
        try
        {
            doHandleEvent( channel );
        }
        finally
        {
            timerContext.stop();
        }
    }

    private void doHandleEvent( final ConduitStreamSinkChannel channel )
    {
        HttpConduitWrapper http = new HttpConduitWrapper( channel, httpRequest, contentController, cacheProvider );
        if ( httpRequest == null )
        {
            if ( error != null )
            {
                logger.debug( "handling error from request reader: " + error.getMessage(), error );
                handleError( error, http );
            }
            else
            {
                logger.debug( "invalid state (no error or request) from request reader. Sending 400." );
                try
                {
                    http.writeStatus( ApplicationStatus.BAD_REQUEST );
                }
                catch ( final IOException e )
                {
                    logger.error( "Failed to write BAD REQUEST for missing HTTP first-line to response channel.", e );
                }
            }

            return;
        }

        restLogger.info( "START {} (from: {})", httpRequest.getRequestLine(), peerAddress );

        // TODO: Can we handle this?
        final String oldThreadName = Thread.currentThread().getName();
        Thread.currentThread().setName( "PROXY-" + httpRequest.getRequestLine().toString() );
        channel.getCloseSetter().set( ( sinkChannel ) ->
        {
            restLogger.info( "END {} (from: {})", httpRequest.getRequestLine(), peerAddress );
            logger.debug("sink channel closing.");
            Thread.currentThread().setName( oldThreadName );
        } );

        logger.debug( "\n\n\n>>>>>>> Handle write\n\n\n\n\n" );
        if ( error == null )
        {
            try
            {
                if ( repoCreator == null )
                {
                    throw new IndyDataException( "No valid instance of ProxyRepositoryCreator" );
                }

                final UserPass proxyUserPass =
                                UserPass.parse( ApplicationHeader.proxy_authorization, httpRequest, null );

                mdcManager.putExtraHeaders( httpRequest );
                if ( proxyUserPass != null )
                {
                    mdcManager.putExternalID( proxyUserPass.getUser() );
                }

                logger.debug( "Proxy UserPass: {}\nConfig secured? {}\nConfig tracking type: {}", proxyUserPass,
                              config.isSecured(), config.getTrackingType() );
                if ( proxyUserPass == null && ( config.isSecured()
                                || TrackingType.ALWAYS == config.getTrackingType() ) )
                {

                    String realmInfo = String.format( PROXY_AUTHENTICATE_FORMAT, config.getProxyRealm() );

                    logger.info( "Not authenticated to proxy. Sending response: {} / {}: {}",
                                 PROXY_AUTHENTICATION_REQUIRED, proxy_authenticate, realmInfo );

                    http.writeStatus( PROXY_AUTHENTICATION_REQUIRED );
                    http.writeHeader( proxy_authenticate,
                                      realmInfo );
                }
                else
                {
                    RequestLine requestLine = httpRequest.getRequestLine();
                    String method = requestLine.getMethod().toUpperCase();
                    String trackingId = null;
                    boolean authenticated = true;

                    if ( proxyUserPass != null )
                    {
                        TrackingKey trackingKey = getTrackingKey( proxyUserPass );
                        if ( trackingKey != null )
                        {
                            trackingId = trackingKey.getId();
                        }

                        String authCacheKey = generateAuthCacheKey( proxyUserPass );
                        Boolean isAuthToken = proxyAuthCache.get( authCacheKey );
                        if ( Boolean.TRUE.equals( isAuthToken ) )
                        {
                            authenticated = true;
                            logger.debug("Found auth key in cache" );
                        }
                        else
                        {
                            logger.debug( "Passing BASIC authentication credentials to Keycloak bearer-token translation authenticator" );
                            authenticated = proxyAuthenticator.authenticate( proxyUserPass, http );
                            if ( authenticated )
                            {
                                proxyAuthCache.put( authCacheKey, Boolean.TRUE, DEFAULT_AUTH_CACHE_EXPIRATION_HOURS, TimeUnit.HOURS );
                            }
                        }
                        logger.debug( "Authentication done, result: {}", authenticated );
                    }

                    if ( authenticated )
                    {
                        final URL url = new URL( requestLine.getUri() );

                        switch ( method )
                        {
                            case GET_METHOD:
                            case HEAD_METHOD:
                            {
                                logger.debug( "getArtifactStore starts, trackingId: {}, url: {}", trackingId, url );
                                ArtifactStore store = getArtifactStore( trackingId, url );
                                logger.debug( "getArtifactStore done, store: {}", store );

                                transfer( http, store, url.getPath(), GET_METHOD.equals( method ), proxyUserPass );
                                break;
                            }
                            case OPTIONS_METHOD:
                            {
                                http.writeStatus( ApplicationStatus.OK );
                                http.writeHeader( ApplicationHeader.allow, ALLOW_HEADER_VALUE );
                                break;
                            }
                            default:
                            {
                                http.writeStatus( ApplicationStatus.METHOD_NOT_ALLOWED );
                            }
                        }
                    }
                }

                logger.debug( "Response complete." );
            }
            catch ( final Throwable e )
            {
                error = e;
            }
            finally
            {
                mdcManager.clear();
            }
        }

        if ( error != null )
        {
            handleError( error, http );
        }

        try
        {
            http.close();
        }
        catch ( final IOException e )
        {
            logger.error( "Failed to flush/shutdown response.", e );
        }
    }

    private String generateAuthCacheKey( UserPass proxyUserPass )
    {
        return md5Hex( proxyUserPass.getUser() + ":" + proxyUserPass.getPassword() );
    }

    private void handleError( final Throwable error, final HttpWrapper http )
    {
        logger.error( "HTTProx request failed: " + error.getMessage(), error );
        try
        {
            if ( http.isOpen() )
            {
                if ( error instanceof IndyWorkflowException )
                {
                    http.writeStatus( ApplicationStatus.getStatus( ( (IndyWorkflowException) error ).getStatus() ) );
                }
                else
                {
                    http.writeStatus( ApplicationStatus.SERVER_ERROR );
                }

                http.writeError( error );

                logger.debug( "Response error complete." );<commit_msg>decrease proxy auth cache timeout and use SHA-256 for auth cache key hashing<commit_after>/**
 * Copyright (C) 2011-2018 Red Hat, Inc. (https://github.com/Commonjava/indy)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.commonjava.indy.httprox.handler;

import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ClassUtils;
import org.apache.http.HttpRequest;
import org.apache.http.RequestLine;
import org.commonjava.indy.IndyWorkflowException;
import org.commonjava.indy.audit.ChangeSummary;
import org.commonjava.indy.bind.jaxrs.MDCManager;
import org.commonjava.indy.core.ctl.ContentController;
import org.commonjava.indy.data.ArtifactStoreQuery;
import org.commonjava.indy.data.IndyDataException;
import org.commonjava.indy.data.StoreDataManager;
import org.commonjava.indy.folo.ctl.FoloConstants;
import org.commonjava.indy.folo.model.TrackingKey;
import org.commonjava.indy.httprox.conf.HttproxConfig;
import org.commonjava.indy.httprox.conf.TrackingType;
import org.commonjava.indy.httprox.keycloak.KeycloakProxyAuthenticator;
import org.commonjava.indy.httprox.util.HttpConduitWrapper;
import org.commonjava.indy.metrics.conf.IndyMetricsConfig;
import org.commonjava.indy.model.core.AccessChannel;
import org.commonjava.indy.model.core.ArtifactStore;
import org.commonjava.indy.model.core.Group;
import org.commonjava.indy.model.core.HostedRepository;
import org.commonjava.indy.model.core.RemoteRepository;
import org.commonjava.indy.model.core.StoreType;
import org.commonjava.indy.subsys.http.HttpWrapper;
import org.commonjava.indy.subsys.http.util.UserPass;
import org.commonjava.indy.subsys.infinispan.CacheHandle;
import org.commonjava.indy.subsys.infinispan.CacheProducer;
import org.commonjava.indy.util.ApplicationHeader;
import org.commonjava.indy.util.ApplicationStatus;
import org.commonjava.indy.util.UrlInfo;
import org.commonjava.maven.galley.TransferException;
import org.commonjava.maven.galley.event.EventMetadata;
import org.commonjava.maven.galley.model.Transfer;
import org.commonjava.maven.galley.spi.cache.CacheProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xnio.ChannelListener;
import org.xnio.StreamConnection;
import org.xnio.conduits.ConduitStreamSinkChannel;

import java.io.IOException;
import java.net.SocketAddress;
import java.net.URL;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import java.util.stream.Collectors;

import static com.codahale.metrics.MetricRegistry.name;
import static org.apache.commons.codec.digest.DigestUtils.sha256Hex;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.ALLOW_HEADER_VALUE;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.GET_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.HEAD_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.OPTIONS_METHOD;
import static org.commonjava.indy.httprox.util.HttpProxyConstants.PROXY_AUTHENTICATE_FORMAT;
import static org.commonjava.indy.model.core.ArtifactStore.TRACKING_ID;
import static org.commonjava.indy.model.core.GenericPackageTypeDescriptor.GENERIC_PKG_KEY;
import static org.commonjava.indy.util.ApplicationHeader.proxy_authenticate;
import static org.commonjava.indy.util.ApplicationStatus.PROXY_AUTHENTICATION_REQUIRED;
import static org.commonjava.maven.galley.io.SpecialPathConstants.PKG_TYPE_GENERIC_HTTP;

public final class ProxyResponseWriter
                implements ChannelListener<ConduitStreamSinkChannel>
{

    private static final String TRACKED_USER_SUFFIX = "+tracking";

    private static final String HTTP_PROXY_AUTH_CACHE = "httproxy-auth-cache";

    private static final int DEFAULT_AUTH_CACHE_EXPIRATION_HOURS = 1;

    private final Logger logger = LoggerFactory.getLogger( getClass() );

    private final Logger restLogger = LoggerFactory.getLogger( "org.commonjava.topic.httprox.inbound" );

    private final SocketAddress peerAddress;

    private final CacheHandle<String, Boolean> proxyAuthCache;

    private Throwable error;

    private final HttproxConfig config;

    private final ContentController contentController;

    private final StoreDataManager storeManager;

    private KeycloakProxyAuthenticator proxyAuthenticator;

    private CacheProvider cacheProvider;

    private ProxyRepositoryCreator repoCreator;

    private boolean transferred;

    private HttpRequest httpRequest;

    private MDCManager mdcManager;

    private MetricRegistry metricRegistry;

    private IndyMetricsConfig metricsConfig;

    private String cls; // short class name for metrics

    public ProxyResponseWriter( final HttproxConfig config, final StoreDataManager storeManager,
                                final ContentController contentController,
                                final KeycloakProxyAuthenticator proxyAuthenticator, final CacheProvider cacheProvider,
                                final MDCManager mdcManager,
                                final ProxyRepositoryCreator repoCreator, final StreamConnection streamConnection,
                                final IndyMetricsConfig metricsConfig, final MetricRegistry metricRegistry,
                                final CacheProducer cacheProducer )
    {
        this.config = config;
        this.contentController = contentController;
        this.storeManager = storeManager;
        this.proxyAuthenticator = proxyAuthenticator;
        this.cacheProvider = cacheProvider;
        this.mdcManager = mdcManager;
        this.repoCreator = repoCreator;
        this.peerAddress = streamConnection.getPeerAddress();
        this.metricsConfig = metricsConfig;
        this.metricRegistry = metricRegistry;
        this.cls = ClassUtils.getAbbreviatedName( getClass().getName(), 1 ); // e.g., foo.bar.ClassA -> f.b.ClassA
        this.proxyAuthCache = cacheProducer.getCache( HTTP_PROXY_AUTH_CACHE, String.class, Boolean.class );
    }

    @Override
    public void handleEvent( final ConduitStreamSinkChannel channel )
    {
        if ( metricsConfig == null || metricRegistry == null )
        {
            doHandleEvent( channel );
            return;
        }

        Timer timer = metricRegistry.timer( name( metricsConfig.getNodePrefix(), cls, "handleEvent" ) );
        Timer.Context timerContext = timer.time();
        try
        {
            doHandleEvent( channel );
        }
        finally
        {
            timerContext.stop();
        }
    }

    private void doHandleEvent( final ConduitStreamSinkChannel channel )
    {
        HttpConduitWrapper http = new HttpConduitWrapper( channel, httpRequest, contentController, cacheProvider );
        if ( httpRequest == null )
        {
            if ( error != null )
            {
                logger.debug( "handling error from request reader: " + error.getMessage(), error );
                handleError( error, http );
            }
            else
            {
                logger.debug( "invalid state (no error or request) from request reader. Sending 400." );
                try
                {
                    http.writeStatus( ApplicationStatus.BAD_REQUEST );
                }
                catch ( final IOException e )
                {
                    logger.error( "Failed to write BAD REQUEST for missing HTTP first-line to response channel.", e );
                }
            }

            return;
        }

        restLogger.info( "START {} (from: {})", httpRequest.getRequestLine(), peerAddress );

        // TODO: Can we handle this?
        final String oldThreadName = Thread.currentThread().getName();
        Thread.currentThread().setName( "PROXY-" + httpRequest.getRequestLine().toString() );
        channel.getCloseSetter().set( ( sinkChannel ) ->
        {
            restLogger.info( "END {} (from: {})", httpRequest.getRequestLine(), peerAddress );
            logger.debug("sink channel closing.");
            Thread.currentThread().setName( oldThreadName );
        } );

        logger.debug( "\n\n\n>>>>>>> Handle write\n\n\n\n\n" );
        if ( error == null )
        {
            try
            {
                if ( repoCreator == null )
                {
                    throw new IndyDataException( "No valid instance of ProxyRepositoryCreator" );
                }

                final UserPass proxyUserPass =
                                UserPass.parse( ApplicationHeader.proxy_authorization, httpRequest, null );

                mdcManager.putExtraHeaders( httpRequest );
                if ( proxyUserPass != null )
                {
                    mdcManager.putExternalID( proxyUserPass.getUser() );
                }

                logger.debug( "Proxy UserPass: {}\nConfig secured? {}\nConfig tracking type: {}", proxyUserPass,
                              config.isSecured(), config.getTrackingType() );
                if ( proxyUserPass == null && ( config.isSecured()
                                || TrackingType.ALWAYS == config.getTrackingType() ) )
                {

                    String realmInfo = String.format( PROXY_AUTHENTICATE_FORMAT, config.getProxyRealm() );

                    logger.info( "Not authenticated to proxy. Sending response: {} / {}: {}",
                                 PROXY_AUTHENTICATION_REQUIRED, proxy_authenticate, realmInfo );

                    http.writeStatus( PROXY_AUTHENTICATION_REQUIRED );
                    http.writeHeader( proxy_authenticate,
                                      realmInfo );
                }
                else
                {
                    RequestLine requestLine = httpRequest.getRequestLine();
                    String method = requestLine.getMethod().toUpperCase();
                    String trackingId = null;
                    boolean authenticated = true;

                    if ( proxyUserPass != null )
                    {
                        TrackingKey trackingKey = getTrackingKey( proxyUserPass );
                        if ( trackingKey != null )
                        {
                            trackingId = trackingKey.getId();
                        }

                        String authCacheKey = generateAuthCacheKey( proxyUserPass );
                        Boolean isAuthToken = proxyAuthCache.get( authCacheKey );
                        if ( Boolean.TRUE.equals( isAuthToken ) )
                        {
                            authenticated = true;
                            logger.debug("Found auth key in cache" );
                        }
                        else
                        {
                            logger.debug( "Passing BASIC authentication credentials to Keycloak bearer-token translation authenticator" );
                            authenticated = proxyAuthenticator.authenticate( proxyUserPass, http );
                            if ( authenticated )
                            {
                                proxyAuthCache.put( authCacheKey, Boolean.TRUE, DEFAULT_AUTH_CACHE_EXPIRATION_HOURS, TimeUnit.HOURS );
                            }
                        }
                        logger.debug( "Authentication done, result: {}", authenticated );
                    }

                    if ( authenticated )
                    {
                        final URL url = new URL( requestLine.getUri() );

                        switch ( method )
                        {
                            case GET_METHOD:
                            case HEAD_METHOD:
                            {
                                logger.debug( "getArtifactStore starts, trackingId: {}, url: {}", trackingId, url );
                                ArtifactStore store = getArtifactStore( trackingId, url );
                                logger.debug( "getArtifactStore done, store: {}", store );

                                transfer( http, store, url.getPath(), GET_METHOD.equals( method ), proxyUserPass );
                                break;
                            }
                            case OPTIONS_METHOD:
                            {
                                http.writeStatus( ApplicationStatus.OK );
                                http.writeHeader( ApplicationHeader.allow, ALLOW_HEADER_VALUE );
                                break;
                            }
                            default:
                            {
                                http.writeStatus( ApplicationStatus.METHOD_NOT_ALLOWED );
                            }
                        }
                    }
                }

                logger.debug( "Response complete." );
            }
            catch ( final Throwable e )
            {
                error = e;
            }
            finally
            {
                mdcManager.clear();
            }
        }

        if ( error != null )
        {
            handleError( error, http );
        }

        try
        {
            http.close();
        }
        catch ( final IOException e )
        {
            logger.error( "Failed to flush/shutdown response.", e );
        }
    }

    private String generateAuthCacheKey( UserPass proxyUserPass )
    {
        return sha256Hex( proxyUserPass.getUser() + ":" + proxyUserPass.getPassword() );
    }

    private void handleError( final Throwable error, final HttpWrapper http )
    {
        logger.error( "HTTProx request failed: " + error.getMessage(), error );
        try
        {
            if ( http.isOpen() )
            {
                if ( error instanceof IndyWorkflowException )
                {
                    http.writeStatus( ApplicationStatus.getStatus( ( (IndyWorkflowException) error ).getStatus() ) );
                }
                else
                {
                    http.writeStatus( ApplicationStatus.SERVER_ERROR );
                }

                http.writeError( error );

                logger.debug( "Response error complete." );
{'fraction_non_alphanumeric': 0.08222975053895903, 'fraction_numerical': 0.0017452006980802793, 'mean_word_length': 1.9200639488409272, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1571104', 'n_tokens_mistral': 7522, 'n_tokens_neox': 7108, 'n_words': 1552}