<commit_before>/*
 * Copyright 2012 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.netflix.asgard

import com.netflix.asgard.format.JsonpStripper
import grails.converters.JSON
import grails.converters.XML
import groovy.util.slurpersupport.GPathResult
import java.util.concurrent.TimeUnit
import org.apache.http.HttpEntity
import org.apache.http.HttpHost
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.HttpDelete
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.client.methods.HttpPut
import org.apache.http.client.methods.HttpUriRequest
import org.apache.http.conn.params.ConnManagerPNames
import org.apache.http.conn.params.ConnRoutePNames
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager
import org.apache.http.message.BasicNameValuePair
import org.apache.http.params.HttpConnectionParams
import org.apache.http.util.EntityUtils
import org.springframework.beans.factory.InitializingBean

class RestClientService implements InitializingBean {

    static transactional = false

    def configService

    // Change to PoolingClientConnectionManager after upgrade to http-client 4.2.
    final ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager()
    final HttpClient httpClient = new DefaultHttpClient(connectionManager)

    // If the AWS Java SDK upgrades to httpclient 4.2.* then we can set up the client with retries like this.
    /*
    final PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager()
    final HttpClient baseClient = new DefaultHttpClient(connectionManager)
    final HttpClient httpClient = new AutoRetryHttpClient(baseClient, new DefaultServiceUnavailableRetryStrategy())
    */

    public void afterPropertiesSet() throws Exception {
        if (configService.proxyHost) {
            final HttpHost proxy = new HttpHost(configService.proxyHost, configService.proxyPort, 'http')
            httpClient.params.setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy)
        }
        // Switch to ClientPNames.CONN_MANAGER_TIMEOUT when upgrading http-client 4.2
        httpClient.params.setLongParameter(ConnManagerPNames.TIMEOUT, configService.httpConnPoolTimeout)
        connectionManager.maxTotal = configService.httpConnPoolMaxSize
        connectionManager.defaultMaxPerRoute = configService.httpConnPoolMaxForRoute
    }

    def getAsXml(String uri, Integer timeoutMillis = 10000, boolean swallowException = true) {
        try {
            String content = get(uri, 'application/xml; charset=UTF-8', timeoutMillis)
            return content ? XML.parse(content) as GPathResult : null
        } catch (Exception e) {
            log.error "GET from ${uri} failed: ${e}"
            if (swallowException) {
                return null
            } else {
                throw e
            }
        }
    }

    def getAsJson(String uri, Integer timeoutMillis = 10000) {<commit_msg>Use limited conventional retries as well<commit_after>/*
 * Copyright 2012 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.netflix.asgard

import com.netflix.asgard.format.JsonpStripper
import grails.converters.JSON
import grails.converters.XML
import groovy.util.slurpersupport.GPathResult
import java.security.Security
import java.util.concurrent.TimeUnit
import org.apache.http.HttpEntity
import org.apache.http.HttpHost
import org.apache.http.HttpResponse
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.HttpDelete
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost
import org.apache.http.client.methods.HttpPut
import org.apache.http.client.methods.HttpUriRequest
import org.apache.http.conn.params.ConnManagerPNames
import org.apache.http.conn.params.ConnRoutePNames
import org.apache.http.entity.StringEntity
import org.apache.http.impl.client.DefaultHttpClient
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager
import org.apache.http.message.BasicNameValuePair
import org.apache.http.params.HttpConnectionParams
import org.apache.http.util.EntityUtils
import org.springframework.beans.factory.InitializingBean
import sun.net.InetAddressCachePolicy

class RestClientService implements InitializingBean {

    static transactional = false

    def configService

    // Change to PoolingClientConnectionManager after upgrade to http-client 4.2.
    final ThreadSafeClientConnManager connectionManager = new ThreadSafeClientConnManager()
    final DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager)


    public void afterPropertiesSet() throws Exception {
        if (configService.proxyHost) {
            final HttpHost proxy = new HttpHost(configService.proxyHost, configService.proxyPort, 'http')
            httpClient.params.setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy)
        }
        // Switch to ClientPNames.CONN_MANAGER_TIMEOUT when upgrading http-client 4.2
        httpClient.params.setLongParameter(ConnManagerPNames.TIMEOUT, configService.httpConnPoolTimeout)

        // This retry handler only retries in a few specific failure cases, but it's better than nothing.
        httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(3, true))
        // If the AWS Java SDK upgrades to httpclient 4.2.* then here's the new way to set up the client with retries.
        /*
        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager()
        HttpClient baseClient = new DefaultHttpClient(connectionManager)
        HttpClient httpClient = new AutoRetryHttpClient(baseClient, new DefaultServiceUnavailableRetryStrategy())
        */

        avoidLongCachingOfDnsResults()
        connectionManager.maxTotal = configService.httpConnPoolMaxSize
        connectionManager.defaultMaxPerRoute = configService.httpConnPoolMaxForRoute
    }

    /**
     * We often operate in an environment where we expect resolution of DNS names for remote dependencies to change
     * frequently, so it's best to tell the JVM to avoid caching DNS results internally.
     */
    private avoidLongCachingOfDnsResults() {
        //noinspection GroovyAccessibility
        InetAddressCachePolicy.cachePolicy = InetAddressCachePolicy.NEVER // Groovy doesn't care about privates
        Security.setProperty('networkaddress.cache.ttl', '0')
    }

    def getAsXml(String uri, Integer timeoutMillis = 10000, boolean swallowException = true) {
        try {
            String content = get(uri, 'application/xml; charset=UTF-8', timeoutMillis)
            return content ? XML.parse(content) as GPathResult : null
        } catch (Exception e) {
            log.error "GET from ${uri} failed: ${e}"
            if (swallowException) {
                return null
            } else {
                throw e
            }
        }
    }

    def getAsJson(String uri, Integer timeoutMillis = 10000) {
{'fraction_non_alphanumeric': 0.07202982202982203, 'fraction_numerical': 0.006253006253006253, 'mean_word_length': 3.821449275362319, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5395175', 'n_tokens_mistral': 2201, 'n_tokens_neox': 2140, 'n_words': 612}