<commit_before>package marketplace.scheduledimport

import java.text.DateFormat
import java.text.SimpleDateFormat

import java.security.KeyStore

import javax.net.ssl.SSLContext

import javax.ws.rs.core.MediaType

import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.annotation.Propagation

import groovyx.net.http.URIBuilder

import org.codehaus.groovy.grails.web.json.JSONObject

import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.http.conn.ssl.SSLConnectionSocketFactory
import org.apache.http.conn.ssl.SSLContexts
import org.apache.http.conn.ssl.SSLContextBuilder
import org.apache.http.client.methods.HttpGet

import marketplace.ImportTask
import marketplace.Constants

import marketplace.rest.CustomDomainObjectReader

/**
 * This class manages the HTTP retrieval of scheduled import data.  It would
 * be labeled 'protected' but that breaks CGLIB
 */

//doesn't need a transaction
@Transactional(propagation=Propagation.SUPPORTS)
class ScheduledImportHttpService {

    CustomDomainObjectReader customDomainObjectReader
    def mp_RESTInterceptorService

    protected CloseableHttpClient createHttpsClient(ImportTask task) {
        KeyStore trustStore
        KeyStore keyStore
        char[] passwordArray = task.keystorePass?.toCharArray()

        if (task.truststorePath) {
            def trustStoreStream

            try {
                trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
                trustStoreStream = new FileInputStream(new File((String)task.truststorePath));
                trustStore.load(trustStoreStream, null);
            } finally {
                trustStoreStream?.close();
            }
        }

        if (task.keystorePath) {
            def keyStoreStream

            try {
                keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
                keyStoreStream = new FileInputStream(new File((String)task.keystorePath));
                keyStore.load(keyStoreStream, null);
            } finally {
                keyStoreStream?.close();
            }
        }

        SSLContextBuilder sslBuilder = SSLContexts.custom()

        if (trustStore) {
            sslBuilder.loadTrustMaterial(trustStore)
        }

        if (keyStore) {
            sslBuilder.loadKeyMaterial(keyStore, passwordArray)
        }

        SSLContext sslcontext = sslBuilder.build();

        return HttpClients.custom()
                .setSslcontext(sslcontext)
                .build();
    }

    /**
     * @return the URL to use to fetch import data for this ImportTask
     */
    protected URI getRemoteUri(ImportTask task) {
        URIBuilder uriBuilder = new URIBuilder(task.url)
        Date lastRunDate = task.lastSuccessfulRunResult?.runDate

        if (task.updateType == Constants.IMPORT_TYPE_DELTA &&
            lastRunDate != null) {

            DateFormat queryParamDateFormat =
                new SimpleDateFormat(Constants.EXTERNAL_DATE_FORMAT)

            queryParamDateFormat.setTimeZone(TimeZone.getTimeZone('UTC'))

            uriBuilder.addQueryParam(Constants.OMP_IMPORT_DELTA_DATE_FIELD,
                queryParamDateFormat.format(lastRunDate))
        }
<commit_msg>OP-6048: Making partial import from 7.16 work<commit_after>package marketplace.scheduledimport

import java.text.DateFormat
import java.text.SimpleDateFormat

import java.security.KeyStore

import javax.net.ssl.SSLContext

import javax.ws.rs.core.MediaType

import org.springframework.transaction.annotation.Transactional
import org.springframework.transaction.annotation.Propagation

import groovyx.net.http.URIBuilder

import org.codehaus.groovy.grails.web.json.JSONObject

import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.util.EntityUtils
import org.apache.http.conn.ssl.SSLConnectionSocketFactory
import org.apache.http.conn.ssl.SSLContexts
import org.apache.http.conn.ssl.SSLContextBuilder
import org.apache.http.client.methods.HttpGet

import marketplace.ImportTask
import marketplace.Constants

import marketplace.rest.CustomDomainObjectReader

/**
 * This class manages the HTTP retrieval of scheduled import data.  It would
 * be labeled 'protected' but that breaks CGLIB
 */

//doesn't need a transaction
@Transactional(propagation=Propagation.SUPPORTS)
class ScheduledImportHttpService {

    CustomDomainObjectReader customDomainObjectReader
    def mp_RESTInterceptorService

    protected CloseableHttpClient createHttpsClient(ImportTask task) {
        KeyStore trustStore
        KeyStore keyStore
        char[] passwordArray = task.keystorePass?.toCharArray()

        if (task.truststorePath) {
            def trustStoreStream

            try {
                trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
                trustStoreStream = new FileInputStream(new File((String)task.truststorePath));
                trustStore.load(trustStoreStream, null);
            } finally {
                trustStoreStream?.close();
            }
        }

        if (task.keystorePath) {
            def keyStoreStream

            try {
                keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
                keyStoreStream = new FileInputStream(new File((String)task.keystorePath));
                keyStore.load(keyStoreStream, null);
            } finally {
                keyStoreStream?.close();
            }
        }

        SSLContextBuilder sslBuilder = SSLContexts.custom()

        if (trustStore) {
            sslBuilder.loadTrustMaterial(trustStore)
        }

        if (keyStore) {
            sslBuilder.loadKeyMaterial(keyStore, passwordArray)
        }

        SSLContext sslcontext = sslBuilder.build();

        return HttpClients.custom()
                .setSslcontext(sslcontext)
                .build();
    }

    /**
     * @return the URL to use to fetch import data for this ImportTask
     */
    protected URI getRemoteUri(ImportTask task) {
        URIBuilder uriBuilder = new URIBuilder(task.url)
        Date lastRunDate = task.lastSuccessfulRunResult?.runDate

        if (task.updateType == Constants.IMPORT_TYPE_DELTA &&
            lastRunDate != null) {

            DateFormat queryParamDateFormat =
                new SimpleDateFormat(Constants.EXTERNAL_DATE_FORMAT)

            queryParamDateFormat.setTimeZone(TimeZone.getTimeZone('UTC'))

            String formattedDate = queryParamDateFormat.format(lastRunDate)

            uriBuilder.addQueryParam(Constants.OMP_IMPORT_DELTA_DATE_FIELD, formattedDate)

            //newer stores apparently use lastPull instead of editedSinceDate.  For compatibility
            //we include both in the URL
            uriBuilder.addQueryParam('lastPull', formattedDate)
        }

{'fraction_non_alphanumeric': 0.07695578231292517, 'fraction_numerical': 0.000992063492063492, 'mean_word_length': 2.806364617044229, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3319654', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1814, 'n_words': 357}