<commit_before>package com.terradue.jcatalogue.client;

/*
 *    Copyright 2011 Terradue srl
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.commons.beanutils.ConvertUtils.register;
import static org.apache.commons.digester3.binder.DigesterLoader.newLoader;

import java.io.File;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Locale;

import org.apache.commons.digester3.binder.DigesterLoader;

import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.AsyncHttpClient;
import com.ning.http.client.AsyncHttpClientConfig;
import com.ning.http.client.Response;
import com.ning.http.client.extra.ResumableRandomAccessFileListener;
import com.ning.http.client.resumable.ResumableAsyncHandler;
import com.ning.http.client.resumable.ResumableIOExceptionFilter;
import com.terradue.jcatalogue.client.converters.AtomDateConverter;
import com.terradue.jcatalogue.client.converters.CharsetConverter;
import com.terradue.jcatalogue.client.converters.LocaleConverter;
import com.terradue.jcatalogue.client.digester.AtomRulesModule;
import com.terradue.jcatalogue.client.digester.DataSetRulesModule;
import com.terradue.jcatalogue.client.digester.LinkedAtomEntityModule;
import com.terradue.jcatalogue.client.digester.OpenSearchModule;

public final class CatalogueClient
{

    private static final String RW = "rw";

    static
    {
        register( new AtomDateConverter(), Date.class );
        register( new LocaleConverter(), Locale.class );
        register( new CharsetConverter(), Charset.class );
    }

    private final DigesterLoader descriptionDigesterLoader;

    private final DigesterLoader catalogueDigesterLoader;

    private final DigesterLoader serieDigesterLoader;

    private final AsyncHttpClient httpClient;

    public CatalogueClient()
    {
        descriptionDigesterLoader = newLoader( new OpenSearchModule() );
        catalogueDigesterLoader = newLoader( new AtomRulesModule( Catalogue.class ), new LinkedAtomEntityModule() );
        serieDigesterLoader = newLoader( new AtomRulesModule( Serie.class ), new DataSetRulesModule() );
        httpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder()
                            .setAllowPoolingConnection( true )
                            .addIOExceptionFilter( new ResumableIOExceptionFilter() )
                            .setMaximumConnectionsPerHost( 10 )
                            .setMaximumConnectionsTotal( 100 )
                            .setFollowRedirects( true )
                            .build() );
    }

    // Description methods

    public CatalogueDescription discover( String uri, Parameter... parameters )
    {
        return invoke( descriptionDigesterLoader, uri, parameters );
    }

    public CatalogueDescription discover( URI uri, Parameter...parameters )
    {
        return invoke( descriptionDigesterLoader, uri, parameters );
    }

    // Catalogue methods

    public Catalogue getCatalogue( String uri, Parameter... parameters )
    {
        return invoke( catalogueDigesterLoader, uri, parameters );
    }<commit_msg>implemented the downloader registration policy<commit_after>package com.terradue.jcatalogue.client;

/*
 *    Copyright 2011 Terradue srl
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_OK;
import static java.util.ServiceLoader.load;
import static org.apache.commons.beanutils.ConvertUtils.register;
import static org.apache.commons.digester3.binder.DigesterLoader.newLoader;

import java.io.File;
import java.io.RandomAccessFile;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
import java.util.ServiceLoader;

import org.apache.commons.digester3.binder.DigesterLoader;

import com.ning.http.client.AsyncCompletionHandler;
import com.ning.http.client.AsyncHttpClient;
import com.ning.http.client.AsyncHttpClientConfig;
import com.ning.http.client.Response;
import com.ning.http.client.extra.ResumableRandomAccessFileListener;
import com.ning.http.client.resumable.ResumableAsyncHandler;
import com.ning.http.client.resumable.ResumableIOExceptionFilter;
import com.terradue.jcatalogue.client.converters.AtomDateConverter;
import com.terradue.jcatalogue.client.converters.CharsetConverter;
import com.terradue.jcatalogue.client.converters.LocaleConverter;
import com.terradue.jcatalogue.client.digester.AtomRulesModule;
import com.terradue.jcatalogue.client.digester.DataSetRulesModule;
import com.terradue.jcatalogue.client.digester.LinkedAtomEntityModule;
import com.terradue.jcatalogue.client.digester.OpenSearchModule;
import com.terradue.jcatalogue.client.download.Downloader;
import com.terradue.jcatalogue.client.download.Protocol;

public final class CatalogueClient
{

    private static final String RW = "rw";

    static
    {
        register( new AtomDateConverter(), Date.class );
        register( new LocaleConverter(), Locale.class );
        register( new CharsetConverter(), Charset.class );
    }

    private final Map<String, Downloader> downloaders = new HashMap<String, Downloader>();

    private final DigesterLoader descriptionDigesterLoader;

    private final DigesterLoader catalogueDigesterLoader;

    private final DigesterLoader serieDigesterLoader;

    private final AsyncHttpClient httpClient;

    public CatalogueClient()
    {
        descriptionDigesterLoader = newLoader( new OpenSearchModule() );
        catalogueDigesterLoader = newLoader( new AtomRulesModule( Catalogue.class ), new LinkedAtomEntityModule() );
        serieDigesterLoader = newLoader( new AtomRulesModule( Serie.class ), new DataSetRulesModule() );
        httpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder()
                            .setAllowPoolingConnection( true )
                            .addIOExceptionFilter( new ResumableIOExceptionFilter() )
                            .setMaximumConnectionsPerHost( 10 )
                            .setMaximumConnectionsTotal( 100 )
                            .setFollowRedirects( true )
                            .build() );

        ServiceLoader<Downloader> downloaderServiceLoader = load( Downloader.class );
        Iterator<Downloader> downloaderIterator = downloaderServiceLoader.iterator();

        while ( downloaderIterator.hasNext() )
        {
            Downloader downloader = downloaderIterator.next();

            if ( !downloader.getClass().isAnnotationPresent( Protocol.class ) )
            {
                throw new RuntimeException( format( "Class %s must be annotated with %s",
                                                    downloader.getClass().getName(),
                                                    Protocol.class.getName() ) );
            }

            for ( String protocol : downloader.getClass().getAnnotation( Protocol.class ).value() )
            {
                downloaders.put( protocol, downloader );
            }
        }
    }

    // Description methods

    public CatalogueDescription discover( String uri, Parameter... parameters )
    {
        return invoke( descriptionDigesterLoader, uri, parameters );
    }

    public CatalogueDescription discover( URI uri, Parameter...parameters )
    {
        return invoke( descriptionDigesterLoader, uri, parameters );
    }

    // Catalogue methods

    public Catalogue getCatalogue( String uri, Parameter... parameters )
    {
        return invoke( catalogueDigesterLoader, uri, parameters );
    }
{'fraction_non_alphanumeric': 0.0822884012539185, 'fraction_numerical': 0.0033587102552619795, 'mean_word_length': 3.2598950882212683, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '678467', 'n_tokens_mistral': 2348, 'n_tokens_neox': 2318, 'n_words': 572}