<commit_before>
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Set;
import java.util.HashSet;

public class ArtifactDownloader
{
    public static final String SNAPSHOT_SIGNATURE = "-SNAPSHOT";

    private File mavenRepoLocal;

    private List remoteRepos;

    private boolean useTimestamp = true;

    private boolean ignoreErrors = false;

    private String proxyHost;

    private String proxyPort;

    private String proxyUserName;

    private String proxyPassword;

    public ArtifactDownloader( String localRepository, List remoteRepositories )
        throws Exception
    {
        setRemoteRepos( remoteRepositories );

        if ( localRepository == null )
        {
            System.err.println( "local repository not specified" );

            System.exit( 1 );
        }

        mavenRepoLocal = new File( localRepository );

        if ( !mavenRepoLocal.exists() )
        {
            if ( !mavenRepoLocal.mkdirs() )
            {
                System.err.println( "Cannot create the specified local repository: " + mavenRepoLocal );

                System.exit( 1 );
            }
        }

        if ( !mavenRepoLocal.canWrite() )
        {
            System.err.println( "Can't write to " + mavenRepoLocal.getAbsolutePath() );

            System.exit( 1 );
        }

        System.out.println( "Using the following for your local repository: " + mavenRepoLocal );
        System.out.println( "Using the following for your remote repositories: " + remoteRepos );
    }

    public File getMavenRepoLocal()
    {
        return mavenRepoLocal;
    }

    private Set downloadedArtifacts = new HashSet();

    public void downloadDependencies( List files )
        throws Exception
    {
        for ( Iterator j = files.iterator(); j.hasNext(); )
        {
            String file = (String) j.next();

            if ( !downloadedArtifacts.contains( file ) )
            {
                File destinationFile = new File( mavenRepoLocal, file );
                // The directory structure for this project may
                // not exists so create it if missing.
                File directory = destinationFile.getParentFile();

                if ( directory.exists() == false )
                {
                    directory.mkdirs();
                }

                if ( destinationFile.exists() && file.indexOf( SNAPSHOT_SIGNATURE ) < 0 )
                {
                    continue;
                }

                getRemoteArtifact( file, destinationFile );

                if ( !destinationFile.exists() )
                {
                    throw new Exception( "Failed to download " + file );
                }

                downloadedArtifacts.add( file );
            }
        }
    }

    private void setRemoteRepos( List repositories )
    {
        remoteRepos = new ArrayList();

        if ( repositories != null )
        {
            remoteRepos.addAll( repositories );
        }

        if ( repositories.isEmpty() )
        {
            remoteRepos.add( "http://www.ibiblio.org/maven/" );
        }
    }

    private List getRemoteRepos()
    {
        return remoteRepos;
    }

    private boolean getRemoteArtifact( String file, File destinationFile )
    {
        boolean fileFound = false;

        for ( Iterator i = getRemoteRepos().iterator(); i.hasNext(); )
        {
            String remoteRepo = (String) i.next();

            // The username and password parameters are not being
            // used here. Those are the "" parameters you see below.
            String url = remoteRepo + "/" + file;

            if ( !url.startsWith( "file" ) )
            {
                url = replace( url, "//", "/" );
                if ( url.startsWith( "https" ) )
                {
                    url = replace( url, "https:/", "https://" );
                }
                else
                {
                    url = replace( url, "http:/", "http://" );<commit_msg>o pointing the download at the first dispatcher<commit_after>
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;
import java.util.Set;
import java.util.HashSet;

public class ArtifactDownloader
{
    public static final String SNAPSHOT_SIGNATURE = "-SNAPSHOT";

    private File mavenRepoLocal;

    private List remoteRepos;

    private boolean useTimestamp = true;

    private boolean ignoreErrors = false;

    private String proxyHost;

    private String proxyPort;

    private String proxyUserName;

    private String proxyPassword;

    public ArtifactDownloader( String localRepository, List remoteRepositories )
        throws Exception
    {
        setRemoteRepos( remoteRepositories );

        if ( localRepository == null )
        {
            System.err.println( "local repository not specified" );

            System.exit( 1 );
        }

        mavenRepoLocal = new File( localRepository );

        if ( !mavenRepoLocal.exists() )
        {
            if ( !mavenRepoLocal.mkdirs() )
            {
                System.err.println( "Cannot create the specified local repository: " + mavenRepoLocal );

                System.exit( 1 );
            }
        }

        if ( !mavenRepoLocal.canWrite() )
        {
            System.err.println( "Can't write to " + mavenRepoLocal.getAbsolutePath() );

            System.exit( 1 );
        }

        System.out.println( "Using the following for your local repository: " + mavenRepoLocal );
        System.out.println( "Using the following for your remote repositories: " + remoteRepos );
    }

    public File getMavenRepoLocal()
    {
        return mavenRepoLocal;
    }

    private Set downloadedArtifacts = new HashSet();

    public void downloadDependencies( List files )
        throws Exception
    {
        for ( Iterator j = files.iterator(); j.hasNext(); )
        {
            String file = (String) j.next();

            if ( !downloadedArtifacts.contains( file ) )
            {
                File destinationFile = new File( mavenRepoLocal, file );
                // The directory structure for this project may
                // not exists so create it if missing.
                File directory = destinationFile.getParentFile();

                if ( directory.exists() == false )
                {
                    directory.mkdirs();
                }

                if ( destinationFile.exists() && file.indexOf( SNAPSHOT_SIGNATURE ) < 0 )
                {
                    continue;
                }

                getRemoteArtifact( file, destinationFile );

                if ( !destinationFile.exists() )
                {
                    throw new Exception( "Failed to download " + file );
                }

                downloadedArtifacts.add( file );
            }
        }
    }

    private void setRemoteRepos( List repositories )
    {
        remoteRepos = new ArrayList();

        if ( repositories != null )
        {
            remoteRepos.addAll( repositories );
        }

        if ( repositories.isEmpty() )
        {
            remoteRepos.add( "http://repo1.maven.org" );
        }
    }

    private List getRemoteRepos()
    {
        return remoteRepos;
    }

    private boolean getRemoteArtifact( String file, File destinationFile )
    {
        boolean fileFound = false;

        for ( Iterator i = getRemoteRepos().iterator(); i.hasNext(); )
        {
            String remoteRepo = (String) i.next();

            // The username and password parameters are not being
            // used here. Those are the "" parameters you see below.
            String url = remoteRepo + "/" + file;

            if ( !url.startsWith( "file" ) )
            {
                url = replace( url, "//", "/" );
                if ( url.startsWith( "https" ) )
                {
                    url = replace( url, "https:/", "https://" );
                }
                else
                {
                    url = replace( url, "http:/", "http://" );
{'fraction_non_alphanumeric': 0.09581985866570847, 'fraction_numerical': 0.0010779734099892202, 'mean_word_length': 1.6424050632911393, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5136042', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2097, 'n_words': 567}