<commit_before>/**
 * Copyright (C) FuseSource, Inc.
 * http://fusesource.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.fusesource.fabric.agent.download;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ScheduledExecutorService;
import javax.xml.parsers.ParserConfigurationException;

import org.fusesource.fabric.agent.mvn.DownloadableArtifact;
import org.fusesource.fabric.agent.mvn.MavenConfiguration;
import org.fusesource.fabric.agent.mvn.MavenRepositoryURL;
import org.fusesource.fabric.agent.mvn.Parser;
import org.fusesource.fabric.agent.mvn.Version;
import org.fusesource.fabric.agent.mvn.VersionRange;
import org.fusesource.fabric.agent.utils.URLUtils;
import org.fusesource.fabric.agent.utils.XmlUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;

public class MavenDownloadTask extends AbstractDownloadTask implements Runnable {

    /**
     * Logger.
     */
    private static final Log LOG = new Log();
    /**
     * 2 spacess indent;
     */
    private static final String Ix2 = "  ";
    /**
     * 4 spacess indent;
     */
    private static final String Ix4 = "    ";

    private final MavenRepositoryURL system;
    private final MavenConfiguration configuration;

    public MavenDownloadTask(String url, MavenRepositoryURL system, MavenConfiguration configuration, ScheduledExecutorService executor) {
        super(url, executor);
        this.system = system;
        this.configuration = configuration;
    }

    protected File download() throws Exception {
        Parser parser = new Parser(url.substring("mvn:".length()));
        Set<DownloadableArtifact> downloadables;
        if (!parser.getVersion().contains("SNAPSHOT")) {
            downloadables = doCollectPossibleDownloads(parser, Collections.singletonList(system));
            if (!downloadables.isEmpty()) {
                DownloadableArtifact artifact = downloadables.iterator().next();
                URL url = artifact.getArtifactURL();
                File file = new File(URI.create(url.toString()));
                if (file.exists()) {
                    return file;
                }
            }
        }
        downloadables = collectPossibleDownloads(parser);
        if (LOG.isTraceEnabled()) {
            LOG.trace("Possible download locations for [" + url + "]");
            for (DownloadableArtifact artifact : downloadables) {
                LOG.trace("  " + artifact);
            }
        }
        for (DownloadableArtifact artifact : downloadables) {
            LOG.trace("Downloading [" + artifact + "]");
            try {
                configuration.enableProxy(artifact.getArtifactURL());

                String repository = new File(URI.create(system.getURL().toString())).getPath();
                if (!repository.endsWith(Parser.FILE_SEPARATOR)) {
                    repository = repository + Parser.FILE_SEPARATOR;
                }
                InputStream is = artifact.getInputStream();
                File file = new File(repository + parser.getArtifactPath());
                File tmp = new File(file.toString() + ".tmp");
                tmp.getParentFile().mkdirs();
                OutputStream os = new FileOutputStream(tmp);
                copy(is, os);
                file.delete();
                tmp.renameTo(file);
                return file;
            } catch (IOException ignore) {
                // go on with next repository
                LOG.debug(Ix2 + "Could not download [" + artifact + "]");
                LOG.trace(Ix2 + "Reason [" + ignore.getClass().getName() + ": " + ignore.getMessage() + "]");
            }
        }
        // no artifact found
        throw new IOException(
                "URL [" + url + "] could not be resolved."
        );
    }

    /**
     * Searches all available repositories for possible artifacts to download. The returned set of downloadable
     * artifacts (never null, but maybe empty) will be sorted descending by version of the artifact and by positon of
     * repository in the list of repositories to be searched.
     *
     * @return a non null sorted set of artifacts
     * @throws java.net.MalformedURLException re-thrown
     */
    private Set<DownloadableArtifact> collectPossibleDownloads(final Parser parser)
            throws MalformedURLException {
        final List<MavenRepositoryURL> repositories = new ArrayList<MavenRepositoryURL>();
        repositories.addAll(configuration.getRepositories());
        repositories.add(configuration.getLocalRepository());
        // if the url contains a prefered repository add that repository as the first repository to be searched
        if (parser.getRepositoryURL() != null) {
            repositories.add(
                    repositories.size() == 0 ? 0 : 1,
                    parser.getRepositoryURL()
            );
        }
        return doCollectPossibleDownloads(parser, repositories);
    }

    /**
     * Search the default repositories for possible artifacts to download.
     */
    private Set<DownloadableArtifact> collectDefaultPossibleDownloads(final Parser parser)
            throws MalformedURLException {
        return doCollectPossibleDownloads(parser, configuration.getDefaultRepositories());
    }

    private Set<DownloadableArtifact> doCollectPossibleDownloads(final Parser parser,
                                                                 final List<MavenRepositoryURL> repositories)
            throws MalformedURLException {<commit_msg>Allow usage of system repo for snapshots, but as a fallback<commit_after>/**
 * Copyright (C) FuseSource, Inc.
 * http://fusesource.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.fusesource.fabric.agent.download;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ScheduledExecutorService;
import javax.xml.parsers.ParserConfigurationException;

import org.fusesource.fabric.agent.mvn.DownloadableArtifact;
import org.fusesource.fabric.agent.mvn.MavenConfiguration;
import org.fusesource.fabric.agent.mvn.MavenRepositoryURL;
import org.fusesource.fabric.agent.mvn.Parser;
import org.fusesource.fabric.agent.mvn.Version;
import org.fusesource.fabric.agent.mvn.VersionRange;
import org.fusesource.fabric.agent.utils.URLUtils;
import org.fusesource.fabric.agent.utils.XmlUtils;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.xml.sax.SAXException;

public class MavenDownloadTask extends AbstractDownloadTask implements Runnable {

    /**
     * Logger.
     */
    private static final Log LOG = new Log();
    /**
     * 2 spacess indent;
     */
    private static final String Ix2 = "  ";
    /**
     * 4 spacess indent;
     */
    private static final String Ix4 = "    ";

    private final MavenRepositoryURL system;
    private final MavenConfiguration configuration;

    public MavenDownloadTask(String url, MavenRepositoryURL system, MavenConfiguration configuration, ScheduledExecutorService executor) {
        super(url, executor);
        this.system = system;
        this.configuration = configuration;
    }

    protected File download() throws Exception {
        Parser parser = new Parser(url.substring("mvn:".length()));
        Set<DownloadableArtifact> downloadables;
        if (!parser.getVersion().contains("SNAPSHOT")) {
            downloadables = doCollectPossibleDownloads(parser, Collections.singletonList(system));
            if (!downloadables.isEmpty()) {
                DownloadableArtifact artifact = downloadables.iterator().next();
                URL url = artifact.getArtifactURL();
                File file = new File(URI.create(url.toString()));
                if (file.exists()) {
                    return file;
                }
            }
        }
        downloadables = collectPossibleDownloads(parser);
        if (LOG.isTraceEnabled()) {
            LOG.trace("Possible download locations for [" + url + "]");
            for (DownloadableArtifact artifact : downloadables) {
                LOG.trace("  " + artifact);
            }
        }
        for (DownloadableArtifact artifact : downloadables) {
            LOG.trace("Downloading [" + artifact + "]");
            try {
                configuration.enableProxy(artifact.getArtifactURL());

                String repository = new File(URI.create(system.getURL().toString())).getPath();
                if (!repository.endsWith(Parser.FILE_SEPARATOR)) {
                    repository = repository + Parser.FILE_SEPARATOR;
                }
                InputStream is = artifact.getInputStream();
                File file = new File(repository + parser.getArtifactPath());
                File tmp = new File(file.toString() + ".tmp");
                tmp.getParentFile().mkdirs();
                OutputStream os = new FileOutputStream(tmp);
                copy(is, os);
                file.delete();
                tmp.renameTo(file);
                return file;
            } catch (IOException ignore) {
                // go on with next repository
                LOG.debug(Ix2 + "Could not download [" + artifact + "]");
                LOG.trace(Ix2 + "Reason [" + ignore.getClass().getName() + ": " + ignore.getMessage() + "]");
            }
        }
        // no artifact found
        throw new IOException(
                "URL [" + url + "] could not be resolved."
        );
    }

    /**
     * Searches all available repositories for possible artifacts to download. The returned set of downloadable
     * artifacts (never null, but maybe empty) will be sorted descending by version of the artifact and by positon of
     * repository in the list of repositories to be searched.
     *
     * @return a non null sorted set of artifacts
     * @throws java.net.MalformedURLException re-thrown
     */
    private Set<DownloadableArtifact> collectPossibleDownloads(final Parser parser)
            throws MalformedURLException {
        final List<MavenRepositoryURL> repositories = new ArrayList<MavenRepositoryURL>();
        repositories.addAll(configuration.getRepositories());
        repositories.add(system);
        repositories.add(configuration.getLocalRepository());
        // if the url contains a prefered repository add that repository as the first repository to be searched
        if (parser.getRepositoryURL() != null) {
            repositories.add(
                    repositories.size() == 0 ? 0 : 1,
                    parser.getRepositoryURL()
            );
        }
        return doCollectPossibleDownloads(parser, repositories);
    }

    /**
     * Search the default repositories for possible artifacts to download.
     */
    private Set<DownloadableArtifact> collectDefaultPossibleDownloads(final Parser parser)
            throws MalformedURLException {
        return doCollectPossibleDownloads(parser, configuration.getDefaultRepositories());
    }

    private Set<DownloadableArtifact> doCollectPossibleDownloads(final Parser parser,
                                                                 final List<MavenRepositoryURL> repositories)
            throws MalformedURLException {
{'fraction_non_alphanumeric': 0.0946134740763738, 'fraction_numerical': 0.0023284694194349582, 'mean_word_length': 2.550564893910168, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 70, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7665182', 'n_tokens_mistral': 3384, 'n_tokens_neox': 3198, 'n_words': 883}