<commit_before>package net.sf.mmm.code.impl.java.source.maven;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import net.sf.mmm.code.java.maven.api.DependencyHelper;
import net.sf.mmm.code.java.maven.api.MavenBridge;
import net.sf.mmm.code.java.maven.api.MavenConstants;
import net.sf.mmm.code.java.maven.api.ModelHelper;
import net.sf.mmm.code.java.maven.impl.MavenBridgeImpl;

import org.apache.maven.model.Dependency;
import org.apache.maven.model.Model;
import org.apache.maven.model.Parent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Class to collect {@link Dependency dependencies} of a maven project.
 *
 * @since 1.0.0
 * @see #collect(Model)
 * @see #asUrls()
 * @see #asClassLoader()
 */
public class MavenDependencyCollector {

  private static final Logger LOG = LoggerFactory.getLogger(MavenDependencyCollector.class);

  final MavenBridge mavenBridge;

  private final List<URL> dependencyList;

  private final Set<URL> dependencySet;

  private final Map<String, Model> gav2ProjectMap;

  private final Map<String, Model> dir2ProjectMap;

  private final boolean buildReactor;

  private final boolean includeTestDependencies;

  private final String altBuildDir;

  /**
   * The constructor.
   *
   * @param includeTestDependencies - {@code true} to include test-dependencies, {@code false} otherwise.
   * @param buildReactor - {@code true} to build the maven project reactor by traversing all parents and modules to
   *        resolve dependencies to sibling modules (slower but more accurate), {@code false} otherwise.
   * @param altBuildDir the alternative build directory (e.g. "eclipse-target").
   */
  public MavenDependencyCollector(boolean includeTestDependencies, boolean buildReactor, String altBuildDir) {

    this(MavenBridgeImpl.getDefault(), includeTestDependencies, buildReactor, altBuildDir);
  }

  /**
   * The constructor.
   *
   * @param mavenBridge the {@link MavenBridge} instance to use.
   * @param includeTestDependencies - {@code true} to include test-dependencies, {@code false} otherwise.
   * @param buildReactor - {@code true} to build the maven project reactor by traversing all parents and modules to
   *        resolve dependencies to sibling modules (slower but more accurate), {@code false} otherwise.
   * @param altBuildDir the alternative build directory (e.g. "eclipse-target").
   */
  public MavenDependencyCollector(MavenBridge mavenBridge, boolean includeTestDependencies, boolean buildReactor,
      String altBuildDir) {

    super();
    this.mavenBridge = mavenBridge;
    this.dependencyList = new ArrayList<>();
    this.dependencySet = new HashSet<>();
    this.buildReactor = buildReactor;
    if (buildReactor) {
      this.gav2ProjectMap = new HashMap<>();
      this.dir2ProjectMap = new HashMap<>();
    } else {
      this.gav2ProjectMap = null;
      this.dir2ProjectMap = null;
    }
    this.includeTestDependencies = includeTestDependencies;
    this.altBuildDir = altBuildDir;
  }

  private boolean add(File dependencyFile, String source) {

    if (!dependencyFile.exists()) {
      return false;
    }
    try {
      return add(dependencyFile.toURI().toURL(), source);
    } catch (MalformedURLException e) {
      throw new IllegalStateException("Failed to convert file to URL: " + dependencyFile, e);
    }
  }

  private boolean add(URL dependencyUrl, String source) {

    boolean added = this.dependencySet.add(dependencyUrl);
    if (added) {
      if (source == null) {
        LOG.debug("Adding dependency {}", dependencyUrl);
      } else {
        LOG.debug("Adding dependency {} for {}", dependencyUrl, source);
      }
      this.dependencyList.add(dependencyUrl);
    }
    return added;<commit_msg>#29: improved logging to trace problems<commit_after>package net.sf.mmm.code.impl.java.source.maven;

import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import net.sf.mmm.code.java.maven.api.DependencyHelper;
import net.sf.mmm.code.java.maven.api.MavenBridge;
import net.sf.mmm.code.java.maven.api.MavenConstants;
import net.sf.mmm.code.java.maven.api.ModelHelper;
import net.sf.mmm.code.java.maven.impl.MavenBridgeImpl;

import org.apache.maven.model.Dependency;
import org.apache.maven.model.Model;
import org.apache.maven.model.Parent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Class to collect {@link Dependency dependencies} of a maven project.
 *
 * @since 1.0.0
 * @see #collect(Model)
 * @see #asUrls()
 * @see #asClassLoader()
 */
public class MavenDependencyCollector {

  private static final Logger LOG = LoggerFactory.getLogger(MavenDependencyCollector.class);

  final MavenBridge mavenBridge;

  private final List<URL> dependencyList;

  private final Set<URL> dependencySet;

  private final Map<String, Model> gav2ProjectMap;

  private final Map<String, Model> dir2ProjectMap;

  private final boolean buildReactor;

  private final boolean includeTestDependencies;

  private final String altBuildDir;

  /**
   * The constructor.
   *
   * @param includeTestDependencies - {@code true} to include test-dependencies, {@code false} otherwise.
   * @param buildReactor - {@code true} to build the maven project reactor by traversing all parents and modules to
   *        resolve dependencies to sibling modules (slower but more accurate), {@code false} otherwise.
   * @param altBuildDir the alternative build directory (e.g. "eclipse-target").
   */
  public MavenDependencyCollector(boolean includeTestDependencies, boolean buildReactor, String altBuildDir) {

    this(MavenBridgeImpl.getDefault(), includeTestDependencies, buildReactor, altBuildDir);
  }

  /**
   * The constructor.
   *
   * @param mavenBridge the {@link MavenBridge} instance to use.
   * @param includeTestDependencies - {@code true} to include test-dependencies, {@code false} otherwise.
   * @param buildReactor - {@code true} to build the maven project reactor by traversing all parents and modules to
   *        resolve dependencies to sibling modules (slower but more accurate), {@code false} otherwise.
   * @param altBuildDir the alternative build directory (e.g. "eclipse-target").
   */
  public MavenDependencyCollector(MavenBridge mavenBridge, boolean includeTestDependencies, boolean buildReactor,
      String altBuildDir) {

    super();
    this.mavenBridge = mavenBridge;
    this.dependencyList = new ArrayList<>();
    this.dependencySet = new HashSet<>();
    this.buildReactor = buildReactor;
    if (buildReactor) {
      this.gav2ProjectMap = new HashMap<>();
      this.dir2ProjectMap = new HashMap<>();
    } else {
      this.gav2ProjectMap = null;
      this.dir2ProjectMap = null;
    }
    this.includeTestDependencies = includeTestDependencies;
    this.altBuildDir = altBuildDir;
  }

  private boolean add(File dependencyFile, String source) {

    if (!dependencyFile.exists()) {
      LOG.debug("Omitting dependency due to missing file {} for source {}", dependencyFile, source);
      return false;
    }
    try {
      return add(dependencyFile.toURI().toURL(), source);
    } catch (MalformedURLException e) {
      throw new IllegalStateException("Failed to convert file to URL: " + dependencyFile, e);
    }
  }

  private boolean add(URL dependencyUrl, String source) {

    boolean added = this.dependencySet.add(dependencyUrl);
    if (added) {
      if (source == null) {
        LOG.debug("Adding dependency {}", dependencyUrl);
      } else {
        LOG.debug("Adding dependency {} for {}", dependencyUrl, source);
      }
      this.dependencyList.add(dependencyUrl);
    } else {
      LOG.trace("Omitting duplicate dependency {} that was already been collected for source {}.", dependencyUrl,
          source);
    }
    return added;
{'fraction_non_alphanumeric': 0.10548937737934422, 'fraction_numerical': 0.0029473167137418644, 'mean_word_length': 3.894230769230769, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1798318', 'n_tokens_mistral': 2428, 'n_tokens_neox': 2305, 'n_words': 645}