<commit_before>/*
 * Bach - Library Support Tool
 *
 * Copyright (C) 2019 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// default package

import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.module.ModuleDescriptor;
import java.lang.module.ModuleFinder;
import java.lang.module.ModuleReference;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.lang.model.SourceVersion;

/**
 * Library Support Tool.
 *
 * <p>{@code java Lib.java [org.junit.jupiter[@5.5.1],...]}
 */
public class Lib {

  public static void main(String... args) {
    var path = Path.of(System.getProperty("lib", "lib"));
    var src = Path.of(System.getProperty("src", "src"));
    var projectModuleNames = findAllDirectoryNamesIn(src);

    var lib = new Lib(path, projectModuleNames);
    lib.list();

    var synthetic = ModuleDescriptor.newModule("$", Set.of(ModuleDescriptor.Modifier.SYNTHETIC));
    if (args.length == 0) {
      var requiresPattern =
          Pattern.compile(
              "(?:requires)" // key word
                  + "(?:\\s+[\\w.]+)?" // optional modifiers
                  + "\\s+([\\w.]+)" // module name
                  + "(?:\\s*/\\*\\s*([\\w.\\-+]+)\\s*\\*/\\s*)?" // optional '/*' version '*/'
                  + ";"); // end marker
      for (var name : projectModuleNames) {
        var root = src.resolve(name);
        try (var stream = Files.find(root, 9, (p, __) -> isModuleInfo(p))) {
          for (var moduleInfo : stream.collect(Collectors.toSet())) {
            var source = Files.readString(moduleInfo);
            var requiresMatcher = requiresPattern.matcher(source);
            while (requiresMatcher.find()) {
              var requiredName = requiresMatcher.group(1);
              Optional.ofNullable(requiresMatcher.group(2))
                  .ifPresentOrElse(
                      version ->
                          synthetic.requires(
                              Set.of(), requiredName, ModuleDescriptor.Version.parse(version)),
                      () -> synthetic.requires(requiredName));
            }
          }
        } catch (IOException e) {
          throw new UncheckedIOException("reading module-info.java files failed: " + root, e);
        }
      }
    }
    for (var arg : args) {
      if (arg.indexOf('@') >= 0) {
        var split = arg.split("@");
        synthetic.requires(Set.of(), split[0], ModuleDescriptor.Version.parse(split[1]));
        continue;
      }
      synthetic.requires(Set.of(), arg);
    }

    lib.load(synthetic.build());
    lib.sync();
  }

  private static Set<String> findAllDirectoryNamesIn(Path root) {
    if (!Files.isDirectory(root)) {
      return Set.of();
    }
    try (var entries = Files.newDirectoryStream(root, Files::isDirectory)) {
      return StreamSupport.stream(entries.spliterator(), false)
          .map(Path::getFileName)
          .map(Path::toString)
          .collect(Collectors.toCollection(TreeSet::new));
    } catch (IOException e) {
      throw new UncheckedIOException("reading directory failed: " + root, e);
    }
  }

  private static boolean isModuleInfo(Path path) {
    return Files.isRegularFile(path) && path.getFileName().toString().equals("module-info.java");<commit_msg>Extract logic from main method<commit_after>/*
 * Bach - Library Support Tool
 *
 * Copyright (C) 2019 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// default package

import java.io.IOException;
import java.io.UncheckedIOException;
import java.lang.module.ModuleDescriptor;
import java.lang.module.ModuleFinder;
import java.lang.module.ModuleReference;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.FileTime;
import java.text.SimpleDateFormat;
import java.util.Locale;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.lang.model.SourceVersion;

/**
 * Library Support Tool.
 *
 * <p>{@code java Lib.java [org.junit.jupiter[@5.5.1],...]}
 */
public class Lib {

  public static void main(String... args) {
    var path = Path.of(System.getProperty("lib", "lib"));
    var src = Path.of(System.getProperty("src", "src"));
    var projectModuleNames = findAllDirectoryNamesIn(src);
    var synthetic = synthesize(src, projectModuleNames, args);
    var lib = new Lib(path, projectModuleNames);
    lib.load(synthetic);
    lib.sync();
    lib.list();
  }

  private static Set<String> findAllDirectoryNamesIn(Path root) {
    if (!Files.isDirectory(root)) {
      return Set.of();
    }
    try (var entries = Files.newDirectoryStream(root, Files::isDirectory)) {
      return StreamSupport.stream(entries.spliterator(), false)
          .map(Path::getFileName)
          .map(Path::toString)
          .collect(Collectors.toCollection(TreeSet::new));
    } catch (IOException e) {
      throw new UncheckedIOException("reading directory failed: " + root, e);
    }
  }

  private static ModuleDescriptor synthesize(Path src, Set<String> modules, String... args) {
    var synthetic = ModuleDescriptor.newModule("$", Set.of(ModuleDescriptor.Modifier.SYNTHETIC));
    if (args.length == 0) {
      var requiresPattern =
          Pattern.compile(
              "(?:requires)" // key word
                  + "(?:\\s+[\\w.]+)?" // optional modifiers
                  + "\\s+([\\w.]+)" // module name
                  + "(?:\\s*/\\*\\s*([\\w.\\-+]+)\\s*\\*/\\s*)?" // optional '/*' version '*/'
                  + ";"); // end marker
      for (var module : modules) {
        var root = src.resolve(module);
        try (var stream = Files.find(root, 9, (p, __) -> isModuleInfo(p))) {
          for (var moduleInfo : stream.collect(Collectors.toSet())) {
            var source = Files.readString(moduleInfo);
            var requiresMatcher = requiresPattern.matcher(source);
            while (requiresMatcher.find()) {
              var requiredName = requiresMatcher.group(1);
              Optional.ofNullable(requiresMatcher.group(2))
                  .ifPresentOrElse(
                      version ->
                          synthetic.requires(
                              Set.of(), requiredName, ModuleDescriptor.Version.parse(version)),
                      () -> synthetic.requires(requiredName));
            }
          }
        } catch (IOException e) {
          throw new UncheckedIOException("reading module-info.java files failed: " + root, e);
        }
      }
    }
    for (var arg : args) {
      if (arg.indexOf('@') >= 0) {
        var split = arg.split("@");
        synthetic.requires(Set.of(), split[0], ModuleDescriptor.Version.parse(split[1]));
        continue;
      }
      synthetic.requires(Set.of(), arg);
    }
    return synthetic.build();
  }

  private static boolean isModuleInfo(Path path) {
    return Files.isRegularFile(path) && path.getFileName().toString().equals("module-info.java");
{'fraction_non_alphanumeric': 0.13827496316445653, 'fraction_numerical': 0.004080244814688882, 'mean_word_length': 2.7405680373039423, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4180729', 'n_tokens_mistral': 2522, 'n_tokens_neox': 2464, 'n_words': 617}