<commit_before>const konan = require("./getImports");
const path = require("path");
const fs = require("fs");
const builtInPackages = require("module").builtinModules || [];

// recursively generate list of (relative) files imported by given file
function getRelativeFiles(file) {
  const extension = path.extname(file);
  var deps = [];
  if (!fs.existsSync(file, "utf8")) return deps;

  // js based files
  if (
    extension === ".js" ||
    extension === ".jsx" ||
    extension === ".md" ||
    extension === ".mdx" ||
    extension === ".ts" ||
    extension === ".tsx" ||
    extension === ".vue"
  ) {
    var imports = konan(file, fs.readFileSync(file, "utf8"));
    // only strings for now.
    imports.strings.forEach(imp => {
      // skip package imports
      if (imp.startsWith(".")) {
        // some imports dont have extension. We got to handle those
        if (path.extname(imp)) {
          deps.push(path.join(path.dirname(file), imp));
        } else {
          var baseName = path.join(path.dirname(file), imp);
          if (fs.existsSync(baseName + ".js")) deps.push(baseName + ".js");
          else if (fs.existsSync(baseName + ".jsx"))
            deps.push(baseName + ".jsx");
          else if (fs.existsSync(baseName + ".json"))
            deps.push(baseName + ".json");
          else if (fs.existsSync(baseName + ".md")) deps.push(baseName + ".md");
          else if (fs.existsSync(baseName + ".mdx"))
            deps.push(baseName + ".mdx");
          else if (fs.existsSync(baseName + ".ts")) deps.push(baseName + ".ts");
          else if (fs.existsSync(baseName + ".tsx"))
            deps.push(baseName + ".tsx");
          else if (fs.existsSync(baseName + ".vue"))
            deps.push(baseName + ".vue");
        }
      }
    });
  }
  deps.forEach(dep => {
    deps = deps.concat(getRelativeFiles(dep));
  });
  return deps;
}

function getPackages(file) {
  var deps = [];
  const extension = path.extname(file);
  if (
    extension === ".js" ||
    extension === ".jsx" ||
    extension === ".md" ||
    extension === ".mdx" ||
    extension === ".ts" ||
    extension === ".tsx" ||
    extension === ".vue"
  ) {
    var imports = konan(file, fs.readFileSync(file, "utf8"));
    // only strings for now.
    imports.strings.forEach(imp => {
      imp = trimPackageName(imp);
      // skip relative imports and built-in imports (on newer node versions only)
      if (!imp.startsWith(".") && builtInPackages.indexOf(imp) === -1) {
        deps.push(imp);
      }
    });
  }

  return deps;
}

// trim submodule imports to only install main package (ie. 'bootstrap' for: import 'bootstrap/dist/css/bootstrap.min.css')<commit_msg>Code repetition has been removed in package "dep-tree-js". Made more readable.<commit_after>const konan = require("./getImports");
const path = require("path");
const fs = require("fs");
const builtInPackages = require("module").builtinModules || [];

const avaibleExtensions = [".js", ".jsx", ".md", ".mdx", ".ts", ".tsx", ".vue"];

// recursively generate list of (relative) files imported by given file
function getRelativeFiles(file) {
  const extension = path.extname(file);
  var deps = [];

  if (!fs.existsSync(file, "utf8")) {
    return deps;
  }

  // js based files
  if (avaibleExtensions.includes(extension)) {
    var imports = konan(file, fs.readFileSync(file, "utf8"));
    // only strings for now.
    imports.strings.forEach(imp => {
      // skip package imports
      if (imp.startsWith(".")) {
        // some imports dont have extension. We got to handle those
        if (path.extname(imp)) {
          deps.push(path.join(path.dirname(file), imp));
        } else {
          var baseName = path.join(path.dirname(file), imp);

          if (fs.existsSync(baseName + ".js"))
            deps.push(baseName + ".js");
          else if (fs.existsSync(baseName + ".jsx"))
            deps.push(baseName + ".jsx");
          else if (fs.existsSync(baseName + ".json"))
            deps.push(baseName + ".json");
          else if (fs.existsSync(baseName + ".md"))
            deps.push(baseName + ".md");
          else if (fs.existsSync(baseName + ".mdx"))
            deps.push(baseName + ".mdx");
          else if (fs.existsSync(baseName + ".ts"))
            deps.push(baseName + ".ts");
          else if (fs.existsSync(baseName + ".tsx"))
            deps.push(baseName + ".tsx");
          else if (fs.existsSync(baseName + ".vue"))
            deps.push(baseName + ".vue");
        }
      }
    });
  }
  deps.forEach(dep => {
    deps = deps.concat(getRelativeFiles(dep));
  });
  return deps;
}

function getPackages(file) {
  var deps = [];
  const extension = path.extname(file);
  if (avaibleExtensions.includes(extension)) {
    var imports = konan(file, fs.readFileSync(file, "utf8"));
    // only strings for now.
    imports.strings.forEach(imp => {
      imp = trimPackageName(imp);
      // skip relative imports and built-in imports (on newer node versions only)
      if (!imp.startsWith(".") && !builtInPackages.includes(imp)) {
        deps.push(imp);
      }
    });
  }

  return deps;
}

// trim submodule imports to only install main package (ie. 'bootstrap' for: import 'bootstrap/dist/css/bootstrap.min.css')
{'fraction_non_alphanumeric': 0.17169019458222054, 'fraction_numerical': 0.0013353681800839375, 'mean_word_length': 2.523521505376344, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3601694', 'n_tokens_mistral': 1643, 'n_tokens_neox': 1646, 'n_words': 397}