<commit_before>import * as ts from "typescript";

import { ImmutableDirectory, ImmutableFile, io } from "nofiles";

export type Registry = {
  indexes: { [key: string]: number };
  paths: string[];
  deps: number[][];
  size: number;
};

type FolderDeps = { [key: string]: FolderDeps | FileDeps };

class FileDeps {
  imports: string[] = [];
}

export default function analyze(directoryPath: string): Registry {
  let directory = io.read(directoryPath) as ImmutableDirectory;
  let deps = getDeps(directory);
  let registry = createRegistry(deps);
  removeDuplicateDeps(registry);
  return registry;
}

function getDeps(source: ImmutableDirectory, path: string[] = []): FolderDeps {
  let folderDeps: FolderDeps = {};
  for (let [childName, child] of source.list().entries()) {
    if (child instanceof ImmutableFile) {
      let parsed: ts.SourceFile;
      let text = child.getBuffer().toString("utf8");
      if (childName.endsWith(".js")) {
        parsed = parseSourceFile(text, ts.ScriptKind.JS);
      } else if (childName.endsWith(".ts")) {
        parsed = parseSourceFile(text, ts.ScriptKind.TS);
      } else if (childName.endsWith(".jsx")) {
        parsed = parseSourceFile(text, ts.ScriptKind.JSX);
      } else if (childName.endsWith(".tsx")) {
        parsed = parseSourceFile(text, ts.ScriptKind.TSX);
      } else {
        // Ignore.
        continue;
      }
      let fileDeps = new FileDeps();
      for (let statement of parsed.statements) {
        if (ts.isImportDeclaration(statement)) {
          if (!ts.isStringLiteral(statement.moduleSpecifier)) {
            throw new Error(
              "Found an import that is not a string literal: " +
                statement.moduleSpecifier
            );
          }
          fileDeps.imports.push(
            absolutePath(path, statement.moduleSpecifier.text)
          );
        }
      }
      folderDeps[childName] = fileDeps;
    } else if (childName != "node_modules") {
      folderDeps[childName] = getDeps(child, path.concat(childName));
    }
  }
  return folderDeps;
}

function parseSourceFile(
  source: string,<commit_msg>Handle error cases better and don't load large files unnecessarily<commit_after>import * as ts from "typescript";

import { ImmutableDirectory, ImmutableFile, io } from "nofiles";

export type Registry = {
  indexes: { [key: string]: number };
  paths: string[];
  deps: number[][];
  size: number;
};

type FolderDeps = { [key: string]: FolderDeps | FileDeps };

class FileDeps {
  imports: string[] = [];
}

export default function analyze(directoryPath: string): Registry {
  let directory = io.read(directoryPath) as ImmutableDirectory;
  let deps = getDeps(directory);
  let registry = createRegistry(deps);
  removeDuplicateDeps(registry);
  return registry;
}

function getDeps(source: ImmutableDirectory, path: string[] = []): FolderDeps {
  let folderDeps: FolderDeps = {};
  for (let [childName, child] of source.list().entries()) {
    let childPath = path.concat(childName);
    if (child instanceof ImmutableFile) {
      try {
        let parsed: ts.SourceFile;
        if (childName.endsWith(".js")) {
          parsed = parseSourceFile(
            child.getBuffer().toString("utf8"),
            ts.ScriptKind.JS
          );
        } else if (childName.endsWith(".ts")) {
          parsed = parseSourceFile(
            child.getBuffer().toString("utf8"),
            ts.ScriptKind.TS
          );
        } else if (childName.endsWith(".jsx")) {
          parsed = parseSourceFile(
            child.getBuffer().toString("utf8"),
            ts.ScriptKind.JSX
          );
        } else if (childName.endsWith(".tsx")) {
          parsed = parseSourceFile(
            child.getBuffer().toString("utf8"),
            ts.ScriptKind.TSX
          );
        } else {
          // Ignore.
          continue;
        }
        let fileDeps = new FileDeps();
        for (let statement of parsed.statements) {
          if (ts.isImportDeclaration(statement)) {
            if (!ts.isStringLiteral(statement.moduleSpecifier)) {
              throw new Error(
                "Found an import that is not a string literal in " +
                  childPath.join("/")
              );
            }
            fileDeps.imports.push(
              absolutePath(path, statement.moduleSpecifier.text)
            );
          }
        }
        folderDeps[childName] = fileDeps;
      } catch (e) {
        console.error("Could not parse " + childPath.join("/"), e);
      }
    } else if (childName != "node_modules") {
      folderDeps[childName] = getDeps(child, childPath);
    }
  }
  return folderDeps;
}

function parseSourceFile(
  source: string,
{'fraction_non_alphanumeric': 0.12582781456953643, 'fraction_numerical': 0.0010681478316599017, 'mean_word_length': 2.4810408921933087, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '615694', 'n_tokens_mistral': 1376, 'n_tokens_neox': 1377, 'n_words': 319}