<commit_before>import { Repository, Config, Project, getRepositoryByIdSafe } from './domain';
import { exec, execFile } from 'child_process';
import { log } from './log';
import { newTmpDir, executeCommand, makePath, addToArray } from './util';
import { gitClone } from './git';
import { uniqBy, find, constant, forEach } from 'lodash';

export function checkoutIntoTmp(repos: Iterable<Repository>): Promise<{ [repoId: string]: string }> {
  const promises: Promise<string>[] = [];
  const pathByRepoId: { [repoId: string]: string } = {};
  for (const repo of repos) {
    const tmpDir = newTmpDir();
    log.debug(`checking out ${repo.url} into ${tmpDir}`);
    pathByRepoId[repo.id] = tmpDir;
    promises.push(gitClone(repo.url, tmpDir));
  }
  return Promise.all(promises).then(constant(pathByRepoId));
}

export type PathByProjectId = { [projectId: string]: { projectPath: string, repoPath: string } };

export function checkoutAllProjects(projects: Project[]): Promise<PathByProjectId> {
  const repoByProjectId: { [projectId: string]: Repository }[] = [];
  const projectsByRepoId: { [repoId: string]: Project[] } = {};
  const allRepos: Set<Repository> = new Set<Repository>();
  for (const project of projects) {
    const repo = getRepositoryByIdSafe(project.repositoryId);
    addToArray(projectsByRepoId, repo.id, project);
    repoByProjectId[project.id] = repo;
    allRepos.add(repo);
  }
  return checkoutIntoTmp(allRepos).then((pathByRepoId) => {
    log.debug(`checked out all repos`);
    const pathByProjectId: PathByProjectId = {};
    forEach(pathByRepoId, (path, repoId: string) => {
      const projects = projectsByRepoId[repoId];
      for (const project of projects) {
        pathByProjectId[project.id] = {
          repoPath: path,
          projectPath: makePath(path, project.path)
        };
      }
    });
    return pathByProjectId;
  });
}


<commit_msg>refactoring<commit_after>import { Repository, Config, Project, getRepositoryByIdSafe } from './domain';
import { exec, execFile } from 'child_process';
import { log } from './log';
import { newTmpDir, executeCommand, makePath, addToArray } from './util';
import { gitClone } from './git';
import { flatMap, keys, map, reduce, uniqBy, find, constant, forEach } from 'lodash';

export function checkoutIntoTmp(repos: Iterable<Repository>): Promise<{ [repoId: string]: string }> {
  const promises: Promise<string>[] = [];
  const pathByRepoId: { [repoId: string]: string } = {};
  for (const repo of repos) {
    const tmpDir = newTmpDir();
    log.debug(`checking out ${repo.url} into ${tmpDir}`);
    pathByRepoId[repo.id] = tmpDir;
    promises.push(gitClone(repo.url, tmpDir));
  }
  return Promise.all(promises).then(constant(pathByRepoId));
}

export type PathByProjectId = { [projectId: string]: { projectPath: string, repoPath: string } };

export function checkoutAllProjects(projects: Project[]): Promise<PathByProjectId> {
  const repoByProjectId: { [projectId: string]: Repository }[] = [];
  const projectsByRepoId: { [repoId: string]: Project[] } = {};
  const allRepos: Set<Repository> = new Set<Repository>();
  for (const project of projects) {
    const repo = getRepositoryByIdSafe(project.repositoryId);
    addToArray(projectsByRepoId, repo.id, project);
    repoByProjectId[project.id] = repo;
    allRepos.add(repo);
  }
  return checkoutIntoTmp(allRepos).then((pathByRepoId) => {
    log.debug(`checked out all repos`);
    type RepoPathProject = [string, Project];
    const repoIds = keys(pathByRepoId);
    const repoPathProjecTuples: RepoPathProject[] = flatMap(repoIds, (repoId): RepoPathProject[] => {
      const projects = projectsByRepoId[repoId];
      const repoPath = pathByRepoId[repoId];
      return map(projects, (project) => [repoPath, project] as RepoPathProject);
    });
    return reduce(repoPathProjecTuples, (acc, [path, project]) => {
      return {
        [project.id]: {
          repoPath: path,
          projectPath: makePath(path, project.path)
        },
        ...acc
      };
    }, {} as PathByProjectId);
  });
}



{'fraction_non_alphanumeric': 0.14415744157441573, 'fraction_numerical': 0.0, 'mean_word_length': 4.239690721649485, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167532', 'n_tokens_mistral': 1273, 'n_tokens_neox': 1277, 'n_words': 304}