<commit_before>import * as crypto from 'crypto';
import * as path from 'path';
import * as sfs from 'fs-extra';

import { Repository, Clone, Checkout, Cred, Reference, Signature, Remote } from 'nodegit';
import { getNwoFromRepoUrl } from './github-api';
import { toIso8601 } from 'iso8601';
import { statNoException, statSyncNoException } from './promise-array';
import { rimraf, mkdirp, mkdirpSync } from './recursive-fs';

import * as fs from 'mz/fs';
import { spawnPromise } from 'spawn-rx';

// tslint:disable-next-line:no-var-requires
const d = require('debug')('surf:git-api');

type FreeMethod = ((f: any) => any);

function using<TRet>(block: (f: FreeMethod) => TRet): TRet {
  let toFree: any[] = [];

  try {
    return block((f) => { toFree.push(f); return f; });
  } finally {
    toFree.reverse().forEach((f) => f.free());
  }
}

export async function getHeadForRepo(targetDirname: string) {
  let repoDir = (await Repository.discover(targetDirname, 0, '')) as string;

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let commit = ds(await repo.getHeadCommit());
    return commit.sha();
  });
}

export async function getOriginForRepo(targetDirname: string) {
  let repoDir = await Repository.discover(targetDirname, 0, '');

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let origin = ds(await Remote.lookup(repo, 'origin', () => {}));

    return origin.pushurl() || origin.url();
  });
}

export async function getOriginDefaultBranchName(targetDirname: string, token?: string) {
  let repoDir = await Repository.discover(targetDirname, 0, '');

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let origin = ds(await Remote.lookup(repo, 'origin', () => {}));

    let cb = {
      credentials: () => {
        d(`Returning ${token} for authentication token`);
        return Cred.userpassPlaintextNew(token || '', 'x-oauth-basic');
      },
      certificateCheck: () => {
        // Yolo
        return 1;
       }
    };

    await origin.connect(0, cb, null, null, null);

    try {
      let ret = await origin.defaultBranch();
      return ret.replace('refs/heads/', '');
    } finally {
      origin.disconnect();
    }
  });
}

export async function getAllWorkdirs(repoUrl: string) {
  let tmp = process.env.SURF_ORIGINAL_TMPDIR || process.env.TMPDIR || process.env.TEMP || '/tmp';
  let ret = await fs.readdir(tmp);

  return ret.reduce((acc: string[], x) => {
    let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
    if (x.match(/^surfg-/i)) {
      let tgt = path.join(tmp, x);
      let stats = fs.statSync(tgt);
      let now = new Date();

      if ((now.getTime() - stats.mtime.getTime()) > 1000 * 60 * 60 * 2) {
        acc.push(path.join(tmp, x));
      }

      return acc;
    }

    if (!x.match(/-[a-f0-9A-F]{6}/i)) return acc;
    if (x.indexOf(`${nwo}-`) < 0) return acc;

    acc.push(path.join(tmp, x));
    return acc;
  }, []);
}

export function parseGitDiffOutput(output: string): string[] {
  return output.split('\n')
    .filter(line => line.length > 1)
    .map(line => {
      let pathSegment = line.split('\t')[2];
      if (pathSegment.indexOf('{') < 0) return pathSegment;

      // Fix up renames, which are of the format:
      // src/job-installers/{systemd.js => systemd.ts}
      return pathSegment.replace(/(.*){.*=> (.*)}$/, '$1$2');
    });
}

export async function getChangedFiles(targetDirname: string): Promise<string[]> {
  let opts = { cwd: targetDirname };

  let ourCommit = (await getHeadForRepo(targetDirname));
  d(`Got our commit: ${ourCommit}`);
  let defaultRemoteBranch = await getOriginDefaultBranchName(targetDirname);

  d(`Using origin/${defaultRemoteBranch} as remote default branch`);
  let remoteHeadCommit = (await spawnPromise('git', ['rev-parse', `origin/${defaultRemoteBranch}`], opts)).trim();

  // If we're on the remote master branch, there are no changes,
  // so just return every file
  if (ourCommit === remoteHeadCommit) {
    return (await spawnPromise('git', ['ls-files'], opts))
      .split('\n')
      .filter(x => x.length > 1);
  }

  return parseGitDiffOutput(
    await spawnPromise('git', ['diff', '--numstat', 'origin/HEAD...HEAD']));
}

export function getWorkdirForRepoUrl(repoUrl: string, sha: string, dontCreate= false) {
  let tmp = process.env.TMPDIR || process.env.TEMP || '/tmp';
  let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
  let date = toIso8601(new Date()).replace(/:/g, '.');
  let shortSha = sha.substr(0,6);

  let ret = path.join(tmp, `${nwo}-${shortSha}`);

  if (statSyncNoException(ret)) {
    ret = path.join(tmp, `${nwo}-${shortSha}-${date}`);
  }

  if (!dontCreate) mkdirpSync(ret);
  return ret;
}

export function getTempdirForRepoUrl(repoUrl: string, sha: string, dontCreate= false) {
  let tmp = process.env.TMPDIR || process.env.TEMP || '/tmp';
  let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
  let date = toIso8601(new Date()).replace(/:/g, '.');<commit_msg>Fix up our token usage<commit_after>import * as crypto from 'crypto';
import * as path from 'path';
import * as sfs from 'fs-extra';

import { Repository, Clone, Checkout, Cred, Reference, Signature, Remote } from 'nodegit';
import { getNwoFromRepoUrl } from './github-api';
import { toIso8601 } from 'iso8601';
import { statNoException, statSyncNoException } from './promise-array';
import { rimraf, mkdirp, mkdirpSync } from './recursive-fs';

import * as fs from 'mz/fs';
import { spawnPromise } from 'spawn-rx';

// tslint:disable-next-line:no-var-requires
const d = require('debug')('surf:git-api');

type FreeMethod = ((f: any) => any);

function using<TRet>(block: (f: FreeMethod) => TRet): TRet {
  let toFree: any[] = [];

  try {
    return block((f) => { toFree.push(f); return f; });
  } finally {
    toFree.reverse().forEach((f) => f.free());
  }
}

export async function getHeadForRepo(targetDirname: string) {
  let repoDir = (await Repository.discover(targetDirname, 0, '')) as string;

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let commit = ds(await repo.getHeadCommit());
    return commit.sha();
  });
}

export async function getOriginForRepo(targetDirname: string) {
  let repoDir = await Repository.discover(targetDirname, 0, '');

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let origin = ds(await Remote.lookup(repo, 'origin', () => {}));

    return origin.pushurl() || origin.url();
  });
}

export async function getOriginDefaultBranchName(targetDirname: string, token?: string) {
  let repoDir = await Repository.discover(targetDirname, 0, '');
  token = token || process.env.GITHUB_TOKEN;

  return await using(async (ds) => {
    let repo = ds(await Repository.open(repoDir));
    let origin = ds(await Remote.lookup(repo, 'origin', () => {}));

    let cb = {
      credentials: () => {
        d(`Returning ${token} for authentication token`);
        return Cred.userpassPlaintextNew(token || '', 'x-oauth-basic');
      },
      certificateCheck: () => {
        // Yolo
        return 1;
       }
    };

    await origin.connect(0, cb, null, null, null);

    try {
      let ret = await origin.defaultBranch();
      return ret.replace('refs/heads/', '');
    } finally {
      origin.disconnect();
    }
  });
}

export async function getAllWorkdirs(repoUrl: string) {
  let tmp = process.env.SURF_ORIGINAL_TMPDIR || process.env.TMPDIR || process.env.TEMP || '/tmp';
  let ret = await fs.readdir(tmp);

  return ret.reduce((acc: string[], x) => {
    let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
    if (x.match(/^surfg-/i)) {
      let tgt = path.join(tmp, x);
      let stats = fs.statSync(tgt);
      let now = new Date();

      if ((now.getTime() - stats.mtime.getTime()) > 1000 * 60 * 60 * 2) {
        acc.push(path.join(tmp, x));
      }

      return acc;
    }

    if (!x.match(/-[a-f0-9A-F]{6}/i)) return acc;
    if (x.indexOf(`${nwo}-`) < 0) return acc;

    acc.push(path.join(tmp, x));
    return acc;
  }, []);
}

export function parseGitDiffOutput(output: string): string[] {
  return output.split('\n')
    .filter(line => line.length > 1)
    .map(line => {
      let pathSegment = line.split('\t')[2];
      if (pathSegment.indexOf('{') < 0) return pathSegment;

      // Fix up renames, which are of the format:
      // src/job-installers/{systemd.js => systemd.ts}
      return pathSegment.replace(/(.*){.*=> (.*)}$/, '$1$2');
    });
}

export async function getChangedFiles(targetDirname: string, token?: string): Promise<string[]> {
  let opts = { cwd: targetDirname };
  token = token || process.env.GITHUB_TOKEN;

  let ourCommit = (await getHeadForRepo(targetDirname));
  d(`Got our commit: ${ourCommit}`);
  let defaultRemoteBranch = await getOriginDefaultBranchName(targetDirname, token);

  d(`Using origin/${defaultRemoteBranch} as remote default branch`);
  let remoteHeadCommit = (await spawnPromise('git', ['rev-parse', `origin/${defaultRemoteBranch}`], opts)).trim();

  // If we're on the remote master branch, there are no changes,
  // so just return every file
  if (ourCommit === remoteHeadCommit) {
    return (await spawnPromise('git', ['ls-files'], opts))
      .split('\n')
      .filter(x => x.length > 1);
  }

  return parseGitDiffOutput(
    await spawnPromise('git', ['diff', '--numstat', 'origin/HEAD...HEAD']));
}

export function getWorkdirForRepoUrl(repoUrl: string, sha: string, dontCreate = false) {
  let tmp = process.env.TMPDIR || process.env.TEMP || '/tmp';
  let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
  let date = toIso8601(new Date()).replace(/:/g, '.');
  let shortSha = sha.substr(0,6);

  let ret = path.join(tmp, `${nwo}-${shortSha}`);

  if (statSyncNoException(ret)) {
    ret = path.join(tmp, `${nwo}-${shortSha}-${date}`);
  }

  if (!dontCreate) mkdirpSync(ret);
  return ret;
}

export function getTempdirForRepoUrl(repoUrl: string, sha: string, dontCreate = false) {
  let tmp = process.env.TMPDIR || process.env.TEMP || '/tmp';
  let nwo = getNwoFromRepoUrl(repoUrl).split('/')[1];
  let date = toIso8601(new Date()).replace(/:/g, '.');
{'fraction_non_alphanumeric': 0.17754262198706644, 'fraction_numerical': 0.008818342151675485, 'mean_word_length': 3.5526315789473686, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165700', 'n_tokens_mistral': 3489, 'n_tokens_neox': 3490, 'n_words': 823}