<commit_before>import * as request from 'request';
import { createWriteStream } from 'fs';
import { dirname } from 'path';
import { writeJsonFile, ensureDirectory } from './fs';
import { getFilePath, getConfig } from './utils';
import { INpmPackage } from './package';
import { findPackage, storage, updatePkgStorage } from './storage';
import * as logger from './logger';

export function fetchUplinkPackage(packageName: string, version: string): Promise<any> {
  return new Promise(resolve => {
    let config = getConfig();
    let upstream = config.upstreams[0];

    packageName = version ? `${packageName}/${version}` : packageName;
    let url = `${upstream}/${packageName}`;

    getResponse(url).then(body => {
      body = JSON.parse(body);

      if (!config.saveUpstreamPackages) {
        resolve(body);
      } else {
        version = version || body['dist-tags']['latest'] || null;
        if (!version) {
          resolve(body);
        }

        url = body.versions[version].dist.tarball;
        let tarball = url.split('/').slice(-1)[0];
        packageName = packageName.replace(/\%2f/ig, '/');

        downloadTarball(url, tarball, packageName).then(() => {
          let pkg = findPackage(packageName);
          if (!pkg) {
            writePackageJson(packageName, version, body).then(() => resolve(body));
          } else {
            updatePkgStorage(packageName, body).then(() => resolve(body));
          }
        });
      }
    });
  });
}

function getResponse(url: string): Promise<any> {
  return new Promise(resolve => {
    logger.httpOut(url, 'GET');
    request(url, { method: 'GET' }, (err, resp, body) => resolve(body));
  });
}

function downloadTarball(url: string, tarball: string, packageName: string): Promise<null> {
  return new Promise(resolve => {
    let filePath = getFilePath(`tarballs/${packageName}/${tarball}`);
    ensureDirectory(dirname(filePath)).then(() => {
      request.get(url)
        .on('response', (resp: any) => {
          logger.httpIn(url, 'GET', resp);
          logger.info(`${filePath} saved locally.`);
          resolve();
        })
        .pipe(createWriteStream(filePath));
    });
  });
}

function writePackageJson(packageName: string, version: string, body: any): Promise<null> {
  let packageData: INpmPackage = {
    _id: packageName,
    name: packageName,
    description: body.description,
    'dist-tags': { 'latest': version },
    versions: body.versions
  };
  let filePath = getFilePath(`packages/${packageName}/${version}/package.json`);

  return ensureDirectory(dirname(filePath))
    .then(() => writeJsonFile(getFilePath(`packages/${packageName}/${version}/package.json`), packageData))
    .then(() => updatePkgStorage(packageName, packageData));
}<commit_msg>chore(proxy): do not overwrite previous tarball<commit_after>import * as request from 'request';
import { createWriteStream } from 'fs';
import { dirname } from 'path';
import { writeJsonFile, ensureDirectory, exists } from './fs';
import { getFilePath, getConfig } from './utils';
import { INpmPackage } from './package';
import { findPackage, storage, updatePkgStorage } from './storage';
import * as logger from './logger';

export function fetchUplinkPackage(packageName: string, version: string): Promise<any> {
  return new Promise(resolve => {
    let config = getConfig();
    let upstream = config.upstreams[0];

    packageName = version ? `${packageName}/${version}` : packageName;
    let url = `${upstream}/${packageName}`;

    getResponse(url).then(body => {
      body = JSON.parse(body);

      if (!config.saveUpstreamPackages) {
        resolve(body);
      } else {
        version = version || body['dist-tags']['latest'] || null;
        if (!version) {
          resolve(body);
        }

        url = body.versions[version].dist.tarball;
        let tarball = url.split('/').slice(-1)[0];
        packageName = packageName.replace(/\%2f/ig, '/');

        downloadTarball(url, tarball, packageName).then(() => {
          let pkg = findPackage(packageName);
          if (!pkg) {
            writePackageJson(packageName, version, body).then(() => resolve(body));
          } else {
            updatePkgStorage(packageName, body).then(() => resolve(body));
          }
        });
      }
    });
  });
}

function getResponse(url: string): Promise<any> {
  return new Promise(resolve => {
    logger.httpOut(url, 'GET');
    request(url, { method: 'GET' }, (err, resp, body) => resolve(body));
  });
}

function downloadTarball(url: string, tarball: string, packageName: string): Promise<null> {
  return new Promise(resolve => {
    let filePath = getFilePath(`tarballs/${packageName}/${tarball}`);
    exists(filePath).then(tarballExists => {
      if (tarballExists) {
        resolve();
      } else {
        ensureDirectory(dirname(filePath)).then(() => {
          request.get(url)
            .on('response', (resp: any) => {
              logger.httpIn(url, 'GET', resp);
              logger.info(`${filePath} saved locally.`);
              resolve();
            })
            .pipe(createWriteStream(filePath));
        });
      }
    });
  });
}

function writePackageJson(packageName: string, version: string, body: any): Promise<null> {
  let packageData: INpmPackage = {
    _id: packageName,
    name: packageName,
    description: body.description,
    'dist-tags': { 'latest': version },
    versions: body.versions
  };
  let filePath = getFilePath(`packages/${packageName}/${version}/package.json`);

  return ensureDirectory(dirname(filePath))
    .then(() => writeJsonFile(getFilePath(`packages/${packageName}/${version}/package.json`), packageData))
    .then(() => updatePkgStorage(packageName, packageData));
}
{'fraction_non_alphanumeric': 0.1648639218422889, 'fraction_numerical': 0.0013956734124214933, 'mean_word_length': 3.1724890829694323, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4686651', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1649, 'n_words': 359}