<commit_before>/**
 * Copyright 2018 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import * as acorn from 'acorn';
import * as fs from 'fs';
import * as path from 'path';
import pify from 'pify';

// import { PackageLockJson, Dependency } from './packageLock.d';

import * as analysis from './detection-functions';
import * as util from './util';

export interface ReadFileP {
  (path: string, encoding: string): Promise<string>;
}

const readFilep: ReadFileP = pify(fs.readFile);

export interface PointOfInterest {
  type: string;
  fileName: string;
  position: Position;
}

export interface Position {
  lineStart: number;
  lineEnd: number;
  colStart: number;
  colEnd: number;
}

export interface PackageTree<T = null> {
  name: string;
  version: string;
  data: T;
  dependencies: Array<PackageTree<T>>;
}

export interface PackageLock {
  name: string;
  version: string;
  lockfileVersion: number;
  packageIntegrity?: string;
  preserveSymlinks?: boolean;
  requires?: boolean;
  dependencies?: {[moduleName: string]: Dependency};
}

export interface Dependency {
  version: string;
  integrity?: string;
  resolved?: string;
  bundled?: boolean;
  dev?: boolean;
  optional?: boolean;
  requires?: {[moduleName: string]: string};
  dependencies?: {[moduleName: string]: Dependency};
}

/**
 * Replaces the data of each node in the packageTree with the Points Of
 * Interest array
 *
 * @param packageTree the original package tree with data property as the
 *    node's path
 */
export async function populatePOIInPackageTree(
    packageTree: PackageTree<string>): Promise<PackageTree<PointOfInterest[]>> {
  // Get package trees with POI arrays in data field
  const dependenciesWithPOI: Array<PackageTree<PointOfInterest[]>> = [];
  await Promise.all(packageTree.dependencies.map(async (pkg) => {
    const dependencyPOIList: PackageTree<PointOfInterest[]> =
        await populatePOIInPackageTree(pkg);
    dependenciesWithPOI.push(dependencyPOIList);
  }));

  // Get the POI list for this current package
  const poiList: PointOfInterest[] = await getPackagePOIList(packageTree.data);

  // Create new tree using POI list as data and new list of package trees as
  // dependencies
  const tree: PackageTree<PointOfInterest[]> = {
    name: packageTree.name,
    version: packageTree.version,
    data: poiList,
    dependencies: dependenciesWithPOI
  };
  return tree;
}

/**
 * Gets a packageTree node's Points of Interest array
 *
 * @param path the path to the packageTree node
 */
export async function getPackagePOIList(path: string):
    Promise<PointOfInterest[]> {
  const packagePOIList: PointOfInterest[] = [];
  const files = await getJSFiles(path);

  await Promise.all(files.map(async (file) => {
    const content = await util.readFile(file, 'utf8');
    const functionArr: Function[] = [
      analysis.getIOModules, analysis.getArbitraryExecutionMods,
      analysis.unusualUsesOfRequire, analysis.getEvalCalls,
      analysis.getEnvAccesses, analysis.getFunctionClassAccesses

    ];
    const filePOIList = getPointsOfInterest(content, file, functionArr);
    packagePOIList.push(...filePOIList);
  }));

  return packagePOIList;
}

/**
 * Creates a new packageTree node with a new data property that contains
 * the path to the package and replaces its dependency property with updated
 * packageTree nodes
 *
 * @param rootNode the original packageTree root node
 * @param rootPath the path to the root node
 */
export async function resolvePaths(
    rootNode: PackageTree, rootPath: string): Promise<PackageTree<string>> {
  const updatedNodesMap = new Map<string, PackageTree<string>>();
  const resolvedNodes: Array<PackageTree<string>> = [];

  await Promise.all(rootNode.dependencies.map(async (child) => {
    const resolvedDependency =
        await resolvePathsRec(child, rootPath, updatedNodesMap);
    resolvedNodes.push(resolvedDependency);
  }));

  const updatedRoot: PackageTree<string> = {
    name: rootNode.name,
    version: rootNode.version,
    data: rootPath,
    dependencies: resolvedNodes
  };

  return updatedRoot;

  async function resolvePathsRec(
      packageNode: PackageTree, parentPath: string,
      updatedNodesMap: Map<string, PackageTree<string>>):
      Promise<PackageTree<string>> {
    const resolvedNodes: Array<PackageTree<string>> = [];

    const path: string = await findPath(packageNode.name, parentPath);
    await Promise.all(packageNode.dependencies.map(async (child) => {
      resolvedNodes.push(await resolvePathsRec(child, path, updatedNodesMap));
    }));

    // creates new node if node doesn't exist already
    if (!updatedNodesMap.has(path)) {
      const updatedNode: PackageTree<string> = {
        name: packageNode.name,
        version: packageNode.version,
        data: path,
        dependencies: resolvedNodes
      };
      updatedNodesMap.set(path, updatedNode);
      return updatedNode;
    } else {
      return updatedNodesMap.get(path)!;
    }
  }
}

/**
 * Gets the package path based on the location of the parent package
 *
 * @param mod the package/module
 * @param parentPath the path of the parent package that depends on mod
 */
export async function findPath(
    mod: string, parentPath: string): Promise<string> {
  const moduleFolder = path.join(parentPath, 'node_modules');
  if ((await util.exists(moduleFolder)) &&
      (await util.stat(moduleFolder)).isDirectory()) {
    const filesInFolder = await util.readdir(moduleFolder);
    if (filesInFolder.includes(mod)) {
      return path.join(moduleFolder, mod);
    }
  }
  let currPath = path.dirname(parentPath);<commit_msg>fix: findPath now works with scoped dependencies (#22)<commit_after>/**
 * Copyright 2018 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import * as acorn from 'acorn';
import * as fs from 'fs';
import * as path from 'path';
import pify from 'pify';

// import { PackageLockJson, Dependency } from './packageLock.d';

import * as analysis from './detection-functions';
import * as util from './util';

export interface ReadFileP {
  (path: string, encoding: string): Promise<string>;
}

const readFilep: ReadFileP = pify(fs.readFile);

export interface PointOfInterest {
  type: string;
  fileName: string;
  position: Position;
}

export interface Position {
  lineStart: number;
  lineEnd: number;
  colStart: number;
  colEnd: number;
}

export interface PackageTree<T = null> {
  name: string;
  version: string;
  data: T;
  dependencies: Array<PackageTree<T>>;
}

export interface PackageLock {
  name: string;
  version: string;
  lockfileVersion: number;
  packageIntegrity?: string;
  preserveSymlinks?: boolean;
  requires?: boolean;
  dependencies?: {[moduleName: string]: Dependency};
}

export interface Dependency {
  version: string;
  integrity?: string;
  resolved?: string;
  bundled?: boolean;
  dev?: boolean;
  optional?: boolean;
  requires?: {[moduleName: string]: string};
  dependencies?: {[moduleName: string]: Dependency};
}

/**
 * Replaces the data of each node in the packageTree with the Points Of
 * Interest array
 *
 * @param packageTree the original package tree with data property as the
 *    node's path
 */
export async function populatePOIInPackageTree(
    packageTree: PackageTree<string>): Promise<PackageTree<PointOfInterest[]>> {
  // Get package trees with POI arrays in data field
  const dependenciesWithPOI: Array<PackageTree<PointOfInterest[]>> = [];
  await Promise.all(packageTree.dependencies.map(async (pkg) => {
    const dependencyPOIList: PackageTree<PointOfInterest[]> =
        await populatePOIInPackageTree(pkg);
    dependenciesWithPOI.push(dependencyPOIList);
  }));

  // Get the POI list for this current package
  const poiList: PointOfInterest[] = await getPackagePOIList(packageTree.data);

  // Create new tree using POI list as data and new list of package trees as
  // dependencies
  const tree: PackageTree<PointOfInterest[]> = {
    name: packageTree.name,
    version: packageTree.version,
    data: poiList,
    dependencies: dependenciesWithPOI
  };
  return tree;
}

/**
 * Gets a packageTree node's Points of Interest array
 *
 * @param path the path to the packageTree node
 */
export async function getPackagePOIList(path: string):
    Promise<PointOfInterest[]> {
  const packagePOIList: PointOfInterest[] = [];
  const files = await getJSFiles(path);

  await Promise.all(files.map(async (file) => {
    const content = await util.readFile(file, 'utf8');
    const functionArr: Function[] = [
      analysis.getIOModules, analysis.getArbitraryExecutionMods,
      analysis.unusualUsesOfRequire, analysis.getEvalCalls,
      analysis.getEnvAccesses, analysis.getFunctionClassAccesses

    ];
    const filePOIList = getPointsOfInterest(content, file, functionArr);
    packagePOIList.push(...filePOIList);
  }));

  return packagePOIList;
}

/**
 * Creates a new packageTree node with a new data property that contains
 * the path to the package and replaces its dependency property with updated
 * packageTree nodes
 *
 * @param rootNode the original packageTree root node
 * @param rootPath the path to the root node
 */
export async function resolvePaths(
    rootNode: PackageTree, rootPath: string): Promise<PackageTree<string>> {
  const updatedNodesMap = new Map<string, PackageTree<string>>();
  const resolvedNodes: Array<PackageTree<string>> = [];

  await Promise.all(rootNode.dependencies.map(async (child) => {
    const resolvedDependency =
        await resolvePathsRec(child, rootPath, updatedNodesMap);
    resolvedNodes.push(resolvedDependency);
  }));

  const updatedRoot: PackageTree<string> = {
    name: rootNode.name,
    version: rootNode.version,
    data: rootPath,
    dependencies: resolvedNodes
  };

  return updatedRoot;

  async function resolvePathsRec(
      packageNode: PackageTree, parentPath: string,
      updatedNodesMap: Map<string, PackageTree<string>>):
      Promise<PackageTree<string>> {
    const resolvedNodes: Array<PackageTree<string>> = [];

    const path: string = await findPath(packageNode.name, parentPath);
    await Promise.all(packageNode.dependencies.map(async (child) => {
      resolvedNodes.push(await resolvePathsRec(child, path, updatedNodesMap));
    }));

    // creates new node if node doesn't exist already
    if (!updatedNodesMap.has(path)) {
      const updatedNode: PackageTree<string> = {
        name: packageNode.name,
        version: packageNode.version,
        data: path,
        dependencies: resolvedNodes
      };
      updatedNodesMap.set(path, updatedNode);
      return updatedNode;
    } else {
      return updatedNodesMap.get(path)!;
    }
  }
}

/**
 * Gets the package path based on the location of the parent package
 *
 * @param mod the package/module
 * @param parentPath the path of the parent package that depends on mod
 */
export async function findPath(
    mod: string, parentPath: string): Promise<string> {
  const moduleFolder = path.join(parentPath, 'node_modules', mod);

  // Checks to see if parentPath/node_modules/moduleName exists. If not call
  // findPath on the directory above
  if ((await util.exists(moduleFolder)) &&
      (await util.stat(moduleFolder)).isDirectory()) {
    return moduleFolder;
  }
  let currPath = path.dirname(parentPath);
{'fraction_non_alphanumeric': 0.10249352331606218, 'fraction_numerical': 0.0016191709844559584, 'mean_word_length': 4.083539094650206, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1160222', 'n_tokens_mistral': 3487, 'n_tokens_neox': 3438, 'n_words': 1118}