<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {ENABLED_RULES} from './rule_groups';
import {Checker} from './third_party/tsetse/checker';
import * as path from 'path';
import * as ts from 'typescript';

import {createEmptyExemptionList, ExemptionList, parseConformanceExemptionConfig} from './exemption_config';
import {FORMAT_DIAGNOSTIC_HOST, reportDiagnostic, reportDiagnosticsWithSummary, reportErrorSummary} from './report';
import {ExtendedParsedCommandLine, parseTsConfigFile} from './tsconfig';

/** Check if tsec is invoked in the build mode. */
export function isInBuildMode(cmdArgs: string[]) {
  // --build or -b has to be the first argument
  if (cmdArgs.length && cmdArgs[0].charAt(0) === '-') {
    const optionStart = cmdArgs[0].charAt(1) === '-' ? 2 : 1;
    const firstOption = cmdArgs[0].slice(optionStart);
    return firstOption === 'build' || firstOption === 'b';
  }
  return false;
}

/** Perform conformance checks on a single project. */
export function performConformanceCheck(
    program: ts.Program,
    conformanceExemptionConfig: ExemptionList =
        createEmptyExemptionList()): ts.Diagnostic[] {
  const diagnostics = [...ts.getPreEmitDiagnostics(program)];

  // Create all enabled rules with corresponding exemption list entries.
  const conformanceChecker = new Checker(program);
  const conformanceRules = ENABLED_RULES.map(ruleCtr => {
    const allowlistEntries = [];
    const allowlistEntry = conformanceExemptionConfig.get(ruleCtr.RULE_NAME);
    if (allowlistEntry) {
      allowlistEntries.push(allowlistEntry);
    }
    return new ruleCtr({allowlistEntries});
  });

  // Register all rules.
  for (const rule of conformanceRules) {
    rule.register(conformanceChecker);
  }

  // Run all enabled conformance checks and collect errors.
  for (const sf of program.getSourceFiles()) {
    // We don't emit errors for delcarations, so might as well skip checking
    // declaration files all together.
    if (sf.isDeclarationFile) continue;
    const conformanceDiagErr = conformanceChecker.execute(sf).map(
        failure => failure.toDiagnosticWithStringifiedFixes());
    diagnostics.push(...conformanceDiagErr);
  }

  return diagnostics;
}

/** Perform conformance checks on a monorepo. */
export function performBuild(args: string[]): number {
  // We don't support any of the build options, so we only expect one option
  // in build mode. It may make sense to support incremental build to save
  // time. The others are likely never relevant to tsec.
  const projects = args.slice(1);

  // Bail if there are more than one tsconfig.json (or unsupported options)
  // are provided.
  if (projects.length > 1) {
    return 1;
  }

  // A hack to invalidate any build info generated by the vanilla tsc to make
  // sure tsec has a chance to build the project even nothing is modified
  // since last build.
  // TODO(pwng): Add an additional version suffix to reflect the configuration
  // used in this tsec run.
  // tslint:disable-next-line:ban-module-namespace-object-escape
  (ts as {version: string}).version += '-tsec';

  const buildHost = ts.createSolutionBuilderHost(
      ts.sys,
      /*createProgram*/ undefined,
      reportDiagnostic,
      /*reportSolutionBuilderStatus*/ undefined,
      reportErrorSummary,
  );

  const diagnostics: ts.Diagnostic[] = [];

  buildHost.afterProgramEmitAndDiagnostics = (p) => {
    diagnostics.push(...performConformanceCheck(p.getProgram()));
  };
  const builder =
      ts.createSolutionBuilder(buildHost, projects, /*buildOptions*/ {});

  const exitStatus = builder.build();
  if (exitStatus !== ts.ExitStatus.Success) {
    ts.sys.write(
        'There are build errors in your project. Please make sure your project can be built by tsc');
    ts.sys.write(ts.sys.newLine);
    return 1;
  }

  const errorCount = reportDiagnosticsWithSummary(diagnostics);

  return errorCount === 0 ? 0 : 1;
}<commit_msg>Improved support for build mode.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {ENABLED_RULES} from './rule_groups';
import {Checker} from './third_party/tsetse/checker';
import * as ts from 'typescript';

import {createEmptyExemptionList, ExemptionList} from './exemption_config';
import {reportDiagnostic, reportDiagnosticsWithSummary, reportErrorSummary} from './report';

/** Check if tsec is invoked in the build mode. */
export function isInBuildMode(cmdArgs: string[]) {
  // --build or -b has to be the first argument
  if (cmdArgs.length && cmdArgs[0].charAt(0) === '-') {
    const optionStart = cmdArgs[0].charAt(1) === '-' ? 2 : 1;
    const firstOption = cmdArgs[0].slice(optionStart);
    return firstOption === 'build' || firstOption === 'b';
  }
  return false;
}

/** Perform conformance checks on a single project. */
export function performConformanceCheck(
    program: ts.Program,
    conformanceExemptionConfig: ExemptionList =
        createEmptyExemptionList()): ts.Diagnostic[] {
  const diagnostics = [...ts.getPreEmitDiagnostics(program)];

  // Create all enabled rules with corresponding exemption list entries.
  const conformanceChecker = new Checker(program);
  const conformanceRules = ENABLED_RULES.map(ruleCtr => {
    const allowlistEntries = [];
    const allowlistEntry = conformanceExemptionConfig.get(ruleCtr.RULE_NAME);
    if (allowlistEntry) {
      allowlistEntries.push(allowlistEntry);
    }
    return new ruleCtr({allowlistEntries});
  });

  // Register all rules.
  for (const rule of conformanceRules) {
    rule.register(conformanceChecker);
  }

  // Run all enabled conformance checks and collect errors.
  for (const sf of program.getSourceFiles()) {
    // We don't emit errors for delcarations, so might as well skip checking
    // declaration files all together.
    if (sf.isDeclarationFile) continue;
    const conformanceDiagErr = conformanceChecker.execute(sf).map(
        failure => failure.toDiagnosticWithStringifiedFixes());
    diagnostics.push(...conformanceDiagErr);
  }

  return diagnostics;
}

/** Perform conformance checks on a monorepo. */
export function performBuild(args: string[]): number {
  // This is an internal interface used by the TS compiler.
  interface ParsedBuildCommand {
    buildOptions: ts.BuildOptions;
    watchOptions: ts.WatchOptions|undefined;
    projects: string[];
    errors: ts.Diagnostic[];
  }

  // TypeScript has an API to parse command lines in build mode, but for some
  // reason it's not public. For now we would like to reuse this facility, so
  // we will use type casts to bypass visibility restrictions.
  // TODO(pwng): Replace this hack with our own implementations if this API
  // turns out to be very unstable.
  const parseBuildCommand =
      // tslint:disable-next-line:ban-module-namespace-object-escape
      (ts as unknown as {
        parseBuildCommand: (args: readonly string[]) => ParsedBuildCommand
      }).parseBuildCommand;

  const {buildOptions, projects, errors} = parseBuildCommand(args);

  if (errors.length !== 0) {
    return reportDiagnosticsWithSummary(errors);
  }

  if (projects.length === 0) projects.push('.');

  // A hack to invalidate any build info generated by the vanilla tsc to make
  // sure tsec has a chance to build the project even nothing is modified
  // since last build. `ts.version` is readonly so we have to cast its type.
  // TODO(pwng): Add an additional version suffix to reflect the configurations
  // applied in this tsec run.
  // tslint:disable-next-line:ban-module-namespace-object-escape
  (ts as {version: string}).version += '-tsec';

  const builderHost = ts.createSolutionBuilderHost(
      ts.sys,
      /*createProgram*/ undefined,
      reportDiagnostic,
      /*reportSolutionBuilderStatus*/ undefined,
      reportErrorSummary,
  );

  const allConformanceErrors: ts.Diagnostic[] = [];
  // So far, `afterProgramEmitAndDiagnostics` is the only known API that allows
  // us to hook a callback. Therefore, we can only perform conformance checks
  // after code emission. See b/174168274.
  builderHost.afterProgramEmitAndDiagnostics = (p) => {
    allConformanceErrors.push(...performConformanceCheck(p.getProgram()));
  };
  const builder = ts.createSolutionBuilder(builderHost, projects, buildOptions);
  const exitStatus = buildOptions['clean'] ? builder.clean() : builder.build();

  const freshDate = new Date();
  // Since we do conformance checks after code emission, conformance errors do
  // not prevent JS code from being emitted (if there are no other errors in
  // the TS source file). Therefore, for those TS source files, we need to
  // update their freshness to make sure they will still be built next time.
  // TODO(b/174168274): This can be further optimized when a file containing
  // conformance violations also has semantic errors. For those files, we can
  // directly modify their corresponding `semanticDiagnosticsPerFile` entries in
  // the .tsbuildinfo file, such that the conformance errors are also cached.
  // The ideal solution is to find a better API to insert our conformance check
  // callback. This is blocked by b/174168274.
  for (const error of allConformanceErrors) {
    const sourceFile = error.file;
    if (sourceFile !== undefined) {
      builderHost.setModifiedTime(sourceFile.fileName, freshDate);
    }
  }

  const errorCount = reportDiagnosticsWithSummary(allConformanceErrors);

  return errorCount + exitStatus;
}
{'fraction_non_alphanumeric': 0.10225649814338761, 'fraction_numerical': 0.006283918880319908, 'mean_word_length': 3.727272727272727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5723576', 'n_tokens_mistral': 2941, 'n_tokens_neox': 2858, 'n_words': 1034}