<commit_before>/**
 * @license
 * Copyright (c) 2015 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at
 * http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at
 * http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at
 * http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at
 * http://polymer.github.io/PATENTS.txt
 */

/// <reference path="../custom_typings/main.d.ts" />

import {AnalysisContext} from './core/analysis-context';
import {Document, Package} from './model/model';
import {Parser} from './parser/parser';
import {Scanner} from './scanning/scanner';
import {UrlLoader} from './url-loader/url-loader';
import {UrlResolver} from './url-loader/url-resolver';

export interface Options {
  urlLoader: UrlLoader;
  urlResolver?: UrlResolver;
  parsers?: Map<string, Parser<any>>;
  scanners?: ScannerTable;
  /*
   * Map from url of an HTML Document to another HTML document it lazily depends
   * on.
   */
  lazyEdges?: LazyEdgeMap;
}

/**
 * These are the options available to the `_fork` method.  Currently, only the
 * `urlLoader` override is implemented.
 */
export interface ForkOptions { urlLoader?: UrlLoader; }

export class NoKnownParserError extends Error {};

export type ScannerTable = Map<string, Scanner<any, any, any>[]>;
export type LazyEdgeMap = Map<string, string[]>;

/**
 * A static analyzer for web projects.
 *
 * An Analyzer can load and parse documents of various types, and extract
 * arbitrary information from the documents, and transitively load
 * dependencies. An Analyzer instance is configured with parsers, and scanners
 * which do the actual work of understanding different file types.
 */
export class Analyzer {
  private _context: AnalysisContext;
  constructor(options: Options|AnalysisContext) {
    if (options instanceof AnalysisContext) {
      this._context = options;
    } else {
      this._context = new AnalysisContext(options);
    }
  }

  /**
   * Loads, parses and analyzes the root document of a dependency graph and its
   * transitive dependencies.
   *
   * Note: The analyzer only supports analyzing a single root for now. This
   * is because each analyzed document in the dependency graph has a single
   * root. This mean that we can't properly analyze app-shell-style, lazy
   * loading apps.
   *
   * @param contents Optional contents of the file when it is known without
   * reading it from disk. Clears the caches so that the news contents is used
   * and reanalyzed. Useful for editors that want to re-analyze changed files.
   */
  async analyze(url: string, contents?: string): Promise<Document> {
    if (contents != null) {
      this._context = this._context.filesChanged([url]);
    }
    return this._context.analyze(url, contents);
  }

  async analyzePackage(): Promise<Package> {
    return this._context.analyzePackage();
  }

  /**
   * Clear all cached information from this analyzer instance.
   *
   * Note: if at all possible, instead tell the analyzer about the specific
   * files that changed rather than clearing caches like this. Caching provides
   * large performance gains.
   */
  clearCaches(): void {
    this._context = this._context.clearCaches();
  }

  /**
   * Returns a copy of the analyzer.  If options are given, the AnalysisContext
   * is also forked and individual properties are overridden by the options.
   * is forked with the given options.
   *
   * When the analysis context is forked, its cache is preserved, so any files
   * loaded prior to the fork will remain in the cache until otherwise
   * invalidated.
   *
   * Note: this feature is experimental.
   */
  _fork(options?: ForkOptions): Analyzer {
    const context =
        options ? this._context._fork(undefined, options) : this._context;
    return new Analyzer(context);
  }

  /**
   * Loads the content at the provided resolved URL.
   *
   * Currently does no caching. If the provided contents are given then they
   * are used instead of hitting the UrlLoader (e.g. when you have in-memory
   * contents that should override disk).
   */
  async load(resolvedUrl: string, providedContents?: string) {
    return this._context.load(resolvedUrl, providedContents);
  }<commit_msg>Updated fork comment text to use @rictic's suggestion (is better).<commit_after>/**
 * @license
 * Copyright (c) 2015 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at
 * http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at
 * http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at
 * http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at
 * http://polymer.github.io/PATENTS.txt
 */

/// <reference path="../custom_typings/main.d.ts" />

import {AnalysisContext} from './core/analysis-context';
import {Document, Package} from './model/model';
import {Parser} from './parser/parser';
import {Scanner} from './scanning/scanner';
import {UrlLoader} from './url-loader/url-loader';
import {UrlResolver} from './url-loader/url-resolver';

export interface Options {
  urlLoader: UrlLoader;
  urlResolver?: UrlResolver;
  parsers?: Map<string, Parser<any>>;
  scanners?: ScannerTable;
  /*
   * Map from url of an HTML Document to another HTML document it lazily depends
   * on.
   */
  lazyEdges?: LazyEdgeMap;
}

/**
 * These are the options available to the `_fork` method.  Currently, only the
 * `urlLoader` override is implemented.
 */
export interface ForkOptions { urlLoader?: UrlLoader; }

export class NoKnownParserError extends Error {};

export type ScannerTable = Map<string, Scanner<any, any, any>[]>;
export type LazyEdgeMap = Map<string, string[]>;

/**
 * A static analyzer for web projects.
 *
 * An Analyzer can load and parse documents of various types, and extract
 * arbitrary information from the documents, and transitively load
 * dependencies. An Analyzer instance is configured with parsers, and scanners
 * which do the actual work of understanding different file types.
 */
export class Analyzer {
  private _context: AnalysisContext;
  constructor(options: Options|AnalysisContext) {
    if (options instanceof AnalysisContext) {
      this._context = options;
    } else {
      this._context = new AnalysisContext(options);
    }
  }

  /**
   * Loads, parses and analyzes the root document of a dependency graph and its
   * transitive dependencies.
   *
   * Note: The analyzer only supports analyzing a single root for now. This
   * is because each analyzed document in the dependency graph has a single
   * root. This mean that we can't properly analyze app-shell-style, lazy
   * loading apps.
   *
   * @param contents Optional contents of the file when it is known without
   * reading it from disk. Clears the caches so that the news contents is used
   * and reanalyzed. Useful for editors that want to re-analyze changed files.
   */
  async analyze(url: string, contents?: string): Promise<Document> {
    if (contents != null) {
      this._context = this._context.filesChanged([url]);
    }
    return this._context.analyze(url, contents);
  }

  async analyzePackage(): Promise<Package> {
    return this._context.analyzePackage();
  }

  /**
   * Clear all cached information from this analyzer instance.
   *
   * Note: if at all possible, instead tell the analyzer about the specific
   * files that changed rather than clearing caches like this. Caching provides
   * large performance gains.
   */
  clearCaches(): void {
    this._context = this._context.clearCaches();
  }

  /**
   * Returns a copy of the analyzer.  If options are given, the AnalysisContext
   * is also forked and individual properties are overridden by the options.
   * is forked with the given options.
   *
   * When the analysis context is forked, its cache is preserved, so you will
   * see a mixture of pre-fork and post-fork contents when you analyze with a
   * forked analyzer.
   *
   * Note: this feature is experimental.
   */
  _fork(options?: ForkOptions): Analyzer {
    const context =
        options ? this._context._fork(undefined, options) : this._context;
    return new Analyzer(context);
  }

  /**
   * Loads the content at the provided resolved URL.
   *
   * Currently does no caching. If the provided contents are given then they
   * are used instead of hitting the UrlLoader (e.g. when you have in-memory
   * contents that should override disk).
   */
  async load(resolvedUrl: string, providedContents?: string) {
    return this._context.load(resolvedUrl, providedContents);
  }
{'fraction_non_alphanumeric': 0.0921570833802183, 'fraction_numerical': 0.000900191290649263, 'mean_word_length': 3.5277636271013755, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3342718', 'n_tokens_mistral': 2459, 'n_tokens_neox': 2377, 'n_words': 987}