<commit_before>/*
 * Copyright 2015 The Closure Compiler Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.javascript.jscomp.parsing.parser;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.errorprone.annotations.Immutable;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;

/**
 * Represents various aspects of language version and support.
 *
 * <p>This is somewhat redundant with LanguageMode, but is separate
 * for two reasons: (1) it's used for parsing, which cannot
 * depend on LanguageMode, and (2) it's concerned with slightly
 * different nuances: implemented features and modules rather
 * than strictness.
 *
 * <p>In the long term, it would be good to disentangle all these
 * concerns and pull out a single LanguageSyntax enum with a
 * separate strict mode flag, and then these could possibly be
 * unified.
 *
 * <p>Instances of this class are immutable.
 */
@Immutable
public final class FeatureSet implements Serializable {
  private final ImmutableSet<Feature> features;

  /** The bare minimum set of features. */
  public static final FeatureSet BARE_MINIMUM = new FeatureSet(ImmutableSet.<Feature>of());

  /** Features from ES3. */
  public static final FeatureSet ES3 = BARE_MINIMUM.with(LangVersion.ES3.features());

  /** Features from ES5 only. */
  public static final FeatureSet ES5 = ES3.with(LangVersion.ES5.features());

  /** All ES6 features, including modules. */
  public static final FeatureSet ES6_MODULES = ES5.with(LangVersion.ES6.features());

  /** The full set of ES6 features, not including modules. */
  public static final FeatureSet ES6 = ES6_MODULES.without(Feature.MODULES);

  public static final FeatureSet ES7_MODULES = ES6_MODULES.with(LangVersion.ES7.features());

  public static final FeatureSet ES7 = ES7_MODULES.without(Feature.MODULES);

  public static final FeatureSet ES8_MODULES = ES7_MODULES.with(LangVersion.ES8.features());

  public static final FeatureSet ES8 = ES8_MODULES.without(Feature.MODULES);

  public static final FeatureSet TYPESCRIPT = ES8_MODULES.with(LangVersion.TYPESCRIPT.features());

  public static final FeatureSet NTI_SUPPORTED =
      ES5.with(
          ImmutableSet.<Feature>of(
              Feature.COMPUTED_PROPERTIES,
              Feature.EXPONENT_OP,
              Feature.EXTENDED_OBJECT_LITERALS,
              Feature.FOR_OF,
              Feature.GENERATORS,
              Feature.MEMBER_DECLARATIONS,
              Feature.TEMPLATE_LITERALS));

  private enum LangVersion {
    ES3,
    ES5,
    ES6,
    ES7,
    ES8,
    TYPESCRIPT;

    private Set<Feature> features() {
      Set<Feature> set = new HashSet<>();
      for (Feature feature : Feature.values()) {
        if (feature.version == this) {
          set.add(feature);
        }
      }
      return set;
    }
  }

  /** Specific features that can be included in a FeatureSet. */
  public enum Feature {
    // ES3 features
    // Functions can be declared in the block scope for all ES versions. However, for ES3 and ES5,
    // we want to hoist the functions from the block scope by redeclaring them as vars (i.e.
    // { function f() {} } becomes { var f = function {} }. To prevent this feature from causing
    // code to run additional transpilation passes beyond rewriting block scoped functions, we mark
    // block scoped functions as an ES3 feature and then manually determine whether to rewrite
    // functions inside the processTranspile method of TranspilationPasses.java
    BLOCK_SCOPED_FUNCTION_DECLARATION("block function", LangVersion.ES3),

    // ES5 features
    ES3_KEYWORDS_AS_IDENTIFIERS("ES3 keywords as identifiers", LangVersion.ES5),
    GETTER("getters", LangVersion.ES5),
    KEYWORDS_AS_PROPERTIES("reserved words as properties", LangVersion.ES5),
    SETTER("setters", LangVersion.ES5),
    STRING_CONTINUATION("string continuation", LangVersion.ES5),
    TRAILING_COMMA("trailing comma", LangVersion.ES5),

    // ES6 features (besides modules): all stable browsers are now fully compliant
    ARROW_FUNCTIONS("arrow function", LangVersion.ES6),
    BINARY_LITERALS("binary literal", LangVersion.ES6),
    OCTAL_LITERALS("octal literal", LangVersion.ES6),
    CLASSES("class", LangVersion.ES6),
    COMPUTED_PROPERTIES("computed property", LangVersion.ES6),
    EXTENDED_OBJECT_LITERALS("extended object literal", LangVersion.ES6),
    FOR_OF("for-of loop", LangVersion.ES6),
    GENERATORS("generator", LangVersion.ES6),
    LET_DECLARATIONS("let declaration", LangVersion.ES6),
    MEMBER_DECLARATIONS("member declaration", LangVersion.ES6),
    REGEXP_FLAG_Y("RegExp flag 'y'", LangVersion.ES6),
    ARRAY_PATTERN_REST("array pattern rest", LangVersion.ES6),
    REST_PARAMETERS("rest parameter", LangVersion.ES6),
    SPREAD_EXPRESSIONS("spread expression", LangVersion.ES6),
    SUPER("super", LangVersion.ES6),
    TEMPLATE_LITERALS("template literal", LangVersion.ES6),
    CONST_DECLARATIONS("const declaration", LangVersion.ES6),
    DESTRUCTURING("destructuring", LangVersion.ES6),
    NEW_TARGET("new.target", LangVersion.ES6),
    REGEXP_FLAG_U("RegExp flag 'u'", LangVersion.ES6),
    DEFAULT_PARAMETERS("default parameter", LangVersion.ES6),
    MODULES("modules", LangVersion.ES6),

    // ES 2016 only added one new feature:
    EXPONENT_OP("exponent operator (**)", LangVersion.ES7),

    // ES 2017 features:
    ASYNC_FUNCTIONS("async function", LangVersion.ES8),
    TRAILING_COMMA_IN_PARAM_LIST("trailing comma in param list", LangVersion.ES8),

    // ES6 typed features that are not at all implemented in browsers
    ACCESSIBILITY_MODIFIER("accessibility modifier", LangVersion.TYPESCRIPT),
    AMBIENT_DECLARATION("ambient declaration", LangVersion.TYPESCRIPT),
    CALL_SIGNATURE("call signature", LangVersion.TYPESCRIPT),
    CONSTRUCTOR_SIGNATURE("constructor signature", LangVersion.TYPESCRIPT),
    ENUM("enum", LangVersion.TYPESCRIPT),
    GENERICS("generics", LangVersion.TYPESCRIPT),
    IMPLEMENTS("implements", LangVersion.TYPESCRIPT),
    INDEX_SIGNATURE("index signature", LangVersion.TYPESCRIPT),
    INTERFACE("interface", LangVersion.TYPESCRIPT),
    MEMBER_VARIABLE_IN_CLASS("member variable in class", LangVersion.TYPESCRIPT),
    NAMESPACE_DECLARATION("namespace declaration", LangVersion.TYPESCRIPT),
    OPTIONAL_PARAMETER("optional parameter", LangVersion.TYPESCRIPT),
    TYPE_ALIAS("type alias", LangVersion.TYPESCRIPT),
    TYPE_ANNOTATION("type annotation", LangVersion.TYPESCRIPT);

    private final String name;
    private final LangVersion version;

    private Feature(String name, LangVersion version) {
      this.name = name;
      this.version = version;
    }

    @Override
    public String toString() {
      return name;
    }
  }

  private FeatureSet(Set<Feature> features) {
    this.features = ImmutableSet.copyOf(features);
  }

  /** Returns a string representation suitable for encoding in depgraph and deps.js files. */
  public String version() {
    if (ES3.contains(this)) {
      return "es3";
    }
    if (ES5.contains(this)) {
      return "es5";
    }
    if (ES6_MODULES.contains(this)) {
      return "es6";
    }
    if (ES7_MODULES.contains(this)) {<commit_msg>Add version string to NTI_SUPPORTED.<commit_after>/*
 * Copyright 2015 The Closure Compiler Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.javascript.jscomp.parsing.parser;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
import com.google.errorprone.annotations.Immutable;
import java.io.Serializable;
import java.util.HashSet;
import java.util.Set;

/**
 * Represents various aspects of language version and support.
 *
 * <p>This is somewhat redundant with LanguageMode, but is separate
 * for two reasons: (1) it's used for parsing, which cannot
 * depend on LanguageMode, and (2) it's concerned with slightly
 * different nuances: implemented features and modules rather
 * than strictness.
 *
 * <p>In the long term, it would be good to disentangle all these
 * concerns and pull out a single LanguageSyntax enum with a
 * separate strict mode flag, and then these could possibly be
 * unified.
 *
 * <p>Instances of this class are immutable.
 */
@Immutable
public final class FeatureSet implements Serializable {
  private final ImmutableSet<Feature> features;

  /** The bare minimum set of features. */
  public static final FeatureSet BARE_MINIMUM = new FeatureSet(ImmutableSet.<Feature>of());

  /** Features from ES3. */
  public static final FeatureSet ES3 = BARE_MINIMUM.with(LangVersion.ES3.features());

  /** Features from ES5 only. */
  public static final FeatureSet ES5 = ES3.with(LangVersion.ES5.features());

  /** All ES6 features, including modules. */
  public static final FeatureSet ES6_MODULES = ES5.with(LangVersion.ES6.features());

  /** The full set of ES6 features, not including modules. */
  public static final FeatureSet ES6 = ES6_MODULES.without(Feature.MODULES);

  public static final FeatureSet ES7_MODULES = ES6_MODULES.with(LangVersion.ES7.features());

  public static final FeatureSet ES7 = ES7_MODULES.without(Feature.MODULES);

  public static final FeatureSet ES8_MODULES = ES7_MODULES.with(LangVersion.ES8.features());

  public static final FeatureSet ES8 = ES8_MODULES.without(Feature.MODULES);

  public static final FeatureSet TYPESCRIPT = ES8_MODULES.with(LangVersion.TYPESCRIPT.features());

  // TODO(b/64536685): Remove this FeatureSet once NTI supports all of ES6.
  public static final FeatureSet NTI_SUPPORTED =
      ES5.with(
          ImmutableSet.<Feature>of(
              Feature.COMPUTED_PROPERTIES,
              Feature.EXPONENT_OP,
              Feature.EXTENDED_OBJECT_LITERALS,
              Feature.FOR_OF,
              Feature.GENERATORS,
              Feature.MEMBER_DECLARATIONS,
              Feature.TEMPLATE_LITERALS));

  private enum LangVersion {
    ES3,
    ES5,
    ES6,
    ES7,
    ES8,
    TYPESCRIPT;

    private Set<Feature> features() {
      Set<Feature> set = new HashSet<>();
      for (Feature feature : Feature.values()) {
        if (feature.version == this) {
          set.add(feature);
        }
      }
      return set;
    }
  }

  /** Specific features that can be included in a FeatureSet. */
  public enum Feature {
    // ES3 features
    // Functions can be declared in the block scope for all ES versions. However, for ES3 and ES5,
    // we want to hoist the functions from the block scope by redeclaring them as vars (i.e.
    // { function f() {} } becomes { var f = function {} }. To prevent this feature from causing
    // code to run additional transpilation passes beyond rewriting block scoped functions, we mark
    // block scoped functions as an ES3 feature and then manually determine whether to rewrite
    // functions inside the processTranspile method of TranspilationPasses.java
    BLOCK_SCOPED_FUNCTION_DECLARATION("block function", LangVersion.ES3),

    // ES5 features
    ES3_KEYWORDS_AS_IDENTIFIERS("ES3 keywords as identifiers", LangVersion.ES5),
    GETTER("getters", LangVersion.ES5),
    KEYWORDS_AS_PROPERTIES("reserved words as properties", LangVersion.ES5),
    SETTER("setters", LangVersion.ES5),
    STRING_CONTINUATION("string continuation", LangVersion.ES5),
    TRAILING_COMMA("trailing comma", LangVersion.ES5),

    // ES6 features (besides modules): all stable browsers are now fully compliant
    ARROW_FUNCTIONS("arrow function", LangVersion.ES6),
    BINARY_LITERALS("binary literal", LangVersion.ES6),
    OCTAL_LITERALS("octal literal", LangVersion.ES6),
    CLASSES("class", LangVersion.ES6),
    COMPUTED_PROPERTIES("computed property", LangVersion.ES6),
    EXTENDED_OBJECT_LITERALS("extended object literal", LangVersion.ES6),
    FOR_OF("for-of loop", LangVersion.ES6),
    GENERATORS("generator", LangVersion.ES6),
    LET_DECLARATIONS("let declaration", LangVersion.ES6),
    MEMBER_DECLARATIONS("member declaration", LangVersion.ES6),
    REGEXP_FLAG_Y("RegExp flag 'y'", LangVersion.ES6),
    ARRAY_PATTERN_REST("array pattern rest", LangVersion.ES6),
    REST_PARAMETERS("rest parameter", LangVersion.ES6),
    SPREAD_EXPRESSIONS("spread expression", LangVersion.ES6),
    SUPER("super", LangVersion.ES6),
    TEMPLATE_LITERALS("template literal", LangVersion.ES6),
    CONST_DECLARATIONS("const declaration", LangVersion.ES6),
    DESTRUCTURING("destructuring", LangVersion.ES6),
    NEW_TARGET("new.target", LangVersion.ES6),
    REGEXP_FLAG_U("RegExp flag 'u'", LangVersion.ES6),
    DEFAULT_PARAMETERS("default parameter", LangVersion.ES6),
    MODULES("modules", LangVersion.ES6),

    // ES 2016 only added one new feature:
    EXPONENT_OP("exponent operator (**)", LangVersion.ES7),

    // ES 2017 features:
    ASYNC_FUNCTIONS("async function", LangVersion.ES8),
    TRAILING_COMMA_IN_PARAM_LIST("trailing comma in param list", LangVersion.ES8),

    // ES6 typed features that are not at all implemented in browsers
    ACCESSIBILITY_MODIFIER("accessibility modifier", LangVersion.TYPESCRIPT),
    AMBIENT_DECLARATION("ambient declaration", LangVersion.TYPESCRIPT),
    CALL_SIGNATURE("call signature", LangVersion.TYPESCRIPT),
    CONSTRUCTOR_SIGNATURE("constructor signature", LangVersion.TYPESCRIPT),
    ENUM("enum", LangVersion.TYPESCRIPT),
    GENERICS("generics", LangVersion.TYPESCRIPT),
    IMPLEMENTS("implements", LangVersion.TYPESCRIPT),
    INDEX_SIGNATURE("index signature", LangVersion.TYPESCRIPT),
    INTERFACE("interface", LangVersion.TYPESCRIPT),
    MEMBER_VARIABLE_IN_CLASS("member variable in class", LangVersion.TYPESCRIPT),
    NAMESPACE_DECLARATION("namespace declaration", LangVersion.TYPESCRIPT),
    OPTIONAL_PARAMETER("optional parameter", LangVersion.TYPESCRIPT),
    TYPE_ALIAS("type alias", LangVersion.TYPESCRIPT),
    TYPE_ANNOTATION("type annotation", LangVersion.TYPESCRIPT);

    private final String name;
    private final LangVersion version;

    private Feature(String name, LangVersion version) {
      this.name = name;
      this.version = version;
    }

    @Override
    public String toString() {
      return name;
    }
  }

  private FeatureSet(Set<Feature> features) {
    this.features = ImmutableSet.copyOf(features);
  }

  /** Returns a string representation suitable for encoding in depgraph and deps.js files. */
  public String version() {
    if (ES3.contains(this)) {
      return "es3";
    }
    if (ES5.contains(this)) {
      return "es5";
    }
    if (ES6_MODULES.contains(this)) {
      return "es6";
    }
    if (NTI_SUPPORTED.contains(this)) {
      return "ntiSupported";
    }
    if (ES7_MODULES.contains(this)) {
{'fraction_non_alphanumeric': 0.0986130550960682, 'fraction_numerical': 0.012915129151291513, 'mean_word_length': 3.924498746867168, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1563941', 'n_tokens_mistral': 4710, 'n_tokens_neox': 4564, 'n_words': 1310}