<commit_before>package org.jetbrains.plugins.textmate.language.syntax;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.containers.StringInterner;
import gnu.trove.THashMap;
import gnu.trove.TIntObjectHashMap;
import gnu.trove.TObjectIntHashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.plugins.textmate.Constants;
import org.jetbrains.plugins.textmate.plist.PListValue;
import org.jetbrains.plugins.textmate.plist.Plist;
import org.jetbrains.plugins.textmate.regex.RegexFacade;

import java.util.Map;

/**
 * User: zolotov
 * <p/>
 * Table of textmate syntax rules.
 * Table represents mapping from scopeNames to set of syntax rules {@link SyntaxNodeDescriptor}.
 * <p/>
 * In order to lexing some file with this rules you should retrieve syntax rule
 * by scope name of target language {@link this#getSyntax(String)}.
 * <p/>
 * Scope name of target language can be find in syntax files of TextMate bundles.
 */
public class TextMateSyntaxTable {
  private static final Logger LOG = Logger.getInstance(TextMateSyntaxTable.class);
  private final Map<String, SyntaxNodeDescriptor> rulesMap = new THashMap<>();
  private TObjectIntHashMap<String> ruleIds;
  private StringInterner interner;

  public void compact() {
    ruleIds = null;
    interner = null;
  }

  /**
   * Append table with new syntax rules in order to support new language.
   *
   * @param plist Plist represented syntax file (*.tmLanguage) of target language.
   * @return language scope root name
   */
  @NotNull
  public String loadSyntax(Plist plist) {
    final SyntaxNodeDescriptor rootSyntaxNode = loadRealNode(plist, null);
    return rootSyntaxNode.getScopeName();
  }

  /**
   * Returns root syntax rule by scope name.
   *
   * @param scopeName Name of scope defined for some language.
   * @return root syntax rule from table for language with given scope name.
   * If tables doesn't contain syntax rule for given scope,
   * method returns {@link SyntaxNodeDescriptor#EMPTY_NODE}.
   */
  @NotNull
  public SyntaxNodeDescriptor getSyntax(String scopeName) {
    SyntaxNodeDescriptor syntaxNodeDescriptor = rulesMap.get(scopeName);
    if (syntaxNodeDescriptor == null) {
      LOG.debug("Can't find syntax node for scope: '" + scopeName + "'");
      return SyntaxNodeDescriptor.EMPTY_NODE;
    }
    return syntaxNodeDescriptor;
  }

  public void clear() {
    rulesMap.clear();
  }

  private SyntaxNodeDescriptor loadNestedSyntax(Plist plist, SyntaxNodeDescriptor parentNode) {
    return plist.contains(Constants.INCLUDE_KEY) ? loadProxyNode(plist, parentNode) : loadRealNode(plist, parentNode);
  }

  @NotNull
  private SyntaxNodeDescriptor loadRealNode(Plist plist, SyntaxNodeDescriptor parentNode) {
    MutableSyntaxNodeDescriptor result = new SyntaxNodeDescriptorImpl(parentNode);
    for (Map.Entry<String, PListValue> entry : plist.entries()) {
      PListValue pListValue = entry.getValue();
      if (pListValue != null) {
        String key = intern(entry.getKey());
        if (ArrayUtil.contains(key, Constants.REGEX_KEY_NAMES)) {
          String pattern = pListValue.getString();
          if (pattern != null) {
            result.setRegexAttribute(key, RegexFacade.regex(pattern));
          }
        }
        else if (ArrayUtil.contains(key, Constants.STRING_KEY_NAMES)) {
          if (key.equals(Constants.WHILE_KEY) || key.equals(Constants.END_KEY)) {
            result.setStringAttribute(key, pListValue.getString());
          }
          else {
            result.setStringAttribute(key, intern(pListValue.getString()));
          }
        }
        else if (ArrayUtil.contains(key, Constants.CAPTURES_KEY_NAMES)) {
          result.setCaptures(key, loadCaptures(pListValue.getPlist()));
        }
        else if (Constants.REPOSITORY_KEY.equalsIgnoreCase(key)) {
          loadRepository(result, pListValue);
        }
        else if (Constants.PATTERNS_KEY.equalsIgnoreCase(key)) {
          loadPatterns(result, pListValue);
        }
        else if (Constants.INJECTIONS_KEY.equalsIgnoreCase(key)) {
          loadInjections(result, pListValue);
        }
      }
    }
    if (plist.contains(Constants.SCOPE_NAME_KEY)) {
      final String scopeName = plist.getPlistValue(Constants.SCOPE_NAME_KEY, Constants.DEFAULT_SCOPE_NAME).getString();
      result.setScopeName(scopeName);
      rulesMap.put(scopeName, result);
    }
    result.compact();
    return result;
  }
<commit_msg>TextMate: intern all string attribute values<commit_after>package org.jetbrains.plugins.textmate.language.syntax;

import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.util.ArrayUtil;
import com.intellij.util.containers.StringInterner;
import gnu.trove.THashMap;
import gnu.trove.TIntObjectHashMap;
import gnu.trove.TObjectIntHashMap;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.plugins.textmate.Constants;
import org.jetbrains.plugins.textmate.plist.PListValue;
import org.jetbrains.plugins.textmate.plist.Plist;
import org.jetbrains.plugins.textmate.regex.RegexFacade;

import java.util.Map;

/**
 * User: zolotov
 * <p/>
 * Table of textmate syntax rules.
 * Table represents mapping from scopeNames to set of syntax rules {@link SyntaxNodeDescriptor}.
 * <p/>
 * In order to lexing some file with this rules you should retrieve syntax rule
 * by scope name of target language {@link this#getSyntax(String)}.
 * <p/>
 * Scope name of target language can be find in syntax files of TextMate bundles.
 */
public class TextMateSyntaxTable {
  private static final Logger LOG = Logger.getInstance(TextMateSyntaxTable.class);
  private final Map<String, SyntaxNodeDescriptor> rulesMap = new THashMap<>();
  private TObjectIntHashMap<String> ruleIds;
  private StringInterner interner;

  public void compact() {
    ruleIds = null;
    interner = null;
  }

  /**
   * Append table with new syntax rules in order to support new language.
   *
   * @param plist Plist represented syntax file (*.tmLanguage) of target language.
   * @return language scope root name
   */
  @NotNull
  public String loadSyntax(Plist plist) {
    final SyntaxNodeDescriptor rootSyntaxNode = loadRealNode(plist, null);
    return rootSyntaxNode.getScopeName();
  }

  /**
   * Returns root syntax rule by scope name.
   *
   * @param scopeName Name of scope defined for some language.
   * @return root syntax rule from table for language with given scope name.
   * If tables doesn't contain syntax rule for given scope,
   * method returns {@link SyntaxNodeDescriptor#EMPTY_NODE}.
   */
  @NotNull
  public SyntaxNodeDescriptor getSyntax(String scopeName) {
    SyntaxNodeDescriptor syntaxNodeDescriptor = rulesMap.get(scopeName);
    if (syntaxNodeDescriptor == null) {
      LOG.debug("Can't find syntax node for scope: '" + scopeName + "'");
      return SyntaxNodeDescriptor.EMPTY_NODE;
    }
    return syntaxNodeDescriptor;
  }

  public void clear() {
    rulesMap.clear();
  }

  private SyntaxNodeDescriptor loadNestedSyntax(Plist plist, SyntaxNodeDescriptor parentNode) {
    return plist.contains(Constants.INCLUDE_KEY) ? loadProxyNode(plist, parentNode) : loadRealNode(plist, parentNode);
  }

  @NotNull
  private SyntaxNodeDescriptor loadRealNode(Plist plist, SyntaxNodeDescriptor parentNode) {
    MutableSyntaxNodeDescriptor result = new SyntaxNodeDescriptorImpl(parentNode);
    for (Map.Entry<String, PListValue> entry : plist.entries()) {
      PListValue pListValue = entry.getValue();
      if (pListValue != null) {
        String key = intern(entry.getKey());
        if (ArrayUtil.contains(key, Constants.REGEX_KEY_NAMES)) {
          String pattern = pListValue.getString();
          if (pattern != null) {
            result.setRegexAttribute(key, RegexFacade.regex(pattern));
          }
        }
        else if (ArrayUtil.contains(key, Constants.STRING_KEY_NAMES)) {
          result.setStringAttribute(key, intern(pListValue.getString()));
        }
        else if (ArrayUtil.contains(key, Constants.CAPTURES_KEY_NAMES)) {
          result.setCaptures(key, loadCaptures(pListValue.getPlist()));
        }
        else if (Constants.REPOSITORY_KEY.equalsIgnoreCase(key)) {
          loadRepository(result, pListValue);
        }
        else if (Constants.PATTERNS_KEY.equalsIgnoreCase(key)) {
          loadPatterns(result, pListValue);
        }
        else if (Constants.INJECTIONS_KEY.equalsIgnoreCase(key)) {
          loadInjections(result, pListValue);
        }
      }
    }
    if (plist.contains(Constants.SCOPE_NAME_KEY)) {
      final String scopeName = plist.getPlistValue(Constants.SCOPE_NAME_KEY, Constants.DEFAULT_SCOPE_NAME).getString();
      result.setScopeName(scopeName);
      rulesMap.put(scopeName, result);
    }
    result.compact();
    return result;
  }

{'fraction_non_alphanumeric': 0.09715745058849656, 'fraction_numerical': 0.0, 'mean_word_length': 3.7181770560502883, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '987852', 'n_tokens_mistral': 2670, 'n_tokens_neox': 2601, 'n_words': 633}