<commit_before>package com.intellij.psi.impl.source.parsing;

import com.intellij.FileSetTestCase;
import com.intellij.lang.ASTFactory;
import com.intellij.lexer.FilterLexer;
import com.intellij.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.application.ex.PathManagerEx;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.impl.source.tree.*;
import com.intellij.psi.tree.IElementType;
import junit.framework.Test;

/**
 *  @author dsl
 */
public class GTTokensTest extends FileSetTestCase {
  public GTTokensTest() {
    super(PathManagerEx.getTestDataPath(PathManagerEx.TestDataLookupStrategy.COMMUNITY_FROM_ULTIMATE) + "/psi/gt-tokens");
  }

  @Override
  public String transform(String testName, String[] data) throws Exception {
    final Lexer lexer = new FilterLexer(new JavaLexer(LanguageLevel.HIGHEST),
                                        new FilterLexer.SetFilter(StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET));
    StringBuffer result = new StringBuffer();
    lexer.start(data[0]);
    final CompositeElement parent = ASTFactory.composite(ElementType.CODE_FRAGMENT);
    while(true) {<commit_msg>path to test data fixed<commit_after>package com.intellij.psi.impl.source.parsing;

import com.intellij.FileSetTestCase;
import com.intellij.lang.ASTFactory;
import com.intellij.lexer.FilterLexer;
import com.intellij.lexer.JavaLexer;
import com.intellij.lexer.Lexer;
import com.intellij.openapi.application.ex.PathManagerEx;
import com.intellij.pom.java.LanguageLevel;
import com.intellij.psi.impl.source.tree.*;
import com.intellij.psi.tree.IElementType;
import junit.framework.Test;

/**
 *  @author dsl
 */
public class GTTokensTest extends FileSetTestCase {
  public GTTokensTest() {
    super(PathManagerEx.getTestDataPath(GTTokensTest.class) + "/psi/gt-tokens");
  }

  @Override
  public String transform(String testName, String[] data) throws Exception {
    final Lexer lexer = new FilterLexer(new JavaLexer(LanguageLevel.HIGHEST),
                                        new FilterLexer.SetFilter(StdTokenSets.WHITE_SPACE_OR_COMMENT_BIT_SET));
    StringBuffer result = new StringBuffer();
    lexer.start(data[0]);
    final CompositeElement parent = ASTFactory.composite(ElementType.CODE_FRAGMENT);
    while(true) {
{'fraction_non_alphanumeric': 0.10358391608391608, 'fraction_numerical': 0.0008741258741258741, 'mean_word_length': 4.582926829268293, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7323838', 'n_tokens_mistral': 750, 'n_tokens_neox': 740, 'n_words': 124}