<commit_before>package org.forge.text.syntax.encoder;

import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import org.forge.text.syntax.Encoder;
import org.forge.text.syntax.Theme;
import org.forge.text.syntax.TokenType;
import org.junit.Assert;

public class AssertEncoder extends Encoder.AbstractEncoder {

   public AssertEncoder(OutputStream out, Theme theme) {
      super(out, theme);
      textTokens.clear();
   }

   @Override
   public void textToken(String text, TokenType type) {
      textTokens.add(new TokenPair(text, type));
   }

   @Override
   public void beginGroup(TokenType type) {
   }

   @Override
   public void endGroup(TokenType type) {
   }

   @Override
   public void beginLine(TokenType type) {
   }

   @Override
   public void endLine(TokenType type) {
   }

   private static List<TokenPair> textTokens = new ArrayList<TokenPair>();

   private static class TokenPair {
      private String text;
      private TokenType type;

      public TokenPair(String text, TokenType type) {
         this.text = text;
         this.type= type;
      }

      @Override
      public String toString() {
         return "[text=" + text + ", type=" + type + "]";
      }<commit_msg>Make AssertEncoder extend DebugEncoder for easy debug in IDE<commit_after>package org.forge.text.syntax.encoder;

import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;

import org.forge.text.syntax.Theme;
import org.forge.text.syntax.TokenType;
import org.junit.Assert;

public class AssertEncoder extends DebugEncoder {

   public AssertEncoder(OutputStream out, Theme theme) {
      super(out, theme);
      textTokens.clear();
   }

   @Override
   public void textToken(String text, TokenType type) {
      textTokens.add(new TokenPair(text, type));
      super.textToken(text, type);
   }

   private static List<TokenPair> textTokens = new ArrayList<TokenPair>();

   private static class TokenPair {
      private String text;
      private TokenType type;

      public TokenPair(String text, TokenType type) {
         this.text = text;
         this.type= type;
      }

      @Override
      public String toString() {
         return "[text=" + text + ", type=" + type + "]";
      }
{'fraction_non_alphanumeric': 0.1024390243902439, 'fraction_numerical': 0.0, 'mean_word_length': 3.0943738656987296, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5562513', 'n_tokens_mistral': 664, 'n_tokens_neox': 639, 'n_words': 174}