<commit_before>package com.senseidb.clue.commands;

import java.io.PrintStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;

import com.senseidb.clue.ClueContext;

public class TermsCommand extends ClueCommand {

  public TermsCommand(ClueContext ctx) {
    super(ctx);
  }

  @Override
  public String getName() {
    return "terms";
  }

  @Override
  public String help() {
    return "gets terms from the index, <field:term>, term can be a prefix";
  }

  @Override
  public void execute(String[] args, PrintStream out) throws Exception {
    String field = null;
    String termVal = null;
    try{
      field = args[0];
    }
    catch(Exception e){
      field = null;
    }
    
    if (field != null){
      String[] parts = field.split(":");
      if (parts.length > 1){
        field = parts[0];
        termVal = parts[1];
      }
    }
    
    IndexReader reader = ctx.getIndexReader();
    List<AtomicReaderContext> leaves = reader.leaves();
    TreeMap<BytesRef,TermsEnum> termMap = null;
    HashMap<BytesRef,AtomicInteger> termCountMap = new HashMap<BytesRef,AtomicInteger>();

    int numCount = 0;
    int numPerPage = 20;
    
    for (AtomicReaderContext leaf : leaves){
      AtomicReader atomicReader = leaf.reader();
      
      if (field == null){
        continue;
      }
      
      Terms terms = atomicReader.fields().terms(field);
      
      if (terms == null) {
        continue;
      }
      
      if (termMap == null){
        termMap = new TreeMap<BytesRef,TermsEnum>(terms.getComparator());
      }
      
      
      TermsEnum te = terms.iterator(null);
      BytesRef termBytes;
      if (termVal != null){
        te.seekCeil(new BytesRef(termVal));
        termBytes = te.term();
      }
      else{
        termBytes = te.next();
      }
      
      while(true){
        if (termBytes == null) break;        
        AtomicInteger count = termCountMap.get(termBytes);
        if (count == null){
          termCountMap.put(termBytes, new AtomicInteger(te.docFreq()));
          termMap.put(termBytes, te);
          break;
        }
        count.getAndAdd(te.docFreq());
        termBytes = te.next();
      }
    }
    
    while(termMap != null && !termMap.isEmpty()){
      numCount++;
      Entry<BytesRef,TermsEnum> entry = termMap.pollFirstEntry();
      if (entry == null) break;
      BytesRef key = entry.getKey();
      AtomicInteger count = termCountMap.remove(key);
      out.println(key.utf8ToString()+" ("+count+") ");
      if (ctx.isInteractiveMode()){
        if (numCount % numPerPage == 0){
          out.println("Ctrl-D to break");
          int ch = System.in.read();
          if (ch == -1) {
            out.flush();
            return;
          }
        }
      }
      TermsEnum te = entry.getValue();
      BytesRef nextKey = te.next();
      while(true){
        if (nextKey == null) break;
        count = termCountMap.get(nextKey);
        if (count == null){
          termCountMap.put(nextKey, new AtomicInteger(te.docFreq()));
          termMap.put(nextKey, te);
          break;
        }
        count.getAndAdd(te.docFreq());
        nextKey = te.next();
      }
    }
    out.flush();
  }

}<commit_msg>fixed terms<commit_after>package com.senseidb.clue.commands;

import java.io.PrintStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;

import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;

import com.senseidb.clue.ClueContext;

public class TermsCommand extends ClueCommand {

  public TermsCommand(ClueContext ctx) {
    super(ctx);
  }

  @Override
  public String getName() {
    return "terms";
  }

  @Override
  public String help() {
    return "gets terms from the index, <field:term>, term can be a prefix";
  }

  @Override
  public void execute(String[] args, PrintStream out) throws Exception {
    String field = null;
    String termVal = null;
    try{
      field = args[0];
    }
    catch(Exception e){
      field = null;
    }
    
    if (field != null){
      String[] parts = field.split(":");
      if (parts.length > 1){
        field = parts[0];
        termVal = parts[1];
      }
    }
    else{
      out.println("Usage: field:value");
      return;
    }
    
    boolean isExact = false;
    
    if (termVal != null){
      if (termVal.endsWith("*")){
         termVal = termVal.substring(0, termVal.length()-1);
      }
      else{
        isExact = true; 
      }
    }
        
    IndexReader reader = ctx.getIndexReader();
    List<AtomicReaderContext> leaves = reader.leaves();
    TreeMap<BytesRef,TermsEnum> termMap = null;
    HashMap<BytesRef,AtomicInteger> termCountMap = new HashMap<BytesRef,AtomicInteger>();

    int numCount = 0;
    int numPerPage = 20;
    
    for (AtomicReaderContext leaf : leaves){
      AtomicReader atomicReader = leaf.reader();
      
      Terms terms = atomicReader.fields().terms(field);
      
      if (terms == null) {
        continue;
      }
      
      if (termMap == null){
        termMap = new TreeMap<BytesRef,TermsEnum>(terms.getComparator());
      }
      
      
      TermsEnum te = terms.iterator(null);
      BytesRef termBytes;
      if (termVal != null){
        if (isExact){
          if (!te.seekExact(new BytesRef(termVal), false)){
            continue;
          }
        }
        else{
          te.seekCeil(new BytesRef(termVal));
        }
        termBytes = te.term();
      }
      else{
        termBytes = te.next();
      }
      
      while(true){
        if (termBytes == null) break;        
        AtomicInteger count = termCountMap.get(termBytes);
        if (count == null){
          termCountMap.put(termBytes, new AtomicInteger(te.docFreq()));
          termMap.put(termBytes, te);
          break;
        }
        count.getAndAdd(te.docFreq());
        if (isExact){
          termBytes = null; 
        }
        else{
          termBytes = te.next();
        }
      }
    }
    
    while(termMap != null && !termMap.isEmpty()){
      numCount++;
      Entry<BytesRef,TermsEnum> entry = termMap.pollFirstEntry();
      if (entry == null) break;
      BytesRef key = entry.getKey();
      AtomicInteger count = termCountMap.remove(key);
      out.println(key.utf8ToString()+" ("+count+") ");
      if (ctx.isInteractiveMode()){
        if (numCount % numPerPage == 0){
          out.println("Ctrl-D to break");
          int ch = System.in.read();
          if (ch == -1) {
            out.flush();
            return;
          }
        }
      }
      TermsEnum te = entry.getValue();
      BytesRef nextKey = null;
      if (!isExact){
        nextKey = te.next();
      }
      while(true){
        if (nextKey == null) break;
        count = termCountMap.get(nextKey);
        if (count == null){
          termCountMap.put(nextKey, new AtomicInteger(te.docFreq()));
          termMap.put(nextKey, te);
          break;
        }
        count.getAndAdd(te.docFreq());
        nextKey = te.next();
      }
    }
    out.flush();
  }

}
{'fraction_non_alphanumeric': 0.12874134100117632, 'fraction_numerical': 0.002875441118807999, 'mean_word_length': 2.150267599835323, 'pattern_counts': {'":': 2, '<': 17, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '12422', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2380, 'n_words': 500}