<commit_before>package com.senseidb.clue.commands;

import java.io.PrintStream;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;

import com.senseidb.clue.ClueContext;

public class StoredFieldCommand extends ClueCommand {

  public StoredFieldCommand(ClueContext ctx) {
    super(ctx);
  }

  @Override
  public String getName() {
    return "stored";
  }

  @Override
  public String help() {
    return "displays stored data for a given field";
  }

  @Override
  public void execute(String[] args, PrintStream out) throws Exception {
    if (args.length != 2) {
      out.println("usage: field doc");
      return;
    }
    
    String field = args[0];
    
    int doc = Integer.parseInt(args[1]);
    
    IndexReader reader = ctx.getIndexReader();
    List<AtomicReaderContext> leaves = reader.leaves();
    
    boolean found = false;
    boolean stored = false;
    
    int docID = doc;
    for (AtomicReaderContext ctx : leaves) {
      AtomicReader atomicReader = ctx.reader();
      FieldInfo finfo = atomicReader.getFieldInfos().fieldInfo(field);
      if (finfo == null) continue;
      
      stored = true;
      
      if (docID > ctx.docBase) {
        docID -= ctx.docBase;
        continue;
      }
      
      if (docID >= 0) {
      
        Document storedData = atomicReader.document(docID, new HashSet<String>(Arrays.asList(field)));
        
        if (storedData == null) continue;
        
        String strData = storedData.get(field);
        
        if (strData != null) {
          out.println(strData);
          found = true;
          break;<commit_msg>store docid normalization fix<commit_after>package com.senseidb.clue.commands;

import java.io.PrintStream;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;

import com.senseidb.clue.ClueContext;

public class StoredFieldCommand extends ClueCommand {

  public StoredFieldCommand(ClueContext ctx) {
    super(ctx);
  }

  @Override
  public String getName() {
    return "stored";
  }

  @Override
  public String help() {
    return "displays stored data for a given field";
  }

  @Override
  public void execute(String[] args, PrintStream out) throws Exception {
    if (args.length != 2) {
      out.println("usage: field doc");
      return;
    }
    
    String field = args[0];
    
    int doc = Integer.parseInt(args[1]);
    
    IndexReader reader = ctx.getIndexReader();
    List<AtomicReaderContext> leaves = reader.leaves();
    
    boolean found = false;
    boolean stored = false;
    
    
    for (AtomicReaderContext ctx : leaves) {
      AtomicReader atomicReader = ctx.reader();
      FieldInfo finfo = atomicReader.getFieldInfos().fieldInfo(field);
      if (finfo == null) continue;
      
      stored = true;
      
      int docID = doc - ctx.docBase;
      
      if (docID >= atomicReader.maxDoc()) {
          continue;
      }
      
      if (docID >= 0) {
      
        Document storedData = atomicReader.document(docID, new HashSet<String>(Arrays.asList(field)));
        
        if (storedData == null) continue;
        
        String strData = storedData.get(field);
        
        if (strData != null) {
          out.println(strData);
          found = true;
          break;
{'fraction_non_alphanumeric': 0.10707708497763746, 'fraction_numerical': 0.002104709287029729, 'mean_word_length': 2.6734299516908213, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5106880', 'n_tokens_mistral': 1180, 'n_tokens_neox': 1155, 'n_words': 265}