<commit_before>package org.apache.lucene.search.spell;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.Comparator;
import java.util.PriorityQueue;
import java.util.Queue;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spell.SuggestMode;
import org.apache.lucene.util.BytesRef;

/**
 * <p>
 * A spell checker whose sole function is to offer suggestions by combining
 * multiple terms into one word and/or breaking terms into multiple words.
 * </p>
 */
public class WordBreakSpellChecker {
  private int minSuggestionFrequency = 1;
  private int minBreakWordLength = 1;
  private int maxCombineWordLength = 20;
  private int maxChanges = 1;
  private int maxEvaluations = 1000;
  
  public static final Term SEPARATOR_TERM = new Term("", "");
  
  public enum BreakSuggestionSortMethod {
    /**
     * <p>
     * Sort by Number of word breaks, then by the Sum of all the component
     * term's frequencies
     * </p>
     */
    NUM_CHANGES_THEN_SUMMED_FREQUENCY,
    /**
     * <p>
     * Sort by Number of word breaks, then by the Maximum of all the component
     * term's frequencies
     * </p>
     */
    NUM_CHANGES_THEN_MAX_FREQUENCY
  }
  
  /**
   * <p>
   * Generate suggestions by breaking the passed-in term into multiple words.
   * The scores returned are equal to the number of word breaks needed so a
   * lower score is generally preferred over a higher score.
   * </p>
   * 
   * @param term
   * @param maxSuggestions
   * @param ir
   * @param suggestMode
   *          - default = {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}
   * @param sortMethod
   *          - default =
   *          {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY}
   * @return one or more arrays of words formed by breaking up the original term
   * @throws IOException
   */
  public SuggestWord[][] suggestWordBreaks(Term term, int maxSuggestions,
      IndexReader ir, SuggestMode suggestMode,
      BreakSuggestionSortMethod sortMethod) throws IOException {
    if (maxSuggestions < 1) {
      return new SuggestWord[0][0];
    }
    if (suggestMode == null) {
      suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
    }
    if (sortMethod == null) {
      sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY;
    }
    
    int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions;
    Comparator<SuggestWordArrayWrapper> queueComparator = sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY ? new LengthThenMaxFreqComparator()
        : new LengthThenSumFreqComparator();
    Queue<SuggestWordArrayWrapper> suggestions = new PriorityQueue<SuggestWordArrayWrapper>(
        queueInitialCapacity, queueComparator);
    
    int origFreq = ir.docFreq(term);
    if (origFreq > 0 && suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX) {
      return new SuggestWord[0][];
    }
    
    int useMinSuggestionFrequency = minSuggestionFrequency;
    if (suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) {
      useMinSuggestionFrequency = (origFreq == 0 ? 1 : origFreq);
    }
    
    generateBreakUpSuggestions(term, ir, 1, maxSuggestions,
        useMinSuggestionFrequency, new SuggestWord[0], suggestions, 0,
        sortMethod);
    
    SuggestWord[][] suggestionArray = new SuggestWord[suggestions.size()][];
    for (int i = suggestions.size() - 1; i >= 0; i--) {
      suggestionArray[i] = suggestions.remove().suggestWords;
    }
    
    return suggestionArray;
  }
  
  /**
   * <p>
   * Generate suggestions by combining one or more of the passed-in terms into
   * single words. The returned {@link CombineSuggestion} contains both a
   * {@link SuggestWord} and also an array detailing which passed-in terms were
   * involved in creating this combination. The scores returned are equal to the
   * number of word combinations needed, also one less than the length of the
   * array {@link CombineSuggestion#originalTermIndexes}. Generally, a
   * suggestion with a lower score is preferred over a higher score.
   * </p>
   * <p>
   * To prevent two adjacent terms from being combined (for instance, if one is
   * mandatory and the other is prohibited), separate the two terms with
   * {@link WordBreakSpellChecker#SEPARATOR_TERM}
   * </p>
   * <p>
   * When suggestMode equals {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}, each
   * suggestion will include at least one term not in the index.
   * </p>
   * <p>
   * When suggestMode equals {@link SuggestMode#SUGGEST_MORE_POPULAR}, each
   * suggestion will have the same, or better frequency than the most-popular
   * included term.
   * </p>
   * 
   * @param terms
   * @param maxSuggestions
   * @param ir
   * @param suggestMode
   * @return an array of words generated by combining original terms
   * @throws IOException
   */
  public CombineSuggestion[] suggestWordCombinations(Term[] terms,
      int maxSuggestions, IndexReader ir, SuggestMode suggestMode)
      throws IOException {
    if (maxSuggestions < 1) {
      return new CombineSuggestion[0];
    }
    
    int[] origFreqs = null;
    if (suggestMode != SuggestMode.SUGGEST_ALWAYS) {
      origFreqs = new int[terms.length];
      for (int i = 0; i < terms.length; i++) {
        origFreqs[i] = ir.docFreq(terms[i]);
      }
    }
    
    int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions;
    Comparator<CombineSuggestionWrapper> queueComparator = new CombinationsThenFreqComparator();
    Queue<CombineSuggestionWrapper> suggestions = new PriorityQueue<CombineSuggestionWrapper>(
        queueInitialCapacity, queueComparator);
    
    int thisTimeEvaluations = 0;
    BytesRef reuse = new BytesRef();
    for (int i = 0; i < terms.length - 1; i++) {
      if (terms[i].equals(SEPARATOR_TERM)) {
        continue;
      }
      
      int byteLength = terms[i].bytes().length;
      if (byteLength > maxCombineWordLength) {
        continue;
      }
      
      reuse.grow(byteLength);
      reuse.length = byteLength;
      System.arraycopy(terms[i].bytes().bytes, terms[i].bytes().offset,
          reuse.bytes, 0, byteLength);
      
      int maxFreq = 0;
      int minFreq = Integer.MAX_VALUE;
      if (origFreqs != null) {
        maxFreq = origFreqs[i];
        minFreq = origFreqs[i];
      }
      
      for (int j = i + 1; j < terms.length && j - i <= maxChanges; j++) {
        if (terms[j].equals(SEPARATOR_TERM)) {
          break;
        }
        byteLength += terms[j].bytes().length;
        if (byteLength > maxCombineWordLength) {
          break;
        }
        
        if (origFreqs != null) {
          maxFreq = Math.max(maxFreq, origFreqs[j]);
          minFreq = Math.min(minFreq, origFreqs[j]);
        }
        
        reuse.grow(byteLength);
        System.arraycopy(terms[j].bytes().bytes, terms[j].bytes().offset,
            reuse.bytes, reuse.length, terms[j].bytes().length);
        reuse.length = byteLength;
        
        Term combinedTerm = new Term(terms[0].field(), reuse);
        int combinedTermFreq = ir.docFreq(combinedTerm);
        
        if (suggestMode != SuggestMode.SUGGEST_MORE_POPULAR
            || combinedTermFreq >= maxFreq) {
          if (suggestMode != SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX
              || minFreq == 0) {
            if (combinedTermFreq >= minSuggestionFrequency) {
              int[] origIndexes = new int[j - i + 1];
              origIndexes[0] = i;
              for (int k = 1; k < origIndexes.length; k++) {
                origIndexes[k] = i + k;
              }
              SuggestWord word = new SuggestWord();
              word.freq = combinedTermFreq;
              word.score = origIndexes.length - 1;
              word.string = combinedTerm.text();
              CombineSuggestionWrapper suggestion = new CombineSuggestionWrapper(
                  new CombineSuggestion(word, origIndexes),
                  (origIndexes.length - 1));
              suggestions.offer(suggestion);
              if (suggestions.size() > maxSuggestions) {
                suggestions.poll();
              }
            }
          }
        }
        thisTimeEvaluations++;
        if (thisTimeEvaluations == maxEvaluations) {
          break;
        }
      }
    }
    CombineSuggestion[] combineSuggestions = new CombineSuggestion[suggestions
        .size()];
    for (int i = suggestions.size() - 1; i >= 0; i--) {
      combineSuggestions[i] = suggestions.remove().combineSuggestion;
    }
    return combineSuggestions;
  }
  
  private int generateBreakUpSuggestions(Term term, IndexReader ir,
      int numberBreaks, int maxSuggestions, int useMinSuggestionFrequency,
      SuggestWord[] prefix, Queue<SuggestWordArrayWrapper> suggestions,
      int totalEvaluations, BreakSuggestionSortMethod sortMethod)
      throws IOException {
    int termLength = term.bytes().length;
    int useMinBreakWordLength = minBreakWordLength;
    if (useMinBreakWordLength < 1) {
      useMinBreakWordLength = 1;
    }
    if (termLength <= (useMinBreakWordLength * 2)) {
      return 0;
    }
    
    int thisTimeEvaluations = 0;
    BytesRef termBytes = term.bytes().clone();
    for (int i = useMinBreakWordLength; i < (termLength - useMinBreakWordLength); i++) {
      SuggestWord leftWord = generateSuggestWord(ir, termBytes, 0, i, term
          .field());
      
      if (leftWord.freq >= useMinSuggestionFrequency) {
        SuggestWord rightWord = generateSuggestWord(ir, termBytes, i,
            termLength - i, term.field());
        if (rightWord.freq >= useMinSuggestionFrequency) {
          SuggestWordArrayWrapper suggestion = new SuggestWordArrayWrapper(
              newSuggestion(prefix, leftWord, rightWord));
          suggestions.offer(suggestion);
          if (suggestions.size() > maxSuggestions) {
            suggestions.poll();
          }
        }
        
        int newNumberBreaks = numberBreaks + 1;
        if (newNumberBreaks <= maxChanges) {
          int evaluations = generateBreakUpSuggestions(new Term(term.field(),
              rightWord.string), ir, newNumberBreaks, maxSuggestions,
              useMinSuggestionFrequency, newPrefix(prefix, leftWord),
              suggestions, totalEvaluations, sortMethod);
          totalEvaluations += evaluations;
        }
      }
      thisTimeEvaluations++;
      totalEvaluations++;
      if (totalEvaluations >= maxEvaluations) {
        break;
      }
    }
    return thisTimeEvaluations;
  }
  
  private SuggestWord[] newPrefix(SuggestWord[] oldPrefix, SuggestWord append) {
    SuggestWord[] newPrefix = new SuggestWord[oldPrefix.length + 1];
    System.arraycopy(oldPrefix, 0, newPrefix, 0, oldPrefix.length);
    newPrefix[newPrefix.length - 1] = append;
    return newPrefix;
  }
  
  private SuggestWord[] newSuggestion(SuggestWord[] prefix,
      SuggestWord append1, SuggestWord append2) {
    SuggestWord[] newSuggestion = new SuggestWord[prefix.length + 2];
    int score = prefix.length + 1;
    for (int i = 0; i < prefix.length; i++) {
      SuggestWord word = new SuggestWord();
      word.string = prefix[i].string;
      word.freq = prefix[i].freq;
      word.score = score;
      newSuggestion[i] = word;
    }
    append1.score = score;
    append2.score = score;
    newSuggestion[newSuggestion.length - 2] = append1;
    newSuggestion[newSuggestion.length - 1] = append2;
    return newSuggestion;
  }
  
  private SuggestWord generateSuggestWord(IndexReader ir, BytesRef bytes,
      int offset, int length, String fieldname) throws IOException {
    bytes.offset = offset;
    bytes.length = length;
    Term term = new Term(fieldname, bytes);
    int freq = ir.docFreq(term);
    SuggestWord word = new SuggestWord();
    word.freq = freq;
    word.score = 1;
    word.string = term.text();
    return word;
  }
  
  public int getMinSuggestionFrequency() {
    return minSuggestionFrequency;
  }
  
  public int getMaxCombineWordLength() {
    return maxCombineWordLength;
  }
  
  public int getMinBreakWordLength() {
    return minBreakWordLength;
  }
  
  public int getMaxChanges() {
    return maxChanges;
  }
  
  public int getMaxEvaluations() {
    return maxEvaluations;
  }
  
  /**
   * <p>
   * The minimum frequency a term must have to be included as part of a
   * suggestion. Default=1 Not applicable when used with
   * {@link SuggestMode#SUGGEST_MORE_POPULAR}
   * </p>
   * 
   * @param minSuggestionFrequency
   */
  public void setMinSuggestionFrequency(int minSuggestionFrequency) {
    this.minSuggestionFrequency = minSuggestionFrequency;
  }
  
  /**
   * <p>
   * The maximum length of a suggestion made by combining 1 or more original
   * terms. Default=20
   * </p>
   * 
   * @param maxCombineWordLength
   */
  public void setMaxCombineWordLength(int maxCombineWordLength) {
    this.maxCombineWordLength = maxCombineWordLength;
  }
  
  /**
   * <p>
   * The minimum length to break words down to. Default=1
   * </p>
   * 
   * @param minBreakWordLength
   */
  public void setMinBreakWordLength(int minBreakWordLength) {
    this.minBreakWordLength = minBreakWordLength;
  }
  
  /**
   * <p>
   * The maximum numbers of changes (word breaks or combinations) to make on the
   * original term(s). Default=1
   * </p>
   * 
   * @param maxChanges
   */
  public void setMaxChanges(int maxChanges) {
    this.maxChanges = maxChanges;
  }
  
  /**
   * <p>
   * The maximum number of word combinations to evaluate. Default=1000. A higher
   * value might improve result quality. A lower value might improve
   * performance.
   * </p>
   * 
   * @param maxEvaluations
   */
  public void setMaxEvaluations(int maxEvaluations) {
    this.maxEvaluations = maxEvaluations;
  }
  
  private class LengthThenMaxFreqComparator implements
      Comparator<SuggestWordArrayWrapper> {
    @Override
    public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) {
      if (o1.suggestWords.length != o2.suggestWords.length) {
        return o2.suggestWords.length - o1.suggestWords.length;
      }
      if (o1.freqMax != o2.freqMax) {
        return o1.freqMax - o2.freqMax;
      }
      return 0;
    }
  }
  
  private class LengthThenSumFreqComparator implements
      Comparator<SuggestWordArrayWrapper> {
    @Override
    public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) {
      if (o1.suggestWords.length != o2.suggestWords.length) {
        return o2.suggestWords.length - o1.suggestWords.length;
      }
      if (o1.freqSum != o2.freqSum) {
        return o1.freqSum - o2.freqSum;
      }
      return 0;
    }
  }
  
  private class CombinationsThenFreqComparator implements
      Comparator<CombineSuggestionWrapper> {
    @Override
    public int compare(CombineSuggestionWrapper o1, CombineSuggestionWrapper o2) {
      if (o1.numCombinations != o2.numCombinations) {
        return o2.numCombinations - o1.numCombinations;
      }
      if (o1.combineSuggestion.suggestion.freq != o2.combineSuggestion.suggestion.freq) {
        return o1.combineSuggestion.suggestion.freq
            - o2.combineSuggestion.suggestion.freq;
      }
      return 0;
    }
  }
  
  private class SuggestWordArrayWrapper {
    final SuggestWord[] suggestWords;
    final int freqMax;
    final int freqSum;
    
    SuggestWordArrayWrapper(SuggestWord[] suggestWords) {
      this.suggestWords = suggestWords;
      int aFreqSum = 0;
      int aFreqMax = 0;
      for (SuggestWord sw : suggestWords) {
        aFreqSum += sw.freq;
        aFreqMax = Math.max(aFreqMax, sw.freq);
      }
      this.freqSum = aFreqSum;
      this.freqMax = aFreqMax;
    }
  }
  
  private class CombineSuggestionWrapper {
    final CombineSuggestion combineSuggestion;
    final int numCombinations;
    
    CombineSuggestionWrapper(CombineSuggestion combineSuggestion,
        int numCombinations) {
      this.combineSuggestion = combineSuggestion;
      this.numCombinations = numCombinations;
    }
  }
}
<commit_msg>LUCENE-3523:  Javadoc fix<commit_after>package org.apache.lucene.search.spell;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.Comparator;
import java.util.PriorityQueue;
import java.util.Queue;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.spell.SuggestMode;
import org.apache.lucene.util.BytesRef;

/**
 * <p>
 * A spell checker whose sole function is to offer suggestions by combining
 * multiple terms into one word and/or breaking terms into multiple words.
 * </p>
 */
public class WordBreakSpellChecker {
  private int minSuggestionFrequency = 1;
  private int minBreakWordLength = 1;
  private int maxCombineWordLength = 20;
  private int maxChanges = 1;
  private int maxEvaluations = 1000;
  
  public static final Term SEPARATOR_TERM = new Term("", "");
  
  /**
   * <p>
   * Determines the order to list word break suggestions
   * </p>
   */
  public enum BreakSuggestionSortMethod {
    /**
     * <p>
     * Sort by Number of word breaks, then by the Sum of all the component
     * term's frequencies
     * </p>
     */
    NUM_CHANGES_THEN_SUMMED_FREQUENCY,
    /**
     * <p>
     * Sort by Number of word breaks, then by the Maximum of all the component
     * term's frequencies
     * </p>
     */
    NUM_CHANGES_THEN_MAX_FREQUENCY
  }
  
  /**
   * <p>
   * Generate suggestions by breaking the passed-in term into multiple words.
   * The scores returned are equal to the number of word breaks needed so a
   * lower score is generally preferred over a higher score.
   * </p>
   * 
   * @param term
   * @param maxSuggestions
   * @param ir
   * @param suggestMode
   *          - default = {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}
   * @param sortMethod
   *          - default =
   *          {@link BreakSuggestionSortMethod#NUM_CHANGES_THEN_MAX_FREQUENCY}
   * @return one or more arrays of words formed by breaking up the original term
   * @throws IOException
   */
  public SuggestWord[][] suggestWordBreaks(Term term, int maxSuggestions,
      IndexReader ir, SuggestMode suggestMode,
      BreakSuggestionSortMethod sortMethod) throws IOException {
    if (maxSuggestions < 1) {
      return new SuggestWord[0][0];
    }
    if (suggestMode == null) {
      suggestMode = SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX;
    }
    if (sortMethod == null) {
      sortMethod = BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY;
    }
    
    int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions;
    Comparator<SuggestWordArrayWrapper> queueComparator = sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY ? new LengthThenMaxFreqComparator()
        : new LengthThenSumFreqComparator();
    Queue<SuggestWordArrayWrapper> suggestions = new PriorityQueue<SuggestWordArrayWrapper>(
        queueInitialCapacity, queueComparator);
    
    int origFreq = ir.docFreq(term);
    if (origFreq > 0 && suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX) {
      return new SuggestWord[0][];
    }
    
    int useMinSuggestionFrequency = minSuggestionFrequency;
    if (suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) {
      useMinSuggestionFrequency = (origFreq == 0 ? 1 : origFreq);
    }
    
    generateBreakUpSuggestions(term, ir, 1, maxSuggestions,
        useMinSuggestionFrequency, new SuggestWord[0], suggestions, 0,
        sortMethod);
    
    SuggestWord[][] suggestionArray = new SuggestWord[suggestions.size()][];
    for (int i = suggestions.size() - 1; i >= 0; i--) {
      suggestionArray[i] = suggestions.remove().suggestWords;
    }
    
    return suggestionArray;
  }
  
  /**
   * <p>
   * Generate suggestions by combining one or more of the passed-in terms into
   * single words. The returned {@link CombineSuggestion} contains both a
   * {@link SuggestWord} and also an array detailing which passed-in terms were
   * involved in creating this combination. The scores returned are equal to the
   * number of word combinations needed, also one less than the length of the
   * array {@link CombineSuggestion#originalTermIndexes}. Generally, a
   * suggestion with a lower score is preferred over a higher score.
   * </p>
   * <p>
   * To prevent two adjacent terms from being combined (for instance, if one is
   * mandatory and the other is prohibited), separate the two terms with
   * {@link WordBreakSpellChecker#SEPARATOR_TERM}
   * </p>
   * <p>
   * When suggestMode equals {@link SuggestMode#SUGGEST_WHEN_NOT_IN_INDEX}, each
   * suggestion will include at least one term not in the index.
   * </p>
   * <p>
   * When suggestMode equals {@link SuggestMode#SUGGEST_MORE_POPULAR}, each
   * suggestion will have the same, or better frequency than the most-popular
   * included term.
   * </p>
   * 
   * @param terms
   * @param maxSuggestions
   * @param ir
   * @param suggestMode
   * @return an array of words generated by combining original terms
   * @throws IOException
   */
  public CombineSuggestion[] suggestWordCombinations(Term[] terms,
      int maxSuggestions, IndexReader ir, SuggestMode suggestMode)
      throws IOException {
    if (maxSuggestions < 1) {
      return new CombineSuggestion[0];
    }
    
    int[] origFreqs = null;
    if (suggestMode != SuggestMode.SUGGEST_ALWAYS) {
      origFreqs = new int[terms.length];
      for (int i = 0; i < terms.length; i++) {
        origFreqs[i] = ir.docFreq(terms[i]);
      }
    }
    
    int queueInitialCapacity = maxSuggestions > 10 ? 10 : maxSuggestions;
    Comparator<CombineSuggestionWrapper> queueComparator = new CombinationsThenFreqComparator();
    Queue<CombineSuggestionWrapper> suggestions = new PriorityQueue<CombineSuggestionWrapper>(
        queueInitialCapacity, queueComparator);
    
    int thisTimeEvaluations = 0;
    BytesRef reuse = new BytesRef();
    for (int i = 0; i < terms.length - 1; i++) {
      if (terms[i].equals(SEPARATOR_TERM)) {
        continue;
      }
      
      int byteLength = terms[i].bytes().length;
      if (byteLength > maxCombineWordLength) {
        continue;
      }
      
      reuse.grow(byteLength);
      reuse.length = byteLength;
      System.arraycopy(terms[i].bytes().bytes, terms[i].bytes().offset,
          reuse.bytes, 0, byteLength);
      
      int maxFreq = 0;
      int minFreq = Integer.MAX_VALUE;
      if (origFreqs != null) {
        maxFreq = origFreqs[i];
        minFreq = origFreqs[i];
      }
      
      for (int j = i + 1; j < terms.length && j - i <= maxChanges; j++) {
        if (terms[j].equals(SEPARATOR_TERM)) {
          break;
        }
        byteLength += terms[j].bytes().length;
        if (byteLength > maxCombineWordLength) {
          break;
        }
        
        if (origFreqs != null) {
          maxFreq = Math.max(maxFreq, origFreqs[j]);
          minFreq = Math.min(minFreq, origFreqs[j]);
        }
        
        reuse.grow(byteLength);
        System.arraycopy(terms[j].bytes().bytes, terms[j].bytes().offset,
            reuse.bytes, reuse.length, terms[j].bytes().length);
        reuse.length = byteLength;
        
        Term combinedTerm = new Term(terms[0].field(), reuse);
        int combinedTermFreq = ir.docFreq(combinedTerm);
        
        if (suggestMode != SuggestMode.SUGGEST_MORE_POPULAR
            || combinedTermFreq >= maxFreq) {
          if (suggestMode != SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX
              || minFreq == 0) {
            if (combinedTermFreq >= minSuggestionFrequency) {
              int[] origIndexes = new int[j - i + 1];
              origIndexes[0] = i;
              for (int k = 1; k < origIndexes.length; k++) {
                origIndexes[k] = i + k;
              }
              SuggestWord word = new SuggestWord();
              word.freq = combinedTermFreq;
              word.score = origIndexes.length - 1;
              word.string = combinedTerm.text();
              CombineSuggestionWrapper suggestion = new CombineSuggestionWrapper(
                  new CombineSuggestion(word, origIndexes),
                  (origIndexes.length - 1));
              suggestions.offer(suggestion);
              if (suggestions.size() > maxSuggestions) {
                suggestions.poll();
              }
            }
          }
        }
        thisTimeEvaluations++;
        if (thisTimeEvaluations == maxEvaluations) {
          break;
        }
      }
    }
    CombineSuggestion[] combineSuggestions = new CombineSuggestion[suggestions
        .size()];
    for (int i = suggestions.size() - 1; i >= 0; i--) {
      combineSuggestions[i] = suggestions.remove().combineSuggestion;
    }
    return combineSuggestions;
  }
  
  private int generateBreakUpSuggestions(Term term, IndexReader ir,
      int numberBreaks, int maxSuggestions, int useMinSuggestionFrequency,
      SuggestWord[] prefix, Queue<SuggestWordArrayWrapper> suggestions,
      int totalEvaluations, BreakSuggestionSortMethod sortMethod)
      throws IOException {
    int termLength = term.bytes().length;
    int useMinBreakWordLength = minBreakWordLength;
    if (useMinBreakWordLength < 1) {
      useMinBreakWordLength = 1;
    }
    if (termLength <= (useMinBreakWordLength * 2)) {
      return 0;
    }
    
    int thisTimeEvaluations = 0;
    BytesRef termBytes = term.bytes().clone();
    for (int i = useMinBreakWordLength; i < (termLength - useMinBreakWordLength); i++) {
      SuggestWord leftWord = generateSuggestWord(ir, termBytes, 0, i, term
          .field());
      
      if (leftWord.freq >= useMinSuggestionFrequency) {
        SuggestWord rightWord = generateSuggestWord(ir, termBytes, i,
            termLength - i, term.field());
        if (rightWord.freq >= useMinSuggestionFrequency) {
          SuggestWordArrayWrapper suggestion = new SuggestWordArrayWrapper(
              newSuggestion(prefix, leftWord, rightWord));
          suggestions.offer(suggestion);
          if (suggestions.size() > maxSuggestions) {
            suggestions.poll();
          }
        }
        
        int newNumberBreaks = numberBreaks + 1;
        if (newNumberBreaks <= maxChanges) {
          int evaluations = generateBreakUpSuggestions(new Term(term.field(),
              rightWord.string), ir, newNumberBreaks, maxSuggestions,
              useMinSuggestionFrequency, newPrefix(prefix, leftWord),
              suggestions, totalEvaluations, sortMethod);
          totalEvaluations += evaluations;
        }
      }
      thisTimeEvaluations++;
      totalEvaluations++;
      if (totalEvaluations >= maxEvaluations) {
        break;
      }
    }
    return thisTimeEvaluations;
  }
  
  private SuggestWord[] newPrefix(SuggestWord[] oldPrefix, SuggestWord append) {
    SuggestWord[] newPrefix = new SuggestWord[oldPrefix.length + 1];
    System.arraycopy(oldPrefix, 0, newPrefix, 0, oldPrefix.length);
    newPrefix[newPrefix.length - 1] = append;
    return newPrefix;
  }
  
  private SuggestWord[] newSuggestion(SuggestWord[] prefix,
      SuggestWord append1, SuggestWord append2) {
    SuggestWord[] newSuggestion = new SuggestWord[prefix.length + 2];
    int score = prefix.length + 1;
    for (int i = 0; i < prefix.length; i++) {
      SuggestWord word = new SuggestWord();
      word.string = prefix[i].string;
      word.freq = prefix[i].freq;
      word.score = score;
      newSuggestion[i] = word;
    }
    append1.score = score;
    append2.score = score;
    newSuggestion[newSuggestion.length - 2] = append1;
    newSuggestion[newSuggestion.length - 1] = append2;
    return newSuggestion;
  }
  
  private SuggestWord generateSuggestWord(IndexReader ir, BytesRef bytes,
      int offset, int length, String fieldname) throws IOException {
    bytes.offset = offset;
    bytes.length = length;
    Term term = new Term(fieldname, bytes);
    int freq = ir.docFreq(term);
    SuggestWord word = new SuggestWord();
    word.freq = freq;
    word.score = 1;
    word.string = term.text();
    return word;
  }
  
  public int getMinSuggestionFrequency() {
    return minSuggestionFrequency;
  }
  
  public int getMaxCombineWordLength() {
    return maxCombineWordLength;
  }
  
  public int getMinBreakWordLength() {
    return minBreakWordLength;
  }
  
  public int getMaxChanges() {
    return maxChanges;
  }
  
  public int getMaxEvaluations() {
    return maxEvaluations;
  }
  
  /**
   * <p>
   * The minimum frequency a term must have to be included as part of a
   * suggestion. Default=1 Not applicable when used with
   * {@link SuggestMode#SUGGEST_MORE_POPULAR}
   * </p>
   * 
   * @param minSuggestionFrequency
   */
  public void setMinSuggestionFrequency(int minSuggestionFrequency) {
    this.minSuggestionFrequency = minSuggestionFrequency;
  }
  
  /**
   * <p>
   * The maximum length of a suggestion made by combining 1 or more original
   * terms. Default=20
   * </p>
   * 
   * @param maxCombineWordLength
   */
  public void setMaxCombineWordLength(int maxCombineWordLength) {
    this.maxCombineWordLength = maxCombineWordLength;
  }
  
  /**
   * <p>
   * The minimum length to break words down to. Default=1
   * </p>
   * 
   * @param minBreakWordLength
   */
  public void setMinBreakWordLength(int minBreakWordLength) {
    this.minBreakWordLength = minBreakWordLength;
  }
  
  /**
   * <p>
   * The maximum numbers of changes (word breaks or combinations) to make on the
   * original term(s). Default=1
   * </p>
   * 
   * @param maxChanges
   */
  public void setMaxChanges(int maxChanges) {
    this.maxChanges = maxChanges;
  }
  
  /**
   * <p>
   * The maximum number of word combinations to evaluate. Default=1000. A higher
   * value might improve result quality. A lower value might improve
   * performance.
   * </p>
   * 
   * @param maxEvaluations
   */
  public void setMaxEvaluations(int maxEvaluations) {
    this.maxEvaluations = maxEvaluations;
  }
  
  private class LengthThenMaxFreqComparator implements
      Comparator<SuggestWordArrayWrapper> {
    @Override
    public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) {
      if (o1.suggestWords.length != o2.suggestWords.length) {
        return o2.suggestWords.length - o1.suggestWords.length;
      }
      if (o1.freqMax != o2.freqMax) {
        return o1.freqMax - o2.freqMax;
      }
      return 0;
    }
  }
  
  private class LengthThenSumFreqComparator implements
      Comparator<SuggestWordArrayWrapper> {
    @Override
    public int compare(SuggestWordArrayWrapper o1, SuggestWordArrayWrapper o2) {
      if (o1.suggestWords.length != o2.suggestWords.length) {
        return o2.suggestWords.length - o1.suggestWords.length;
      }
      if (o1.freqSum != o2.freqSum) {
        return o1.freqSum - o2.freqSum;
      }
      return 0;
    }
  }
  
  private class CombinationsThenFreqComparator implements
      Comparator<CombineSuggestionWrapper> {
    @Override
    public int compare(CombineSuggestionWrapper o1, CombineSuggestionWrapper o2) {
      if (o1.numCombinations != o2.numCombinations) {
        return o2.numCombinations - o1.numCombinations;
      }
      if (o1.combineSuggestion.suggestion.freq != o2.combineSuggestion.suggestion.freq) {
        return o1.combineSuggestion.suggestion.freq
            - o2.combineSuggestion.suggestion.freq;
      }
      return 0;
    }
  }
  
  private class SuggestWordArrayWrapper {
    final SuggestWord[] suggestWords;
    final int freqMax;
    final int freqSum;
    
    SuggestWordArrayWrapper(SuggestWord[] suggestWords) {
      this.suggestWords = suggestWords;
      int aFreqSum = 0;
      int aFreqMax = 0;
      for (SuggestWord sw : suggestWords) {
        aFreqSum += sw.freq;
        aFreqMax = Math.max(aFreqMax, sw.freq);
      }
      this.freqSum = aFreqSum;
      this.freqMax = aFreqMax;
    }
  }
  
  private class CombineSuggestionWrapper {
    final CombineSuggestion combineSuggestion;
    final int numCombinations;
    
    CombineSuggestionWrapper(CombineSuggestion combineSuggestion,
        int numCombinations) {
      this.combineSuggestion = combineSuggestion;
      this.numCombinations = numCombinations;
    }
  }
}

{'fraction_non_alphanumeric': 0.09269379269379269, 'fraction_numerical': 0.007187407187407188, 'mean_word_length': 2.762543300927478, 'pattern_counts': {'":': 0, '<': 101, '<?xml version=': 0, '>': 105, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7663675', 'n_tokens_mistral': 10493, 'n_tokens_neox': 9935, 'n_words': 2592}