<commit_before>package com.swabunga.spell.engine;

import java.io.*;
import java.util.*;

/**
 * Yet another <code>SpellDictionary</code> this one is based on Damien Guillaume's
 * Diskbased dictionary but adds a cache to try to improve abit on performance.
 *
 * @author <NAME>
 * @version 0.01
 */

public class SpellDictionaryCachedDichoDisk extends SpellDictionaryDichoDisk {
    
    // Only used for testing to measure the effectiveness of the cache.
    static public int hits=0;
    static public int codes=0;

    public static final String PRE_CACHE_FILE_EXT=".pre";

    private static int  MAX_CACHED=10000;

    private HashMap     suggestionCache=new HashMap(MAX_CACHED);
    private String      preCacheFileName;
    /**
     * Dictionary Convienence Constructor.
     */
    public SpellDictionaryCachedDichoDisk(File wordList)
                                     throws FileNotFoundException, IOException {
        super((File) wordList);
        loadPreCache(wordList);
    }
    
    /**
     * Dictionary Convienence Constructor.
     */
    public SpellDictionaryCachedDichoDisk(File wordList, String encoding)
                                     throws FileNotFoundException, IOException {
        super(wordList, encoding);
        loadPreCache(wordList);
    }

    /**
     * Dictionary constructor that uses an aspell phonetic file to
     * build the transformation table.
     */

    public SpellDictionaryCachedDichoDisk(File wordList, File phonetic)
                                     throws FileNotFoundException, IOException {
        super(wordList, phonetic);
        loadPreCache(wordList);
    }

    /**
     * Dictionary constructor that uses an aspell phonetic file to
     * build the transformation table.
     */
    public SpellDictionaryCachedDichoDisk(File wordList, File phonetic, String encoding)
                                     throws FileNotFoundException, IOException {
        super(wordList, phonetic, encoding);
        loadPreCache(wordList);
    }

    /**
     * Add a word permanantly to the dictionary (and the dictionary file).
     * <i>not implemented !</i>
     */
    public void addWord(String word) {
        System.err.println("error: addWord is not implemented for SpellDictionaryDichoDisk");
    }

    /**
     * Clears the cache.
     */
    public void clearCache(){
        suggestionCache.clear();
    }

    /**
     * Returns a list of strings (words) for the code.
     */
    public List getWords(String code) {
        List list;
        codes++;
        if(suggestionCache.containsKey(code)){
            hits++;
            list=getCachedList(code);
            return list;
        }
        list=super.getWords(code);
        addToCache(code,list);
        
        return list;
    }
    /**
     * This method returns the cached suggestionlist and also moves the code to
     * the top of the codeRefQueue to indicate this code has resentlly been
     * referenced.
     */
    private List getCachedList(String code){
        CacheObject obj=(CacheObject)suggestionCache.get(code);
        obj.setRefTime();
        return obj.getSuggestionList();
    }

    /**
     * Adds a code and it's suggestion list to the cache.
     */
    private void addToCache(String code, List l){
        String      c=null;
        String      lowestCode=null;
        long        lowestTime=Long.MAX_VALUE;
        Iterator    it;
        CacheObject obj;
        if(suggestionCache.size()>=MAX_CACHED){
            it=suggestionCache.keySet().iterator();
            while(it.hasNext()){
                c=(String)it.next();
                obj=(CacheObject)suggestionCache.get(c);
                if(obj.getRefTime()==0){
                    lowestCode=c;
                    break;
                }
                if(lowestTime>obj.getRefTime()){
                    lowestCode=c;
                    lowestTime=obj.getRefTime();
                }
            }
            suggestionCache.remove(lowestCode);
        }        
        suggestionCache.put(code,new CacheObject(l));
    }

    /**
     * Load the cache from file. The cach file has the same name as the 
     * dico file with the .pre extension added.
     */
    private void loadPreCache(File dicoFile)throws IOException{
        String  code;
        List    suggestions;
        HashMap map;
        long    size;
        preCacheFileName=dicoFile.getPath()+PRE_CACHE_FILE_EXT;
        File preFile=new File(preCacheFileName);
        if(!preFile.exists()){
            System.out.println("No precache file");
            return;
        }
        System.out.println("Precaching...");
        ObjectInputStream in=new ObjectInputStream(new FileInputStream(preFile));
        try{
            size=in.readLong();
            for(int i=0;i<size;i++){
                code=(String)in.readObject();
                suggestions=(List)in.readObject();
                suggestionCache.put(code,new CacheObject(suggestions));
            }
            in.close();
        }catch(ClassNotFoundException ex){
            System.out.println(ex.getMessage());
        }catch(IOException ex){
            System.out.println("EndOfFile");
        }
    }

    /**
     * Saves the current cache to file.
     */
    public void saveCache() throws IOException{
        String  code;
        if(preCacheFileName==null){
            System.out.println("NULL");
            return;
        }
        File preFile=new File(preCacheFileName);
        System.out.println("Saving cache...");
        ObjectOutputStream out=new ObjectOutputStream(new FileOutputStream(preFile));
        Iterator it=suggestionCache.keySet().iterator();
        out.writeLong(suggestionCache.size());
        while(it.hasNext()){
            code=(String)it.next();
            out.writeObject(code);
            out.writeObject(((CacheObject)suggestionCache.get(code)).getSuggestionList());
        }
        out.close();
    }

    // INNER CLASSES
    // ------------------------------------------------------------------------
    private class CacheObject implements Serializable{
    
        private List    suggestions=null;
        private long    refTime=0;

        public CacheObject(List list){
            this.suggestions=list;
        }
        
        public List getSuggestionList(){
            return suggestions;
        }

        public void setRefTime(){
            refTime=System.currentTimeMillis();
        }

        public long getRefTime(){
            return refTime;
        }
    }
}<commit_msg>Made the precache file store timestamps.<commit_after>package com.swabunga.spell.engine;

import java.io.*;
import java.util.*;

/**
 * Yet another <code>SpellDictionary</code> this one is based on Damien Guillaume's
 * Diskbased dictionary but adds a cache to try to improve abit on performance.
 *
 * @author <NAME>
 * @version 0.01
 */

public class SpellDictionaryCachedDichoDisk extends SpellDictionaryDichoDisk {
    
    // Only used for testing to measure the effectiveness of the cache.
    static public int hits=0;
    static public int codes=0;

    public static final String PRE_CACHE_FILE_EXT=".pre";

    private static int  MAX_CACHED=10000;

    private HashMap     suggestionCache=new HashMap(MAX_CACHED);
    private String      preCacheFileName;
    /**
     * Dictionary Convienence Constructor.
     */
    public SpellDictionaryCachedDichoDisk(File wordList)
                                     throws FileNotFoundException, IOException {
        super((File) wordList);
        loadPreCache(wordList);
    }
    
    /**
     * Dictionary Convienence Constructor.
     */
    public SpellDictionaryCachedDichoDisk(File wordList, String encoding)
                                     throws FileNotFoundException, IOException {
        super(wordList, encoding);
        loadPreCache(wordList);
    }

    /**
     * Dictionary constructor that uses an aspell phonetic file to
     * build the transformation table.
     */

    public SpellDictionaryCachedDichoDisk(File wordList, File phonetic)
                                     throws FileNotFoundException, IOException {
        super(wordList, phonetic);
        loadPreCache(wordList);
    }

    /**
     * Dictionary constructor that uses an aspell phonetic file to
     * build the transformation table.
     */
    public SpellDictionaryCachedDichoDisk(File wordList, File phonetic, String encoding)
                                     throws FileNotFoundException, IOException {
        super(wordList, phonetic, encoding);
        loadPreCache(wordList);
    }

    /**
     * Add a word permanantly to the dictionary (and the dictionary file).
     * <i>not implemented !</i>
     */
    public void addWord(String word) {
        System.err.println("error: addWord is not implemented for SpellDictionaryDichoDisk");
    }

    /**
     * Clears the cache.
     */
    public void clearCache(){
        suggestionCache.clear();
    }

    /**
     * Returns a list of strings (words) for the code.
     */
    public List getWords(String code) {
        List list;
        codes++;
        if(suggestionCache.containsKey(code)){
            hits++;
            list=getCachedList(code);
            return list;
        }
        list=super.getWords(code);
        addToCache(code,list);
        
        return list;
    }
    /**
     * This method returns the cached suggestionlist and also moves the code to
     * the top of the codeRefQueue to indicate this code has resentlly been
     * referenced.
     */
    private List getCachedList(String code){
        CacheObject obj=(CacheObject)suggestionCache.get(code);
        obj.setRefTime();
        return obj.getSuggestionList();
    }

    /**
     * Adds a code and it's suggestion list to the cache.
     */
    private void addToCache(String code, List l){
        String      c=null;
        String      lowestCode=null;
        long        lowestTime=Long.MAX_VALUE;
        Iterator    it;
        CacheObject obj;

        if(suggestionCache.size()>=MAX_CACHED){
            it=suggestionCache.keySet().iterator();
            while(it.hasNext()){
                c=(String)it.next();
                obj=(CacheObject)suggestionCache.get(c);
                if(obj.getRefTime()==0){
                    lowestCode=c;
                    break;
                }
                if(lowestTime>obj.getRefTime()){
                    lowestCode=c;
                    lowestTime=obj.getRefTime();
                }
            }
            suggestionCache.remove(lowestCode);
        }        
        suggestionCache.put(code,new CacheObject(l));
    }

    /**
     * Load the cache from file. The cach file has the same name as the 
     * dico file with the .pre extension added.
     */
    private void loadPreCache(File dicoFile)throws IOException{
        String              code;
        List                suggestions;
        long                size,
                            time;
        File                preFile;
        ObjectInputStream   in;
        
        preCacheFileName=dicoFile.getPath()+PRE_CACHE_FILE_EXT;
        preFile=new File(preCacheFileName);
        if(!preFile.exists()){
            System.out.println("No precache file");
            return;
        }
        System.out.println("Precaching...");
        in=new ObjectInputStream(new FileInputStream(preFile));
        try{
            size=in.readLong();
            for(int i=0;i<size;i++){
                code=(String)in.readObject();
                time=in.readLong();
                suggestions=(List)in.readObject();
                suggestionCache.put(code,new CacheObject(suggestions,time));
            }
        }catch(ClassNotFoundException ex){
            System.out.println(ex.getMessage());
        }
        in.close();
    }

    /**
     * Saves the current cache to file.
     */
    public void saveCache() throws IOException{
        String              code;
        CacheObject         obj;
        File                preFile;
        ObjectOutputStream  out;
        Iterator            it;

        if(preCacheFileName==null){
            System.out.println("NULL");
            return;
        }

        System.out.println("Saving cache to precache file...");
        preFile=new File(preCacheFileName);
        out=new ObjectOutputStream(new FileOutputStream(preFile));
        it=suggestionCache.keySet().iterator();
        out.writeLong(suggestionCache.size());
        while(it.hasNext()){
            code=(String)it.next();
            obj=(CacheObject)suggestionCache.get(code);
            out.writeObject(code);
            out.writeLong(obj.getRefTime());
            out.writeObject(obj.getSuggestionList());
        }
        out.close();
    }

    // INNER CLASSES
    // ------------------------------------------------------------------------
    private class CacheObject implements Serializable{
    
        private List    suggestions=null;
        private long    refTime=0;

        public CacheObject(List list){
            this.suggestions=list;
        }

        public CacheObject(List list, long time){
            this.suggestions=list;
            this.refTime=time;
        }
        
        public List getSuggestionList(){
            return suggestions;
        }

        public void setRefTime(){
            refTime=System.currentTimeMillis();
        }

        public long getRefTime(){
            return refTime;
        }
    }
}
{'fraction_non_alphanumeric': 0.10265145904310473, 'fraction_numerical': 0.0019256406458302473, 'mean_word_length': 1.8119533527696794, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7249676', 'n_tokens_mistral': 3587, 'n_tokens_neox': 3471, 'n_words': 837}