<commit_before>/**
 * Copyright (c) 2005-2013 by Appcelerator, Inc. All Rights Reserved.
 * Licensed under the terms of the Eclipse Public License (EPL).
 * Please see the license.txt included with this distribution for details.
 * Any modifications to this file must keep this entire header intact.
 */
/*
 * Created on 07/09/2005
 */
package com.python.pydev.analysis.additionalinfo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;

import org.eclipse.jface.text.IDocument;
import org.python.pydev.ast.codecompletion.revisited.PyPublicTreeMap;
import org.python.pydev.core.FileUtilsFileBuffer;
import org.python.pydev.core.IInfo;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.MisconfigurationException;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.ModulesKeyForZip;
import org.python.pydev.core.ObjectsInternPool;
import org.python.pydev.core.log.Log;
import org.python.pydev.core.logging.DebugSettings;
import org.python.pydev.parser.fastparser.FastDefinitionsParser;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.parser.visitors.NodeUtils;
import org.python.pydev.parser.visitors.scope.ASTEntry;
import org.python.pydev.parser.visitors.scope.DefinitionsASTIteratorVisitor;
import org.python.pydev.shared_core.string.FastStringBuffer;
import org.python.pydev.shared_core.string.FullRepIterable;
import org.python.pydev.shared_core.string.StringUtils;
import org.python.pydev.shared_core.structure.FastStack;
import org.python.pydev.shared_core.structure.Tuple;
import org.python.pydev.shared_core.structure.Tuple3;

/**
 * This class contains additional information on an interpreter, so that we are able to make code-completion in
 * a context-insensitive way (and make additionally auto-import).
 *
 * The information that is needed for that is the following:
 *
 * - Classes that are available in the global context
 * - Methods that are available in the global context
 *
 * We must access this information very fast, so the underlying structure has to take that into consideration.
 *
 * It should not 'eat' too much memory because it should be all in memory at all times
 *
 * It should also be easy to query it.
 *      Some query situations include:
 *          - which classes have the method xxx and yyy?
 *          - which methods and classes start with xxx?
 *          - is there any class or method with the name xxx?
 *
 * The information must be persisted for reuse (and persisting and restoring it should be fast).
 *
 * We need to store information for any interpreter, be it python, jython...
 *
 * For creating and keeping this information up-to-date, we have to know when:
 * - the interpreter used changes (the InterpreterInfo should be passed after the change)
 * - some file changes (pydev_builder)
 *
 * @author Fabio
 */
public abstract class AbstractAdditionalTokensInfo {

    /**
     * this is the number of initials that is used for indexing
     */
    public static final int NUMBER_OF_INITIALS_TO_INDEX = 3;

    /**
     * Do you want to debug this class?
     */
    private static final boolean DEBUG_ADDITIONAL_INFO = false;

    /**
     * Defines that some operation should be done on top level tokens
     */
    public final static int TOP_LEVEL = 1;

    /**
     * Defines that some operation should be done on inner level tokens
     */
    public final static int INNER = 2;

    /**
     * indexes used so that we can access the information faster - it is ordered through a tree map, and should be
     * very fast to access given its initials.
     *
     * It contains only top/level information for a module
     *
     * This map is persisted.
     */
    protected SortedMap<String, Set<IInfo>> topLevelInitialsToInfo = new PyPublicTreeMap<String, Set<IInfo>>();

    /**
     * indexes so that we can get 'inner information' from classes, such as methods or inner classes from a class
     */
    protected SortedMap<String, Set<IInfo>> innerInitialsToInfo = new PyPublicTreeMap<String, Set<IInfo>>();

    /**
     * Should be used before re-creating the info, so that we have enough memory.
     */
    public void clearAllInfo() {
        synchronized (lock) {
            if (topLevelInitialsToInfo != null) {
                topLevelInitialsToInfo.clear();
            }
            if (innerInitialsToInfo != null) {
                innerInitialsToInfo.clear();
            }
        }
    }

    protected Object lock = new Object();

    /**
     * The filter interface
     */
    public interface Filter {
        boolean doCompare(String lowerCaseQual, IInfo info);

        boolean doCompare(String lowerCaseQual, String infoName);
    }

    /**
     * A filter that checks if tokens are equal
     */
    private final Filter equalsFilter = new Filter() {
        @Override
        public boolean doCompare(String qualifier, IInfo info) {
            return info.getName().equals(qualifier);
        }

        @Override
        public boolean doCompare(String qualifier, String infoName) {
            return infoName.equals(qualifier);
        }
    };

    /**
     * A filter that checks if the tokens starts with a qualifier
     */
    private final Filter startingWithFilter = new Filter() {

        @Override
        public boolean doCompare(String lowerCaseQual, IInfo info) {
            return doCompare(lowerCaseQual, info.getName());
        }

        @Override
        public boolean doCompare(String qualifier, String infoName) {
            return infoName.toLowerCase().startsWith(qualifier);
        }

    };

    /**
     * 2: because we've removed some info (the hash is no longer saved)
     * 3: Changed from string-> list to string->set
     * 4: Keeping file, line and column for entries
     */
    protected static final int version = 4;

    public AbstractAdditionalTokensInfo() {
    }

    /**
     * That's the function actually used to add some info
     *
     * @param info information to be added
     */
    protected void add(IInfo info, int doOn) {
        synchronized (lock) {
            String name = info.getName();
            String initials = getInitials(name);
            SortedMap<String, Set<IInfo>> initialsToInfo;

            if (doOn == TOP_LEVEL) {
                if (info.getPath() != null && info.getPath().length() > 0) {
                    throw new RuntimeException(
                            "Error: the info being added is added as an 'top level' info, but has path. Info:" + info);
                }
                initialsToInfo = topLevelInitialsToInfo;

            } else if (doOn == INNER) {
                if (info.getPath() == null || info.getPath().length() == 0) {
                    throw new RuntimeException(
                            "Error: the info being added is added as an 'inner' info, but does not have a path. Info: "
                                    + info);
                }
                initialsToInfo = innerInitialsToInfo;

            } else {
                throw new RuntimeException("List to add is invalid: " + doOn);
            }
            Set<IInfo> listForInitials = getAndCreateListForInitials(initials, initialsToInfo);
            listForInitials.add(info);
        }
    }

    /**
     * @param name the name from where we want to get the initials
     * @return the initials for the name
     */
    protected String getInitials(String name) {
        if (name.length() < NUMBER_OF_INITIALS_TO_INDEX) {
            return name;
        }
        return name.substring(0, NUMBER_OF_INITIALS_TO_INDEX).toLowerCase();
    }

    /**
     * @param initials the initials we are looking for
     * @param initialsToInfo this is the list we should use (top level or inner)
     * @return the list of tokens with the specified initials (must be exact match)
     */
    protected Set<IInfo> getAndCreateListForInitials(String initials, SortedMap<String, Set<IInfo>> initialsToInfo) {
        Set<IInfo> lInfo = initialsToInfo.get(initials);
        if (lInfo == null) {
            lInfo = new HashSet<IInfo>();
            initialsToInfo.put(initials, lInfo);
        }
        return lInfo;
    }

    private IInfo addAssignTargets(ASTEntry entry, String moduleName, int doOn, String path, boolean lastIsMethod,
            String file) {
        String rep = NodeUtils.getFullRepresentationString(entry.node);
        if (lastIsMethod) {
            List<String> parts = StringUtils.dotSplit(rep);
            if (parts.size() >= 2) {
                //at least 2 parts are required
                if (parts.get(0).equals("self")) {
                    rep = parts.get(1);
                    //no intern construct (locked in the loop that calls this method)
                    AttrInfo info = new AttrInfo(ObjectsInternPool.internUnsynched(rep), moduleName,
                            ObjectsInternPool.internUnsynched(path), false, getNature(), file,
                            entry.node.beginLine, entry.node.beginColumn);
                    add(info, doOn);
                    return info;
                }
            }
        } else {
            //no intern construct (locked in the loop that calls this method)
            AttrInfo info = new AttrInfo(ObjectsInternPool.internUnsynched(FullRepIterable.getFirstPart(rep)),
                    moduleName,
                    ObjectsInternPool.internUnsynched(path), false, getNature(), file,
                    entry.node.beginLine, entry.node.beginColumn);
            add(info, doOn);
            return info;
        }
        return null;
    }

    protected abstract IPythonNature getNature();

    public List<IInfo> addAstInfo(ModulesKey key, boolean generateDelta) throws Exception {
        boolean isZipModule = key instanceof ModulesKeyForZip;
        ModulesKeyForZip modulesKeyForZip = null;
        if (isZipModule) {
            modulesKeyForZip = (ModulesKeyForZip) key;
        }

        Object doc;
        if (isZipModule) {
            doc = FileUtilsFileBuffer.getCustomReturnFromZip(modulesKeyForZip.file, modulesKeyForZip.zipModulePath,
                    null);

        } else {
            doc = FileUtilsFileBuffer.getCustomReturnFromFile(key.file, true, null);
        }

        char[] charArray;
        int len;
        if (doc instanceof IDocument) {
            IDocument document = (IDocument) doc;
            charArray = document.get().toCharArray();
            len = charArray.length;

        } else if (doc instanceof FastStringBuffer) {
            FastStringBuffer fastStringBuffer = (FastStringBuffer) doc;
            //In this case, we can actually get the internal array without doing any copies (and just specifying the len).
            charArray = fastStringBuffer.getInternalCharsArray();
            len = fastStringBuffer.length();

        } else if (doc instanceof String) {
            String str = (String) doc;
            charArray = str.toCharArray();
            len = charArray.length;

        } else if (doc instanceof char[]) {
            charArray = (char[]) doc;
            len = charArray.length;

        } else {
            throw new RuntimeException("Don't know how to handle: " + doc + " -- " + doc.getClass());
        }

        SimpleNode node = FastDefinitionsParser.parse(charArray, key.file.getName(), len, key.file);
        if (node == null) {
            return null;
        }

        return addAstInfo(node, key, generateDelta);
    }

    /**
     * Adds ast info information for a module.
     *
     * @param m the module we want to add to the info
     */
    public List<IInfo> addAstInfo(SimpleNode node, ModulesKey key, boolean generateDelta) {
        List<IInfo> createdInfos = new ArrayList<IInfo>();
        if (node == null || key.name == null) {
            return createdInfos;
        }
        try {
            Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>> tup = getInnerEntriesForAST(node);
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Adding ast info to: " + key.name);
            }

            try {
                Iterator<ASTEntry> entries = tup.o2;

                FastStack<SimpleNode> tempStack = new FastStack<SimpleNode>(10);

                synchronized (this.lock) {
                    synchronized (ObjectsInternPool.lock) {
                        final String file = key.file != null ? ObjectsInternPool.internUnsynched(key.file.toString())
                                : null;
                        key.name = ObjectsInternPool.internUnsynched(key.name);

                        while (entries.hasNext()) {
                            ASTEntry entry = entries.next();
                            IInfo infoCreated = null;

                            if (entry.parent == null) { //we only want those that are in the global scope
                                if (entry.node instanceof ClassDef) {
                                    //no intern construct (locked in this loop)
                                    NameTok name = (NameTok) ((ClassDef) entry.node).name;
                                    ClassInfo info = new ClassInfo(
                                            ObjectsInternPool.internUnsynched(name.id),
                                            key.name, null, false, getNature(), file, name.beginLine,
                                            name.beginColumn);
                                    add(info, TOP_LEVEL);
                                    infoCreated = info;

                                } else if (entry.node instanceof FunctionDef) {
                                    //no intern construct (locked in this loop)
                                    NameTok name = (NameTok) ((FunctionDef) entry.node).name;
                                    FuncInfo info2 = new FuncInfo(
                                            ObjectsInternPool.internUnsynched(name.id),
                                            key.name, null, false, getNature(), file, name.beginLine,
                                            name.beginColumn);
                                    add(info2, TOP_LEVEL);
                                    infoCreated = info2;

                                } else {
                                    //it is an assign
                                    infoCreated = this.addAssignTargets(entry, key.name, TOP_LEVEL, null, false, file);

                                }
                            } else {
                                if (entry.node instanceof ClassDef || entry.node instanceof FunctionDef) {
                                    //ok, it has a parent, so, let's check to see if the path we got only has class definitions
                                    //as the parent (and get that path)
                                    Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, false, false,
                                            tempStack);
                                    if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                        //if the root is not valid, it is not only classes in the path (could be a method inside
                                        //a method, or something similar).

                                        if (entry.node instanceof ClassDef) {
                                            NameTok name = ((NameTok) ((ClassDef) entry.node).name);
                                            ClassInfo info = new ClassInfo(
                                                    ObjectsInternPool.internUnsynched(name.id),
                                                    key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false,
                                                    getNature(), file, name.beginLine, name.beginColumn);
                                            add(info, INNER);
                                            infoCreated = info;

                                        } else {
                                            //FunctionDef
                                            NameTok name = ((NameTok) ((FunctionDef) entry.node).name);
                                            FuncInfo info2 = new FuncInfo(
                                                    ObjectsInternPool.internUnsynched(name.id),
                                                    key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false,
                                                    getNature(), file, name.beginLine, name.beginColumn);
                                            add(info2, INNER);
                                            infoCreated = info2;

                                        }
                                    }
                                } else {
                                    //it is an assign
                                    Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, true, false,
                                            tempStack);
                                    if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                        infoCreated = this.addAssignTargets(entry, key.name, INNER, pathToRoot.o1,
                                                pathToRoot.o2, file);
                                    }
                                }
                            }

                            if (infoCreated != null) {
                                createdInfos.add(infoCreated);
                            }

                        } //end while

                    } //end lock ObjectsPool.lock

                } //end this.lock

            } catch (Exception e) {
                Log.log(e);
            }
        } catch (Exception e) {
            Log.log(e);
        }
        return createdInfos;
    }

    /**
     * @return an iterator that'll get the outline entries for the given ast.
     */
    public static Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>> getInnerEntriesForAST(SimpleNode node)
            throws Exception {
        DefinitionsASTIteratorVisitor visitor = new DefinitionsASTIteratorVisitor();
        node.accept(visitor);
        Iterator<ASTEntry> entries = visitor.getOutline();
        return new Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>>(visitor, entries);
    }

    /**
     * @return a set with the module names that have tokens.
     */
    public Set<String> getAllModulesWithTokens() {
        HashSet<String> ret = new HashSet<String>();
        synchronized (lock) {
            Set<Entry<String, Set<IInfo>>> entrySet = this.topLevelInitialsToInfo.entrySet();
            for (Entry<String, Set<IInfo>> entry : entrySet) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    ret.add(info.getDeclaringModuleName());
                }
            }

            entrySet = this.innerInitialsToInfo.entrySet();
            for (Entry<String, Set<IInfo>> entry : entrySet) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    ret.add(info.getDeclaringModuleName());
                }
            }
        }
        return ret;

    }

    /**
     * @param lastMayBeMethod if true, it gets the path and accepts a method (if it is the last in the stack)
     * if false, null is returned if a method is found.
     *
     * @param tempStack is a temporary stack object (which may be cleared)
     *
     * @return a tuple, where the first element is the path where the entry is located (may return null).
     * and the second element is a boolean that indicates if the last was actually a method or not.
     */
    private Tuple<String, Boolean> getPathToRoot(ASTEntry entry, boolean lastMayBeMethod, boolean acceptAny,
            FastStack<SimpleNode> tempStack) {
        if (entry.parent == null) {
            return null;
        }
        //just to be sure that it's empty
        tempStack.clear();

        boolean lastIsMethod = false;
        //if the last 'may be a method', in this case, we have to remember that it will actually be the first one
        //to be analyzed.

        //let's get the stack
        while (entry.parent != null) {
            if (entry.parent.node instanceof ClassDef) {
                tempStack.push(entry.parent.node);

            } else if (entry.parent.node instanceof FunctionDef) {
                if (!acceptAny) {
                    if (lastIsMethod) {
                        //already found a method
                        return null;
                    }

                    if (!lastMayBeMethod) {
                        return null;
                    }

                    //ok, the last one may be a method... (in this search, it MUST be the first one...)
                    if (tempStack.size() != 0) {
                        return null;
                    }
                }

                //ok, there was a class, so, let's go and set it
                tempStack.push(entry.parent.node);
                lastIsMethod = true;

            } else {
                return null;

            }
            entry = entry.parent;
        }

        //now that we have the stack, let's make it into a path...
        FastStringBuffer buf = new FastStringBuffer();
        while (tempStack.size() > 0) {
            String rep = NodeUtils.getRepresentationString(tempStack.pop());
            if (rep != null) {
                if (buf.length() > 0) {
                    buf.append(".");
                }
                buf.append(rep);
            }
        }
        return new Tuple<String, Boolean>(buf.toString(), lastIsMethod);
    }

    /**
     * Removes all the info associated with a given module
     * @param moduleName the name of the module we want to remove info from
     */
    public void removeInfoFromModule(String moduleName, boolean generateDelta) {
        if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Removing ast info from: " + moduleName);
        }
        synchronized (lock) {
            removeInfoFromMap(moduleName, topLevelInitialsToInfo);
            removeInfoFromMap(moduleName, innerInitialsToInfo);
        }

    }

    /**
     * @param moduleName
     * @param initialsToInfo
     */
    private void removeInfoFromMap(String moduleName, SortedMap<String, Set<IInfo>> initialsToInfo) {
        Iterator<Set<IInfo>> itListOfInfo = initialsToInfo.values().iterator();
        while (itListOfInfo.hasNext()) {

            Iterator<IInfo> it = itListOfInfo.next().iterator();
            while (it.hasNext()) {

                IInfo info = it.next();
                if (info != null && info.getDeclaringModuleName() != null) {
                    if (info.getDeclaringModuleName().equals(moduleName)) {
                        it.remove();
                    }
                }
            }
        }
    }

    /**
     * This is the function for which we are most optimized!
     *
     * @param qualifier the tokens returned have to start with the given qualifier
     * @return a list of info, all starting with the given qualifier
     */
    public Collection<IInfo> getTokensStartingWith(String qualifier, int getWhat) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, startingWithFilter, true, null);
        }
    }

    public Collection<IInfo> getTokensStartingWith(String qualifier, int getWhat, Collection<IInfo> result) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, startingWithFilter, true, result);
        }
    }

    public Collection<IInfo> getTokensEqualTo(String qualifier, int getWhat) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, equalsFilter, false, null);
        }
    }

    public Collection<IInfo> getTokensEqualTo(String qualifier, int getWhat, Collection<IInfo> result) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, equalsFilter, false, result);
        }
    }

    protected Collection<IInfo> getWithFilter(String qualifier, int getWhat, Filter filter, boolean useLowerCaseQual,
            Collection<IInfo> result) {
        synchronized (lock) {
            if (result == null) {
                result = new ArrayList<IInfo>();
            }

            if ((getWhat & TOP_LEVEL) != 0) {
                getWithFilter(qualifier, topLevelInitialsToInfo, result, filter, useLowerCaseQual);
            }
            if ((getWhat & INNER) != 0) {
                getWithFilter(qualifier, innerInitialsToInfo, result, filter, useLowerCaseQual);
            }
            return result;
        }
    }

    /**
     * @param qualifier
     * @param initialsToInfo this is where we are going to get the info from (currently: inner or top level list)
     * @param toks (out) the tokens will be added to this list
     * @return
     */
    protected void getWithFilter(String qualifier, SortedMap<String, Set<IInfo>> initialsToInfo,
            Collection<IInfo> toks, Filter filter, boolean useLowerCaseQual) {
        String initials = getInitials(qualifier);
        String qualToCompare = qualifier;
        if (useLowerCaseQual) {
            qualToCompare = qualifier.toLowerCase();
        }

        //get until the end of the alphabet
        SortedMap<String, Set<IInfo>> subMap = initialsToInfo.subMap(initials, initials + "\uffff\uffff\uffff\uffff");

        for (Set<IInfo> listForInitials : subMap.values()) {

            for (IInfo info : listForInitials) {
                if (filter.doCompare(qualToCompare, info)) {
                    toks.add(info);
                }
            }
        }
    }

    /**
     * @return all the tokens that are in this info (top level or inner)
     */
    public Collection<IInfo> getAllTokens() {
        synchronized (lock) {
            Collection<Set<IInfo>> lInfo = this.topLevelInitialsToInfo.values();

            ArrayList<IInfo> toks = new ArrayList<IInfo>();
            for (Set<IInfo> list : lInfo) {
                for (IInfo info : list) {
                    toks.add(info);
                }
            }

            lInfo = this.innerInitialsToInfo.values();
            for (Set<IInfo> list : lInfo) {
                for (IInfo info : list) {
                    toks.add(info);
                }
            }
            return toks;
        }
    }

    /**
     * this can be used to save the file
     */
    public void save() {
        File persistingLocation;
        try {
            persistingLocation = getPersistingLocation();
        } catch (MisconfigurationException e) {
            Log.log("Error. Unable to get persisting location for additional interprer info. Configuration may be corrupted.",
                    e);
            return;
        }
        if (DEBUG_ADDITIONAL_INFO) {
            System.out.println("Saving to " + persistingLocation);
        }

        save(persistingLocation);

    }

    protected void save(File persistingLocation) {
        try {
            FileOutputStream stream = new FileOutputStream(persistingLocation);
            OutputStreamWriter writer = new OutputStreamWriter(stream);
            try {
                FastStringBuffer tempBuf = new FastStringBuffer();
                tempBuf.append("-- VERSION_");
                tempBuf.append(AbstractAdditionalTokensInfo.version);
                tempBuf.append('\n');
                writer.write(tempBuf.getInternalCharsArray(), 0, tempBuf.length());
                tempBuf.clear();

                saveTo(writer, tempBuf, persistingLocation);
            } finally {
                try {
                    writer.close();
                } finally {
                    stream.close();
                }
            }
        } catch (Exception e) {
            Log.log(e);
        }
    }

    /**
     * @return the location where we can persist this info.
     * @throws MisconfigurationException
     */
    protected abstract File getPersistingLocation() throws MisconfigurationException;

    /**
     * @return the path to the folder we want to keep things on
     * @throws MisconfigurationException
     */
    protected abstract File getPersistingFolder();

    protected void saveTo(OutputStreamWriter writer, FastStringBuffer tempBuf, File pathToSave) throws IOException {
        synchronized (lock) {
            if (DEBUG_ADDITIONAL_INFO) {
                System.out.println("Saving info " + this.getClass().getName() + " to file (size = "
                        + getAllTokens().size() + ") " + pathToSave);
            }

            Map<String, Integer> dictionary = new HashMap<String, Integer>();
            tempBuf.append("-- START TREE 1\n");
            TreeIO.dumpTreeToBuffer(this.topLevelInitialsToInfo, tempBuf, dictionary);

            tempBuf.append("-- START TREE 2\n");
            TreeIO.dumpTreeToBuffer(this.innerInitialsToInfo, tempBuf, dictionary);

            FastStringBuffer buf2 = new FastStringBuffer(50 * (dictionary.size() + 4));
            TreeIO.dumpDictToBuffer(dictionary, buf2);

            //Write the dictionary before the actual trees.
            writer.write(buf2.getInternalCharsArray(), 0, buf2.length());
            buf2 = null;

            //Note: tried LZFFileInputStream from https://github.com/ning/compress
            //and Snappy from https://github.com/dain/snappy checking to see if by writing less we'd
            //get a better time but it got a bit slower (gzip was slowest, then snappy and the faster was LZFFileInputStream)
            writer.write(tempBuf.getInternalCharsArray(), 0, tempBuf.length());
        }
    }

    /**
     * Restores the saved info in the object (if overridden, getInfoToSave should be overridden too)
     * @param o the read object from the file
     * @throws MisconfigurationException
     */
    @SuppressWarnings("unchecked")
    protected void restoreSavedInfo(Object o) throws MisconfigurationException {
        synchronized (lock) {
            Tuple3<Object, Object, Object> readFromFile = (Tuple3<Object, Object, Object>) o;
            SortedMap<String, Set<IInfo>> o1 = (SortedMap<String, Set<IInfo>>) readFromFile.o1;
            SortedMap<String, Set<IInfo>> o2 = (SortedMap<String, Set<IInfo>>) readFromFile.o2;

            if (o1 == null) {
                throw new RuntimeException("Error in I/O (topLevelInitialsToInfo is null). Rebuilding internal info.");
            }
            if (o2 == null) {
                throw new RuntimeException("Error in I/O (innerInitialsToInfo is null). Rebuilding internal info.");
            }
            this.topLevelInitialsToInfo = o1;
            this.innerInitialsToInfo = o2;
            if (readFromFile.o3 != null) {
                //may be null in new format (where that's checked during load time).
                if (AbstractAdditionalTokensInfo.version != (Integer) readFromFile.o3) {
                    throw new RuntimeException("I/O version doesn't match. Rebuilding internal info.");
                }
            }
        }
    }

    @Override
    public String toString() {
        synchronized (lock) {
            FastStringBuffer buffer = new FastStringBuffer();
            buffer.append("AdditionalInfo{");

            buffer.append("topLevel=[");
            entrySetToString(buffer, this.topLevelInitialsToInfo.entrySet());
            buffer.append("]\n");
            buffer.append("inner=[");
            entrySetToString(buffer, this.innerInitialsToInfo.entrySet());
            buffer.append("]");

            buffer.append("}");
            return buffer.toString();
        }
    }

    /**
     * @param buffer
     * @param name
     */
    private void entrySetToString(FastStringBuffer buffer, Set<Entry<String, Set<IInfo>>> name) {
        synchronized (lock) {
            for (Entry<String, Set<IInfo>> entry : name) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    buffer.append(info.toString());
                    buffer.append("\n");
                }
            }
        }
    }

}

class IOUtils {

    public static Object readFromFile(File astOutputFile) {
        try {
            InputStream in = new BufferedInputStream(new FileInputStream(astOutputFile));
            try {
                ObjectInputStream stream = new ObjectInputStream(in);
                try {
                    Object o = stream.readObject();
                    return o;
                } finally {
                    stream.close();
                }
            } finally {
                in.close();
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

}<commit_msg>Update version of additional info to restore information which may be wrong.<commit_after>/**
 * Copyright (c) 2005-2013 by Appcelerator, Inc. All Rights Reserved.
 * Licensed under the terms of the Eclipse Public License (EPL).
 * Please see the license.txt included with this distribution for details.
 * Any modifications to this file must keep this entire header intact.
 */
/*
 * Created on 07/09/2005
 */
package com.python.pydev.analysis.additionalinfo;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;

import org.eclipse.jface.text.IDocument;
import org.python.pydev.ast.codecompletion.revisited.PyPublicTreeMap;
import org.python.pydev.core.FileUtilsFileBuffer;
import org.python.pydev.core.IInfo;
import org.python.pydev.core.IPythonNature;
import org.python.pydev.core.MisconfigurationException;
import org.python.pydev.core.ModulesKey;
import org.python.pydev.core.ModulesKeyForZip;
import org.python.pydev.core.ObjectsInternPool;
import org.python.pydev.core.log.Log;
import org.python.pydev.core.logging.DebugSettings;
import org.python.pydev.parser.fastparser.FastDefinitionsParser;
import org.python.pydev.parser.jython.SimpleNode;
import org.python.pydev.parser.jython.ast.ClassDef;
import org.python.pydev.parser.jython.ast.FunctionDef;
import org.python.pydev.parser.jython.ast.NameTok;
import org.python.pydev.parser.visitors.NodeUtils;
import org.python.pydev.parser.visitors.scope.ASTEntry;
import org.python.pydev.parser.visitors.scope.DefinitionsASTIteratorVisitor;
import org.python.pydev.shared_core.string.FastStringBuffer;
import org.python.pydev.shared_core.string.FullRepIterable;
import org.python.pydev.shared_core.string.StringUtils;
import org.python.pydev.shared_core.structure.FastStack;
import org.python.pydev.shared_core.structure.Tuple;
import org.python.pydev.shared_core.structure.Tuple3;

/**
 * This class contains additional information on an interpreter, so that we are able to make code-completion in
 * a context-insensitive way (and make additionally auto-import).
 *
 * The information that is needed for that is the following:
 *
 * - Classes that are available in the global context
 * - Methods that are available in the global context
 *
 * We must access this information very fast, so the underlying structure has to take that into consideration.
 *
 * It should not 'eat' too much memory because it should be all in memory at all times
 *
 * It should also be easy to query it.
 *      Some query situations include:
 *          - which classes have the method xxx and yyy?
 *          - which methods and classes start with xxx?
 *          - is there any class or method with the name xxx?
 *
 * The information must be persisted for reuse (and persisting and restoring it should be fast).
 *
 * We need to store information for any interpreter, be it python, jython...
 *
 * For creating and keeping this information up-to-date, we have to know when:
 * - the interpreter used changes (the InterpreterInfo should be passed after the change)
 * - some file changes (pydev_builder)
 *
 * @author Fabio
 */
public abstract class AbstractAdditionalTokensInfo {

    /**
     * this is the number of initials that is used for indexing
     */
    public static final int NUMBER_OF_INITIALS_TO_INDEX = 3;

    /**
     * Do you want to debug this class?
     */
    private static final boolean DEBUG_ADDITIONAL_INFO = false;

    /**
     * Defines that some operation should be done on top level tokens
     */
    public final static int TOP_LEVEL = 1;

    /**
     * Defines that some operation should be done on inner level tokens
     */
    public final static int INNER = 2;

    /**
     * indexes used so that we can access the information faster - it is ordered through a tree map, and should be
     * very fast to access given its initials.
     *
     * It contains only top/level information for a module
     *
     * This map is persisted.
     */
    protected SortedMap<String, Set<IInfo>> topLevelInitialsToInfo = new PyPublicTreeMap<String, Set<IInfo>>();

    /**
     * indexes so that we can get 'inner information' from classes, such as methods or inner classes from a class
     */
    protected SortedMap<String, Set<IInfo>> innerInitialsToInfo = new PyPublicTreeMap<String, Set<IInfo>>();

    /**
     * Should be used before re-creating the info, so that we have enough memory.
     */
    public void clearAllInfo() {
        synchronized (lock) {
            if (topLevelInitialsToInfo != null) {
                topLevelInitialsToInfo.clear();
            }
            if (innerInitialsToInfo != null) {
                innerInitialsToInfo.clear();
            }
        }
    }

    protected Object lock = new Object();

    /**
     * The filter interface
     */
    public interface Filter {
        boolean doCompare(String lowerCaseQual, IInfo info);

        boolean doCompare(String lowerCaseQual, String infoName);
    }

    /**
     * A filter that checks if tokens are equal
     */
    private final Filter equalsFilter = new Filter() {
        @Override
        public boolean doCompare(String qualifier, IInfo info) {
            return info.getName().equals(qualifier);
        }

        @Override
        public boolean doCompare(String qualifier, String infoName) {
            return infoName.equals(qualifier);
        }
    };

    /**
     * A filter that checks if the tokens starts with a qualifier
     */
    private final Filter startingWithFilter = new Filter() {

        @Override
        public boolean doCompare(String lowerCaseQual, IInfo info) {
            return doCompare(lowerCaseQual, info.getName());
        }

        @Override
        public boolean doCompare(String qualifier, String infoName) {
            return infoName.toLowerCase().startsWith(qualifier);
        }

    };

    /**
     * 2: because we've removed some info (the hash is no longer saved)
     * 3: Changed from string-> list to string->set
     * 4: Keeping file, line and column for entries
     * 4: Fix when restoring information
     */
    protected static final int version = 5;

    public AbstractAdditionalTokensInfo() {
    }

    /**
     * That's the function actually used to add some info
     *
     * @param info information to be added
     */
    protected void add(IInfo info, int doOn) {
        synchronized (lock) {
            String name = info.getName();
            String initials = getInitials(name);
            SortedMap<String, Set<IInfo>> initialsToInfo;

            if (doOn == TOP_LEVEL) {
                if (info.getPath() != null && info.getPath().length() > 0) {
                    throw new RuntimeException(
                            "Error: the info being added is added as an 'top level' info, but has path. Info:" + info);
                }
                initialsToInfo = topLevelInitialsToInfo;

            } else if (doOn == INNER) {
                if (info.getPath() == null || info.getPath().length() == 0) {
                    throw new RuntimeException(
                            "Error: the info being added is added as an 'inner' info, but does not have a path. Info: "
                                    + info);
                }
                initialsToInfo = innerInitialsToInfo;

            } else {
                throw new RuntimeException("List to add is invalid: " + doOn);
            }
            Set<IInfo> listForInitials = getAndCreateListForInitials(initials, initialsToInfo);
            listForInitials.add(info);
        }
    }

    /**
     * @param name the name from where we want to get the initials
     * @return the initials for the name
     */
    protected String getInitials(String name) {
        if (name.length() < NUMBER_OF_INITIALS_TO_INDEX) {
            return name;
        }
        return name.substring(0, NUMBER_OF_INITIALS_TO_INDEX).toLowerCase();
    }

    /**
     * @param initials the initials we are looking for
     * @param initialsToInfo this is the list we should use (top level or inner)
     * @return the list of tokens with the specified initials (must be exact match)
     */
    protected Set<IInfo> getAndCreateListForInitials(String initials, SortedMap<String, Set<IInfo>> initialsToInfo) {
        Set<IInfo> lInfo = initialsToInfo.get(initials);
        if (lInfo == null) {
            lInfo = new HashSet<IInfo>();
            initialsToInfo.put(initials, lInfo);
        }
        return lInfo;
    }

    private IInfo addAssignTargets(ASTEntry entry, String moduleName, int doOn, String path, boolean lastIsMethod,
            String file) {
        String rep = NodeUtils.getFullRepresentationString(entry.node);
        if (lastIsMethod) {
            List<String> parts = StringUtils.dotSplit(rep);
            if (parts.size() >= 2) {
                //at least 2 parts are required
                if (parts.get(0).equals("self")) {
                    rep = parts.get(1);
                    //no intern construct (locked in the loop that calls this method)
                    AttrInfo info = new AttrInfo(ObjectsInternPool.internUnsynched(rep), moduleName,
                            ObjectsInternPool.internUnsynched(path), false, getNature(), file,
                            entry.node.beginLine, entry.node.beginColumn);
                    add(info, doOn);
                    return info;
                }
            }
        } else {
            //no intern construct (locked in the loop that calls this method)
            AttrInfo info = new AttrInfo(ObjectsInternPool.internUnsynched(FullRepIterable.getFirstPart(rep)),
                    moduleName,
                    ObjectsInternPool.internUnsynched(path), false, getNature(), file,
                    entry.node.beginLine, entry.node.beginColumn);
            add(info, doOn);
            return info;
        }
        return null;
    }

    protected abstract IPythonNature getNature();

    public List<IInfo> addAstInfo(ModulesKey key, boolean generateDelta) throws Exception {
        boolean isZipModule = key instanceof ModulesKeyForZip;
        ModulesKeyForZip modulesKeyForZip = null;
        if (isZipModule) {
            modulesKeyForZip = (ModulesKeyForZip) key;
        }

        Object doc;
        if (isZipModule) {
            doc = FileUtilsFileBuffer.getCustomReturnFromZip(modulesKeyForZip.file, modulesKeyForZip.zipModulePath,
                    null);

        } else {
            doc = FileUtilsFileBuffer.getCustomReturnFromFile(key.file, true, null);
        }

        char[] charArray;
        int len;
        if (doc instanceof IDocument) {
            IDocument document = (IDocument) doc;
            charArray = document.get().toCharArray();
            len = charArray.length;

        } else if (doc instanceof FastStringBuffer) {
            FastStringBuffer fastStringBuffer = (FastStringBuffer) doc;
            //In this case, we can actually get the internal array without doing any copies (and just specifying the len).
            charArray = fastStringBuffer.getInternalCharsArray();
            len = fastStringBuffer.length();

        } else if (doc instanceof String) {
            String str = (String) doc;
            charArray = str.toCharArray();
            len = charArray.length;

        } else if (doc instanceof char[]) {
            charArray = (char[]) doc;
            len = charArray.length;

        } else {
            throw new RuntimeException("Don't know how to handle: " + doc + " -- " + doc.getClass());
        }

        SimpleNode node = FastDefinitionsParser.parse(charArray, key.file.getName(), len, key.file);
        if (node == null) {
            return null;
        }

        return addAstInfo(node, key, generateDelta);
    }

    /**
     * Adds ast info information for a module.
     *
     * @param m the module we want to add to the info
     */
    public List<IInfo> addAstInfo(SimpleNode node, ModulesKey key, boolean generateDelta) {
        List<IInfo> createdInfos = new ArrayList<IInfo>();
        if (node == null || key.name == null) {
            return createdInfos;
        }
        try {
            Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>> tup = getInnerEntriesForAST(node);
            if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
                org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Adding ast info to: " + key.name);
            }

            try {
                Iterator<ASTEntry> entries = tup.o2;

                FastStack<SimpleNode> tempStack = new FastStack<SimpleNode>(10);

                synchronized (this.lock) {
                    synchronized (ObjectsInternPool.lock) {
                        final String file = key.file != null ? ObjectsInternPool.internUnsynched(key.file.toString())
                                : null;
                        key.name = ObjectsInternPool.internUnsynched(key.name);

                        while (entries.hasNext()) {
                            ASTEntry entry = entries.next();
                            IInfo infoCreated = null;

                            if (entry.parent == null) { //we only want those that are in the global scope
                                if (entry.node instanceof ClassDef) {
                                    //no intern construct (locked in this loop)
                                    NameTok name = (NameTok) ((ClassDef) entry.node).name;
                                    ClassInfo info = new ClassInfo(
                                            ObjectsInternPool.internUnsynched(name.id),
                                            key.name, null, false, getNature(), file, name.beginLine,
                                            name.beginColumn);
                                    add(info, TOP_LEVEL);
                                    infoCreated = info;

                                } else if (entry.node instanceof FunctionDef) {
                                    //no intern construct (locked in this loop)
                                    NameTok name = (NameTok) ((FunctionDef) entry.node).name;
                                    FuncInfo info2 = new FuncInfo(
                                            ObjectsInternPool.internUnsynched(name.id),
                                            key.name, null, false, getNature(), file, name.beginLine,
                                            name.beginColumn);
                                    add(info2, TOP_LEVEL);
                                    infoCreated = info2;

                                } else {
                                    //it is an assign
                                    infoCreated = this.addAssignTargets(entry, key.name, TOP_LEVEL, null, false, file);

                                }
                            } else {
                                if (entry.node instanceof ClassDef || entry.node instanceof FunctionDef) {
                                    //ok, it has a parent, so, let's check to see if the path we got only has class definitions
                                    //as the parent (and get that path)
                                    Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, false, false,
                                            tempStack);
                                    if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                        //if the root is not valid, it is not only classes in the path (could be a method inside
                                        //a method, or something similar).

                                        if (entry.node instanceof ClassDef) {
                                            NameTok name = ((NameTok) ((ClassDef) entry.node).name);
                                            ClassInfo info = new ClassInfo(
                                                    ObjectsInternPool.internUnsynched(name.id),
                                                    key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false,
                                                    getNature(), file, name.beginLine, name.beginColumn);
                                            add(info, INNER);
                                            infoCreated = info;

                                        } else {
                                            //FunctionDef
                                            NameTok name = ((NameTok) ((FunctionDef) entry.node).name);
                                            FuncInfo info2 = new FuncInfo(
                                                    ObjectsInternPool.internUnsynched(name.id),
                                                    key.name, ObjectsInternPool.internUnsynched(pathToRoot.o1), false,
                                                    getNature(), file, name.beginLine, name.beginColumn);
                                            add(info2, INNER);
                                            infoCreated = info2;

                                        }
                                    }
                                } else {
                                    //it is an assign
                                    Tuple<String, Boolean> pathToRoot = this.getPathToRoot(entry, true, false,
                                            tempStack);
                                    if (pathToRoot != null && pathToRoot.o1 != null && pathToRoot.o1.length() > 0) {
                                        infoCreated = this.addAssignTargets(entry, key.name, INNER, pathToRoot.o1,
                                                pathToRoot.o2, file);
                                    }
                                }
                            }

                            if (infoCreated != null) {
                                createdInfos.add(infoCreated);
                            }

                        } //end while

                    } //end lock ObjectsPool.lock

                } //end this.lock

            } catch (Exception e) {
                Log.log(e);
            }
        } catch (Exception e) {
            Log.log(e);
        }
        return createdInfos;
    }

    /**
     * @return an iterator that'll get the outline entries for the given ast.
     */
    public static Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>> getInnerEntriesForAST(SimpleNode node)
            throws Exception {
        DefinitionsASTIteratorVisitor visitor = new DefinitionsASTIteratorVisitor();
        node.accept(visitor);
        Iterator<ASTEntry> entries = visitor.getOutline();
        return new Tuple<DefinitionsASTIteratorVisitor, Iterator<ASTEntry>>(visitor, entries);
    }

    /**
     * @return a set with the module names that have tokens.
     */
    public Set<String> getAllModulesWithTokens() {
        HashSet<String> ret = new HashSet<String>();
        synchronized (lock) {
            Set<Entry<String, Set<IInfo>>> entrySet = this.topLevelInitialsToInfo.entrySet();
            for (Entry<String, Set<IInfo>> entry : entrySet) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    ret.add(info.getDeclaringModuleName());
                }
            }

            entrySet = this.innerInitialsToInfo.entrySet();
            for (Entry<String, Set<IInfo>> entry : entrySet) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    ret.add(info.getDeclaringModuleName());
                }
            }
        }
        return ret;

    }

    /**
     * @param lastMayBeMethod if true, it gets the path and accepts a method (if it is the last in the stack)
     * if false, null is returned if a method is found.
     *
     * @param tempStack is a temporary stack object (which may be cleared)
     *
     * @return a tuple, where the first element is the path where the entry is located (may return null).
     * and the second element is a boolean that indicates if the last was actually a method or not.
     */
    private Tuple<String, Boolean> getPathToRoot(ASTEntry entry, boolean lastMayBeMethod, boolean acceptAny,
            FastStack<SimpleNode> tempStack) {
        if (entry.parent == null) {
            return null;
        }
        //just to be sure that it's empty
        tempStack.clear();

        boolean lastIsMethod = false;
        //if the last 'may be a method', in this case, we have to remember that it will actually be the first one
        //to be analyzed.

        //let's get the stack
        while (entry.parent != null) {
            if (entry.parent.node instanceof ClassDef) {
                tempStack.push(entry.parent.node);

            } else if (entry.parent.node instanceof FunctionDef) {
                if (!acceptAny) {
                    if (lastIsMethod) {
                        //already found a method
                        return null;
                    }

                    if (!lastMayBeMethod) {
                        return null;
                    }

                    //ok, the last one may be a method... (in this search, it MUST be the first one...)
                    if (tempStack.size() != 0) {
                        return null;
                    }
                }

                //ok, there was a class, so, let's go and set it
                tempStack.push(entry.parent.node);
                lastIsMethod = true;

            } else {
                return null;

            }
            entry = entry.parent;
        }

        //now that we have the stack, let's make it into a path...
        FastStringBuffer buf = new FastStringBuffer();
        while (tempStack.size() > 0) {
            String rep = NodeUtils.getRepresentationString(tempStack.pop());
            if (rep != null) {
                if (buf.length() > 0) {
                    buf.append(".");
                }
                buf.append(rep);
            }
        }
        return new Tuple<String, Boolean>(buf.toString(), lastIsMethod);
    }

    /**
     * Removes all the info associated with a given module
     * @param moduleName the name of the module we want to remove info from
     */
    public void removeInfoFromModule(String moduleName, boolean generateDelta) {
        if (DebugSettings.DEBUG_ANALYSIS_REQUESTS) {
            org.python.pydev.shared_core.log.ToLogFile.toLogFile(this, "Removing ast info from: " + moduleName);
        }
        synchronized (lock) {
            removeInfoFromMap(moduleName, topLevelInitialsToInfo);
            removeInfoFromMap(moduleName, innerInitialsToInfo);
        }

    }

    /**
     * @param moduleName
     * @param initialsToInfo
     */
    private void removeInfoFromMap(String moduleName, SortedMap<String, Set<IInfo>> initialsToInfo) {
        Iterator<Set<IInfo>> itListOfInfo = initialsToInfo.values().iterator();
        while (itListOfInfo.hasNext()) {

            Iterator<IInfo> it = itListOfInfo.next().iterator();
            while (it.hasNext()) {

                IInfo info = it.next();
                if (info != null && info.getDeclaringModuleName() != null) {
                    if (info.getDeclaringModuleName().equals(moduleName)) {
                        it.remove();
                    }
                }
            }
        }
    }

    /**
     * This is the function for which we are most optimized!
     *
     * @param qualifier the tokens returned have to start with the given qualifier
     * @return a list of info, all starting with the given qualifier
     */
    public Collection<IInfo> getTokensStartingWith(String qualifier, int getWhat) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, startingWithFilter, true, null);
        }
    }

    public Collection<IInfo> getTokensStartingWith(String qualifier, int getWhat, Collection<IInfo> result) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, startingWithFilter, true, result);
        }
    }

    public Collection<IInfo> getTokensEqualTo(String qualifier, int getWhat) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, equalsFilter, false, null);
        }
    }

    public Collection<IInfo> getTokensEqualTo(String qualifier, int getWhat, Collection<IInfo> result) {
        synchronized (lock) {
            return getWithFilter(qualifier, getWhat, equalsFilter, false, result);
        }
    }

    protected Collection<IInfo> getWithFilter(String qualifier, int getWhat, Filter filter, boolean useLowerCaseQual,
            Collection<IInfo> result) {
        synchronized (lock) {
            if (result == null) {
                result = new ArrayList<IInfo>();
            }

            if ((getWhat & TOP_LEVEL) != 0) {
                getWithFilter(qualifier, topLevelInitialsToInfo, result, filter, useLowerCaseQual);
            }
            if ((getWhat & INNER) != 0) {
                getWithFilter(qualifier, innerInitialsToInfo, result, filter, useLowerCaseQual);
            }
            return result;
        }
    }

    /**
     * @param qualifier
     * @param initialsToInfo this is where we are going to get the info from (currently: inner or top level list)
     * @param toks (out) the tokens will be added to this list
     * @return
     */
    protected void getWithFilter(String qualifier, SortedMap<String, Set<IInfo>> initialsToInfo,
            Collection<IInfo> toks, Filter filter, boolean useLowerCaseQual) {
        String initials = getInitials(qualifier);
        String qualToCompare = qualifier;
        if (useLowerCaseQual) {
            qualToCompare = qualifier.toLowerCase();
        }

        //get until the end of the alphabet
        SortedMap<String, Set<IInfo>> subMap = initialsToInfo.subMap(initials, initials + "\uffff\uffff\uffff\uffff");

        for (Set<IInfo> listForInitials : subMap.values()) {

            for (IInfo info : listForInitials) {
                if (filter.doCompare(qualToCompare, info)) {
                    toks.add(info);
                }
            }
        }
    }

    /**
     * @return all the tokens that are in this info (top level or inner)
     */
    public Collection<IInfo> getAllTokens() {
        synchronized (lock) {
            Collection<Set<IInfo>> lInfo = this.topLevelInitialsToInfo.values();

            ArrayList<IInfo> toks = new ArrayList<IInfo>();
            for (Set<IInfo> list : lInfo) {
                for (IInfo info : list) {
                    toks.add(info);
                }
            }

            lInfo = this.innerInitialsToInfo.values();
            for (Set<IInfo> list : lInfo) {
                for (IInfo info : list) {
                    toks.add(info);
                }
            }
            return toks;
        }
    }

    /**
     * this can be used to save the file
     */
    public void save() {
        File persistingLocation;
        try {
            persistingLocation = getPersistingLocation();
        } catch (MisconfigurationException e) {
            Log.log("Error. Unable to get persisting location for additional interprer info. Configuration may be corrupted.",
                    e);
            return;
        }
        if (DEBUG_ADDITIONAL_INFO) {
            System.out.println("Saving to " + persistingLocation);
        }

        save(persistingLocation);

    }

    protected void save(File persistingLocation) {
        try {
            FileOutputStream stream = new FileOutputStream(persistingLocation);
            OutputStreamWriter writer = new OutputStreamWriter(stream);
            try {
                FastStringBuffer tempBuf = new FastStringBuffer();
                tempBuf.append("-- VERSION_");
                tempBuf.append(AbstractAdditionalTokensInfo.version);
                tempBuf.append('\n');
                writer.write(tempBuf.getInternalCharsArray(), 0, tempBuf.length());
                tempBuf.clear();

                saveTo(writer, tempBuf, persistingLocation);
            } finally {
                try {
                    writer.close();
                } finally {
                    stream.close();
                }
            }
        } catch (Exception e) {
            Log.log(e);
        }
    }

    /**
     * @return the location where we can persist this info.
     * @throws MisconfigurationException
     */
    protected abstract File getPersistingLocation() throws MisconfigurationException;

    /**
     * @return the path to the folder we want to keep things on
     * @throws MisconfigurationException
     */
    protected abstract File getPersistingFolder();

    protected void saveTo(OutputStreamWriter writer, FastStringBuffer tempBuf, File pathToSave) throws IOException {
        synchronized (lock) {
            if (DEBUG_ADDITIONAL_INFO) {
                System.out.println("Saving info " + this.getClass().getName() + " to file (size = "
                        + getAllTokens().size() + ") " + pathToSave);
            }

            Map<String, Integer> dictionary = new HashMap<String, Integer>();
            tempBuf.append("-- START TREE 1\n");
            TreeIO.dumpTreeToBuffer(this.topLevelInitialsToInfo, tempBuf, dictionary);

            tempBuf.append("-- START TREE 2\n");
            TreeIO.dumpTreeToBuffer(this.innerInitialsToInfo, tempBuf, dictionary);

            FastStringBuffer buf2 = new FastStringBuffer(50 * (dictionary.size() + 4));
            TreeIO.dumpDictToBuffer(dictionary, buf2);

            //Write the dictionary before the actual trees.
            writer.write(buf2.getInternalCharsArray(), 0, buf2.length());
            buf2 = null;

            //Note: tried LZFFileInputStream from https://github.com/ning/compress
            //and Snappy from https://github.com/dain/snappy checking to see if by writing less we'd
            //get a better time but it got a bit slower (gzip was slowest, then snappy and the faster was LZFFileInputStream)
            writer.write(tempBuf.getInternalCharsArray(), 0, tempBuf.length());
        }
    }

    /**
     * Restores the saved info in the object (if overridden, getInfoToSave should be overridden too)
     * @param o the read object from the file
     * @throws MisconfigurationException
     */
    @SuppressWarnings("unchecked")
    protected void restoreSavedInfo(Object o) throws MisconfigurationException {
        synchronized (lock) {
            Tuple3<Object, Object, Object> readFromFile = (Tuple3<Object, Object, Object>) o;
            SortedMap<String, Set<IInfo>> o1 = (SortedMap<String, Set<IInfo>>) readFromFile.o1;
            SortedMap<String, Set<IInfo>> o2 = (SortedMap<String, Set<IInfo>>) readFromFile.o2;

            if (o1 == null) {
                throw new RuntimeException("Error in I/O (topLevelInitialsToInfo is null). Rebuilding internal info.");
            }
            if (o2 == null) {
                throw new RuntimeException("Error in I/O (innerInitialsToInfo is null). Rebuilding internal info.");
            }
            this.topLevelInitialsToInfo = o1;
            this.innerInitialsToInfo = o2;
            if (readFromFile.o3 != null) {
                //may be null in new format (where that's checked during load time).
                if (AbstractAdditionalTokensInfo.version != (Integer) readFromFile.o3) {
                    throw new RuntimeException("I/O version doesn't match. Rebuilding internal info.");
                }
            }
        }
    }

    @Override
    public String toString() {
        synchronized (lock) {
            FastStringBuffer buffer = new FastStringBuffer();
            buffer.append("AdditionalInfo{");

            buffer.append("topLevel=[");
            entrySetToString(buffer, this.topLevelInitialsToInfo.entrySet());
            buffer.append("]\n");
            buffer.append("inner=[");
            entrySetToString(buffer, this.innerInitialsToInfo.entrySet());
            buffer.append("]");

            buffer.append("}");
            return buffer.toString();
        }
    }

    /**
     * @param buffer
     * @param name
     */
    private void entrySetToString(FastStringBuffer buffer, Set<Entry<String, Set<IInfo>>> name) {
        synchronized (lock) {
            for (Entry<String, Set<IInfo>> entry : name) {
                Set<IInfo> value = entry.getValue();
                for (IInfo info : value) {
                    buffer.append(info.toString());
                    buffer.append("\n");
                }
            }
        }
    }

}

class IOUtils {

    public static Object readFromFile(File astOutputFile) {
        try {
            InputStream in = new BufferedInputStream(new FileInputStream(astOutputFile));
            try {
                ObjectInputStream stream = new ObjectInputStream(in);
                try {
                    Object o = stream.readObject();
                    return o;
                } finally {
                    stream.close();
                }
            } finally {
                in.close();
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

}
{'fraction_non_alphanumeric': 0.08847681726550456, 'fraction_numerical': 0.0023864579627653265, 'mean_word_length': 1.7057431619475416, 'pattern_counts': {'":': 0, '<': 191, '<?xml version=': 0, '>': 205, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 184, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4269958', 'n_tokens_mistral': 17321, 'n_tokens_neox': 16615, 'n_words': 4860}