<commit_before>package org.flymine.dataloader;

/*
 * Copyright (C) 2002-2003 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.lang.reflect.Field;
import java.util.Collection;
import java.util.Iterator;

import org.acedb.Ace;
import org.acedb.AceException;
import org.acedb.AceURL;
import org.acedb.AceSet;
import org.acedb.AceNode;
import org.acedb.AceObject;
import org.acedb.DateValue;
import org.acedb.StringValue;
import org.acedb.IntValue;
import org.acedb.FloatValue;
import org.acedb.Reference;
import org.acedb.staticobj.StaticAceObject;

import org.flymine.FlyMineException;
import org.flymine.util.StringUtil;
import org.flymine.util.TypeUtil;
import org.flymine.metadata.Model;
import org.flymine.metadata.ClassDescriptor;
import org.flymine.metadata.FieldDescriptor;
import org.flymine.metadata.CollectionDescriptor;
import org.flymine.modelproduction.acedb.AceModelParser;

import org.apache.log4j.Logger;

/**
 * DataLoader for AceDB data
 * @author <NAME>
 */
public class AceDataLoader extends DataLoader
{
    protected static final Logger LOG = Logger.getLogger(AceDataLoader.class);

    protected static final String DATETYPE = "DateType";
    protected static final String FLOAT = "Float";
    protected static final String INT = "Int";
    protected static final String TEXT = "Text";

    protected Model model;

    protected String pkgName; //as ace namespace is flat, assume all classes in same package
    protected String dateType, floatType, intType, textType;

    /**
     * Constructor for testing purposes
     * @param model a Model independent of an IntegrationWriter
     */
    protected AceDataLoader(Model model) {
        this.model = model;
        initialise();
    }

    /**
     * @see DataLoader#DataLoader(IntegrationWriter)
     */
    public AceDataLoader(IntegrationWriter iw) {
        super(iw);
        model = iw.getObjectStore().getModel();
        initialise();
    }

    /**
     * Initialise the built-in types
     */
    protected void initialise() {
        pkgName = TypeUtil.packageName(((String) model.getClassNames().iterator().next())) + ".";
        dateType = pkgName + DATETYPE;
        floatType = pkgName + FLOAT;
        intType = pkgName + INT;
        textType = pkgName + TEXT;
    }

    /**
     * Static method to unmarshall business objects from a given Ace server and call
     * store on each.
     *
     * @param source access to AceDb
     * @throws FlyMineException if anything goes wrong with ace or storing
     */
    public void processAce(AceURL source) throws FlyMineException {
        try {
            Ace.registerDriver(new org.acedb.socket.SocketDriver());

            // Go through each class in the model and get a dump of the objects of
            // that class

            Iterator clsIter = model.getClassNames().iterator();
            while (clsIter.hasNext()) {
                String clsName = (String) clsIter.next();
                if (true) {
                    String aceClazzName = AceModelParser
                        .unformatAceName(TypeUtil.unqualifiedName(clsName));
                    AceURL objURL = source.relativeURL(aceClazzName);
                    AceSet fetchedAceObjects = (AceSet) Ace.fetch(objURL);
                    if (fetchedAceObjects != null) {
                        LOG.info("Processing " + fetchedAceObjects.size() + " "
                                  + aceClazzName + " objects...");
                        Iterator aceObjIter = fetchedAceObjects.iterator();
                        while (aceObjIter.hasNext()) {
                            try {
                                AceObject aceObj = (AceObject) aceObjIter.next();
                                store(processAceObject(aceObj));
                            } catch (Exception e) {
                                LOG.error("Object not retrievable: " + e.getMessage());
                            }
                        }
                    } else {
                        LOG.info("No " + aceClazzName + " objects found");
                    }
                } else {
                    LOG.info("Not bothering with class " + clsName);
                }
            }
        } catch (Exception e) {
            throw new FlyMineException(e);
        }
    }
    
    /**
     * Process an AceObject. This will create a new instance of the
     * object and set the identifier.
     *
     * @param aceObject the AceObject to process
     * @return an instance of the object
     *
     * @throws AceException if an error occurs with the Ace data
     * @throws FlyMineException if object cannot be instantiated
     */
    protected Object processAceObject(AceObject aceObject)
        throws AceException, FlyMineException {
        if (aceObject == null) {
            throw new NullPointerException("aceObject must not be null");
        }
        LOG.info("Processing " + aceObject.getClassName() + " " + aceObject.getName());

        String clsName = pkgName
            + AceModelParser.formatAceName(((AceObject) aceObject).getClassName());
        Object currentObject = instantiate(clsName);
        Object identifier = aceObject.getName();
//         if ("".equals(identifier)) {
//             identifier = null;
//         }
        setField(currentObject, "identifier", identifier);

        processAceNode(aceObject, currentObject);
        return currentObject;
    }

    /**
     * Process an AceNode. This will set field values in the given
     * object if the node is a data node.
     *
     * @param aceNode the AceNode to process
     * @param currentObject the object in which to set field
     *
     * @throws AceException if an error occurs with the Ace data
     * @throws FlyMineException if object cannot be instantiated
     */
    protected void processAceNode(AceNode aceNode, Object currentObject)
        throws AceException, FlyMineException {
        String nodeName; //name of the field in currentObject
        Object nodeValue; //identifier of the referred to object
        if (aceNode instanceof Reference) {
            nodeName = getName(aceNode);
            nodeValue = aceNode.getName();
            // nodeClass is the class of the referred to object, and is part of the target AceURL
            String nodeClass = ((Reference) aceNode).getTarget().getPath();
            nodeClass = nodeClass.substring(0, nodeClass.indexOf("/", 1));
            AceObject referredToAceObject = new StaticAceObject((String) nodeValue,
                                                                null, nodeClass);
            Object referredToObject = processAceObject(referredToAceObject);
            setField(currentObject, nodeName, referredToObject);
        } else if (aceNode instanceof DateValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(dateType);
                TypeUtil.setFieldValue(nodeValue, "identifier", ((DateValue) aceNode).toDate());
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof FloatValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(floatType);
                TypeUtil.setFieldValue(nodeValue, "identifier",
                                       new Float(((FloatValue) aceNode).toFloat()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof IntValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(intType);
                TypeUtil.setFieldValue(nodeValue, "identifier",
                                       new Integer(((IntValue) aceNode) .toInt()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof StringValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(textType);
                TypeUtil.setFieldValue(nodeValue, "identifier", StringUtil
                                       .duplicateQuotes(((StringValue) aceNode).toString()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (!(aceNode instanceof AceObject)) { //node representing a field
            nodeName = aceNode.getName();
            //check whether currentObject has a field of this name (if not, it's a nesting tag)
            Field nodeField = TypeUtil.getField(currentObject.getClass(), nodeName);
            if (nodeField != null) {
                 if (!aceNode.iterator().hasNext()) {
                    if (nodeField.getType().equals(Boolean.TYPE)) {
                        // there's no boolean value type - if the tag's present, the value is true
                        nodeValue = Boolean.TRUE;
                    } else {
                        // no children at all - a non-boolean tag without value. not handled (yet).
                        nodeValue = null;
                    }
                    setField(currentObject, nodeName, nodeValue);
                } else if (!hasChildValues(aceNode)) {
                    // node has child values, so it's a hash. we create a new object in its place
                    String nodeClass = null;
                    ClassDescriptor cld = model.getClassDescriptorByName(currentObject
                                                                         .getClass().getName());
                    FieldDescriptor fd = cld.getFieldDescriptorByName(nodeName);
                    // the node could be a collection or reference
                    if (fd instanceof CollectionDescriptor) {
                        // find out the type of the elements
                        CollectionDescriptor fdCld = (CollectionDescriptor) fd;
                        ClassDescriptor referencedCld = fdCld.getReferencedClassDescriptor();
                        nodeClass = TypeUtil.unqualifiedName(referencedCld.getClassName());
                    } else {
                        nodeClass = TypeUtil.unqualifiedName(nodeField.getType().getName());
                    }
                    StaticAceObject referredToAceObject =
                        new StaticAceObject(StringUtil.uniqueString(), null, nodeClass);
                    // add all of the child nodes to our new AceObject
                    Iterator nodesIter = aceNode.iterator();
                    while (nodesIter.hasNext()) {
                        referredToAceObject.addNode((AceNode) nodesIter.next());
                    }
                    nodeValue = processAceObject(referredToAceObject);
                    setField(currentObject, nodeName, nodeValue);
                }
            } else {
                LOG.warn(currentObject.getClass() + " has no field named " + nodeName);
            }
        }
        // Now iterate through all the child nodes
        Iterator objIter = aceNode.iterator();
        while (objIter.hasNext()) {
            processAceNode((AceNode) objIter.next(), currentObject);
        }
    }

    /**
     * Sets a field in a target object, or adds the piece of data to a collection
     *
     * @param target the object in which to set the field
     * @param fieldName the name of the field to set
     * @param fieldValue the value to set or to be added to a collection
     * @throws FlyMineException if the field cannot be accessed
     */
    protected void setField(Object target, String fieldName, Object fieldValue)
        throws FlyMineException {
        try {
            if ((fieldValue instanceof String)) {  // single quotes need to duplicated for DB
                fieldValue = StringUtil.duplicateQuotes((String) fieldValue);
            }
            Field field = TypeUtil.getField(target.getClass(), fieldName);
            if (field != null) {
                if (Collection.class.isAssignableFrom(field.getType()) && fieldValue != null) {
                    ((Collection) TypeUtil.getFieldValue(target, fieldName)).add(fieldValue);
                } else {
                    try {
                        if (fieldValue != null && isBuiltIn(fieldValue.getClass())) {
                            fieldValue = TypeUtil.getFieldValue(fieldValue, "identifier");
                        }
                    } catch (Exception e) {
                        throw new FlyMineException(e);
                    }
                    TypeUtil.setFieldValue(target, fieldName, fieldValue);
                }
            } else {
                // else the field cannot be found -- do nothing
                if (!"Quoted_in".equals(fieldName)) {
                    LOG.error("Field \"" + fieldName + "\" not found in object of type \""
                              + target.getClass().getName() + "\" - would have set to \"" 
                              + fieldValue + "\"");
                }
            }
        } catch (IllegalAccessException e) {
            throw new FlyMineException(e);
        }
    }

    /**
     * Get the name of the parent of an AceNode, with suffix if a multi-valued tag
     *
     * @param node the node
     * @return the name of the parent of the node, or the parent's name if this node is a data node
     * @throws AceException if error occurs with the Ace data
     */
    protected String getName(AceSet node) throws AceException {
        String name = node.getParent().getName();
        int count = 1;
        while (((node = node.getParent()) != null)
               && (node instanceof DateValue
                   || node instanceof FloatValue
                   || node instanceof IntValue
                   || node instanceof StringValue
                   || node instanceof Reference)) {
            count++;
        }
        if (count > 1) {
            name = node.getName() + "_" + count;
        }
        return name;
    }

    /**
     * Returns true if the given node has values as children
     *
     * @param node the node to test
     * @return true if the node has values as children
     * @throws AceException if error occurs with the Ace data
     */
    protected boolean hasChildValues(AceNode node) throws AceException {
        Iterator childIter = node.iterator();
        while (childIter.hasNext()) {
            AceNode childNode = (AceNode) childIter.next();
            if (childNode instanceof DateValue
                || childNode instanceof FloatValue
                || childNode instanceof IntValue
                || childNode instanceof StringValue
                || childNode instanceof Reference) {
                return true;
            }
        }
        return false;
    }

    private Object instantiate(String clsName) {
        Object obj = null;
        try {
            obj = Class.forName(clsName).newInstance();
        } catch (Exception e) {
            LOG.error("Cannot instantiate '" + clsName + "':" + e);
        }
        return obj;
    }

    private boolean isBuiltIn(Class c) {
        String clsName = c.getName();
        if (dateType.equals(clsName)
            || floatType.equals(clsName)
            || intType.equals(clsName)
            || textType.equals(clsName)) {
            return true;
        }
        return false;
    }

    /**
     * Used for testing access to an AceDB server
     *
     * @param args command line arguments
     * @throws Exception if an error occurs
     */
    /*
    public static void main(String [] args) throws Exception {
        if (args.length != 5) {
            throw new RuntimeException("AceDataLoader hostname port username password classname");
        }

        String host = args[0];
        String port = args[1];
        String user = args[2];
        String passwd = args[3];
        String clsName = args[4];

        AceURL dbURL = new AceURL("acedb://" + user + ':' + passwd + '@' + host + ':' + port);

        Ace.registerDriver(new org.acedb.socket.SocketDriver());
        AceURL objURL = dbURL.relativeURL(clsName);
        AceSet fetchedAceObjects = (AceSet) Ace.fetch(objURL);

        Iterator iter = fetchedAceObjects.iterator();
        while (iter.hasNext()) {
            AceNode node = (AceNode) iter.next();
            System.
            out.println(node);
        }
    }
    */
}<commit_msg>Multiple acedb getter threads speed up data loading<commit_after>package org.flymine.dataloader;

/*
 * Copyright (C) 2002-2003 FlyMine
 *
 * This code may be freely distributed and modified under the
 * terms of the GNU Lesser General Public Licence.  This should
 * be distributed with the code.  See the LICENSE file for more
 * information or http://www.gnu.org/copyleft/lesser.html.
 *
 */

import java.lang.reflect.Field;
import java.util.Collection;
import java.util.Iterator;

import org.acedb.Ace;
import org.acedb.AceException;
import org.acedb.AceURL;
import org.acedb.AceSet;
import org.acedb.AceNode;
import org.acedb.AceObject;
import org.acedb.DateValue;
import org.acedb.StringValue;
import org.acedb.IntValue;
import org.acedb.FloatValue;
import org.acedb.Reference;
import org.acedb.staticobj.StaticAceObject;

import org.flymine.FlyMineException;
import org.flymine.util.StringUtil;
import org.flymine.util.TypeUtil;
import org.flymine.metadata.Model;
import org.flymine.metadata.ClassDescriptor;
import org.flymine.metadata.FieldDescriptor;
import org.flymine.metadata.CollectionDescriptor;
import org.flymine.modelproduction.acedb.AceModelParser;

import org.apache.log4j.Logger;

/**
 * DataLoader for AceDB data
 * @author <NAME>
 */
public class AceDataLoader extends DataLoader
{
    protected static final Logger LOG = Logger.getLogger(AceDataLoader.class);

    protected static final String DATETYPE = "DateType";
    protected static final String FLOAT = "Float";
    protected static final String INT = "Int";
    protected static final String TEXT = "Text";

    protected Model model;

    protected String pkgName; //as ace namespace is flat, assume all classes in same package
    protected String dateType, floatType, intType, textType;

    /**
     * Constructor for testing purposes
     * @param model a Model independent of an IntegrationWriter
     */
    protected AceDataLoader(Model model) {
        this.model = model;
        initialise();
    }

    /**
     * @see DataLoader#DataLoader(IntegrationWriter)
     */
    public AceDataLoader(IntegrationWriter iw) {
        super(iw);
        model = iw.getObjectStore().getModel();
        initialise();
    }

    /**
     * Initialise the built-in types
     */
    protected void initialise() {
        pkgName = TypeUtil.packageName(((String) model.getClassNames().iterator().next())) + ".";
        dateType = pkgName + DATETYPE;
        floatType = pkgName + FLOAT;
        intType = pkgName + INT;
        textType = pkgName + TEXT;
    }

    /**
     * Static method to unmarshall business objects from a given Ace server and call
     * store on each.
     *
     * @param source access to AceDb
     * @throws FlyMineException if anything goes wrong with ace or storing
     */
    public void processAce(AceURL source) throws FlyMineException {
        try {
            Ace.registerDriver(new org.acedb.socket.SocketDriver());

            // Go through each class in the model and get a dump of the objects of
            // that class

            Iterator clsIter = model.getClassNames().iterator();
            WorkSource workSource = new WorkSource(clsIter, source);
            for (int i = 0; i < 30 ; i++) {
                (new ServiceThread(workSource, this)).start();
            }
            WorkItem todo = workSource.next();
            while (todo != null) {
                try {
                    AceObject aceObj = (AceObject) todo.getFetched().retrieve(todo.getName());
                    synchronized(this) {
                        store(processAceObject(aceObj));
                    }
                } catch (Exception e) {
                    LOG.error("Object not retrievable: " + e.getMessage());
                }
                todo = workSource.next();
            }
        } catch (Exception e) {
            throw new FlyMineException(e);
        }
    }
    
    /**
     * Process an AceObject. This will create a new instance of the
     * object and set the identifier.
     *
     * @param aceObject the AceObject to process
     * @return an instance of the object
     *
     * @throws AceException if an error occurs with the Ace data
     * @throws FlyMineException if object cannot be instantiated
     */
    protected Object processAceObject(AceObject aceObject)
        throws AceException, FlyMineException {
        if (aceObject == null) {
            throw new NullPointerException("aceObject must not be null");
        }
        LOG.info("Processing " + aceObject.getClassName() + " " + aceObject.getName());

        String clsName = pkgName
            + AceModelParser.formatAceName(((AceObject) aceObject).getClassName());
        Object currentObject = instantiate(clsName);
        Object identifier = aceObject.getName();
//         if ("".equals(identifier)) {
//             identifier = null;
//         }
        setField(currentObject, "identifier", identifier);

        processAceNode(aceObject, currentObject);
        return currentObject;
    }

    /**
     * Process an AceNode. This will set field values in the given
     * object if the node is a data node.
     *
     * @param aceNode the AceNode to process
     * @param currentObject the object in which to set field
     *
     * @throws AceException if an error occurs with the Ace data
     * @throws FlyMineException if object cannot be instantiated
     */
    protected void processAceNode(AceNode aceNode, Object currentObject)
        throws AceException, FlyMineException {
        String nodeName; //name of the field in currentObject
        Object nodeValue; //identifier of the referred to object
        if (aceNode instanceof Reference) {
            nodeName = getName(aceNode);
            nodeValue = aceNode.getName();
            // nodeClass is the class of the referred to object, and is part of the target AceURL
            String nodeClass = ((Reference) aceNode).getTarget().getPath();
            nodeClass = nodeClass.substring(0, nodeClass.indexOf("/", 1));
            AceObject referredToAceObject = new StaticAceObject((String) nodeValue,
                                                                null, nodeClass);
            Object referredToObject = processAceObject(referredToAceObject);
            setField(currentObject, nodeName, referredToObject);
        } else if (aceNode instanceof DateValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(dateType);
                TypeUtil.setFieldValue(nodeValue, "identifier", ((DateValue) aceNode).toDate());
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof FloatValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(floatType);
                TypeUtil.setFieldValue(nodeValue, "identifier",
                                       new Float(((FloatValue) aceNode).toFloat()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof IntValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(intType);
                TypeUtil.setFieldValue(nodeValue, "identifier",
                                       new Integer(((IntValue) aceNode) .toInt()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (aceNode instanceof StringValue) {
            try {
                nodeName = getName(aceNode);
                nodeValue = instantiate(textType);
                TypeUtil.setFieldValue(nodeValue, "identifier", StringUtil
                                       .duplicateQuotes(((StringValue) aceNode).toString()));
                setField(currentObject, nodeName, nodeValue);
            } catch (Exception e) {
                throw new FlyMineException(e);
            }
        } else if (!(aceNode instanceof AceObject)) { //node representing a field
            nodeName = aceNode.getName();
            //check whether currentObject has a field of this name (if not, it's a nesting tag)
            Field nodeField = TypeUtil.getField(currentObject.getClass(), nodeName);
            if (nodeField != null) {
                 if (!aceNode.iterator().hasNext()) {
                    if (nodeField.getType().equals(Boolean.TYPE)) {
                        // there's no boolean value type - if the tag's present, the value is true
                        nodeValue = Boolean.TRUE;
                    } else {
                        // no children at all - a non-boolean tag without value. not handled (yet).
                        nodeValue = null;
                    }
                    setField(currentObject, nodeName, nodeValue);
                } else if (!hasChildValues(aceNode)) {
                    // node has child values, so it's a hash. we create a new object in its place
                    String nodeClass = null;
                    ClassDescriptor cld = model.getClassDescriptorByName(currentObject
                                                                         .getClass().getName());
                    FieldDescriptor fd = cld.getFieldDescriptorByName(nodeName);
                    // the node could be a collection or reference
                    if (fd instanceof CollectionDescriptor) {
                        // find out the type of the elements
                        CollectionDescriptor fdCld = (CollectionDescriptor) fd;
                        ClassDescriptor referencedCld = fdCld.getReferencedClassDescriptor();
                        nodeClass = TypeUtil.unqualifiedName(referencedCld.getClassName());
                    } else {
                        nodeClass = TypeUtil.unqualifiedName(nodeField.getType().getName());
                    }
                    StaticAceObject referredToAceObject =
                        new StaticAceObject(StringUtil.uniqueString(), null, nodeClass);
                    // add all of the child nodes to our new AceObject
                    Iterator nodesIter = aceNode.iterator();
                    while (nodesIter.hasNext()) {
                        referredToAceObject.addNode((AceNode) nodesIter.next());
                    }
                    nodeValue = processAceObject(referredToAceObject);
                    setField(currentObject, nodeName, nodeValue);
                }
            } else {
                LOG.warn(currentObject.getClass() + " has no field named " + nodeName);
            }
        }
        // Now iterate through all the child nodes
        Iterator objIter = aceNode.iterator();
        while (objIter.hasNext()) {
            processAceNode((AceNode) objIter.next(), currentObject);
        }
    }

    /**
     * Sets a field in a target object, or adds the piece of data to a collection
     *
     * @param target the object in which to set the field
     * @param fieldName the name of the field to set
     * @param fieldValue the value to set or to be added to a collection
     * @throws FlyMineException if the field cannot be accessed
     */
    protected void setField(Object target, String fieldName, Object fieldValue)
        throws FlyMineException {
        try {
            if ((fieldValue instanceof String)) {  // single quotes need to duplicated for DB
                fieldValue = StringUtil.duplicateQuotes((String) fieldValue);
            }
            Field field = TypeUtil.getField(target.getClass(), fieldName);
            if (field != null) {
                if (Collection.class.isAssignableFrom(field.getType()) && fieldValue != null) {
                    ((Collection) TypeUtil.getFieldValue(target, fieldName)).add(fieldValue);
                } else {
                    try {
                        if (fieldValue != null && isBuiltIn(fieldValue.getClass())) {
                            fieldValue = TypeUtil.getFieldValue(fieldValue, "identifier");
                        }
                    } catch (Exception e) {
                        throw new FlyMineException(e);
                    }
                    TypeUtil.setFieldValue(target, fieldName, fieldValue);
                }
            } else {
                // else the field cannot be found -- do nothing
                if (!"Quoted_in".equals(fieldName)) {
                    LOG.warn("Field \"" + fieldName + "\" not found in object of type \""
                              + target.getClass().getName() + "\" - would have set to \"" 
                              + fieldValue + "\"");
                }
            }
        } catch (IllegalAccessException e) {
            throw new FlyMineException(e);
        }
    }

    /**
     * Get the name of the parent of an AceNode, with suffix if a multi-valued tag
     *
     * @param node the node
     * @return the name of the parent of the node, or the parent's name if this node is a data node
     * @throws AceException if error occurs with the Ace data
     */
    protected String getName(AceSet node) throws AceException {
        String name = node.getParent().getName();
        int count = 1;
        while (((node = node.getParent()) != null)
               && (node instanceof DateValue
                   || node instanceof FloatValue
                   || node instanceof IntValue
                   || node instanceof StringValue
                   || node instanceof Reference)) {
            count++;
        }
        if (count > 1) {
            name = node.getName() + "_" + count;
        }
        return name;
    }

    /**
     * Returns true if the given node has values as children
     *
     * @param node the node to test
     * @return true if the node has values as children
     * @throws AceException if error occurs with the Ace data
     */
    protected boolean hasChildValues(AceNode node) throws AceException {
        Iterator childIter = node.iterator();
        while (childIter.hasNext()) {
            AceNode childNode = (AceNode) childIter.next();
            if (childNode instanceof DateValue
                || childNode instanceof FloatValue
                || childNode instanceof IntValue
                || childNode instanceof StringValue
                || childNode instanceof Reference) {
                return true;
            }
        }
        return false;
    }

    private Object instantiate(String clsName) {
        Object obj = null;
        try {
            obj = Class.forName(clsName).newInstance();
        } catch (Exception e) {
            LOG.error("Cannot instantiate '" + clsName + "':" + e);
        }
        return obj;
    }

    private boolean isBuiltIn(Class c) {
        String clsName = c.getName();
        if (dateType.equals(clsName)
            || floatType.equals(clsName)
            || intType.equals(clsName)
            || textType.equals(clsName)) {
            return true;
        }
        return false;
    }

    /**
     * Used for testing access to an AceDB server
     *
     * @param args command line arguments
     * @throws Exception if an error occurs
     */
    /*
    public static void main(String [] args) throws Exception {
        if (args.length != 5) {
            throw new RuntimeException("AceDataLoader hostname port username password classname");
        }

        String host = args[0];
        String port = args[1];
        String user = args[2];
        String passwd = args[3];
        String clsName = args[4];

        AceURL dbURL = new AceURL("acedb://" + user + ':' + passwd + '@' + host + ':' + port);

        Ace.registerDriver(new org.acedb.socket.SocketDriver());
        AceURL objURL = dbURL.relativeURL(clsName);
        AceSet fetchedAceObjects = (AceSet) Ace.fetch(objURL);

        Iterator iter = fetchedAceObjects.iterator();
        while (iter.hasNext()) {
            AceNode node = (AceNode) iter.next();
            System.
            out.println(node);
        }
    }
    */

    private static class WorkSource
    {
        private Iterator classIter;
        private AceSet fetchedAceObjects;
        private Iterator nameIter;
        private AceURL source;

        public WorkSource(Iterator classIter, AceURL source) {
            this.classIter = classIter;
            this.fetchedAceObjects = null;
            this.nameIter = null;
            this.source = source;
        }

        public synchronized WorkItem next() throws AceException {
            while (((nameIter == null) || (!nameIter.hasNext())) && classIter.hasNext()) {
                String className = (String) classIter.next();
                String aceClazzName = AceModelParser
                    .unformatAceName(TypeUtil.unqualifiedName(className));
                AceURL objURL = source.relativeURL(aceClazzName);
                fetchedAceObjects = (AceSet) Ace.fetch(objURL);
                if (fetchedAceObjects != null) {
                    LOG.error("Processing " + fetchedAceObjects.size() + " "
                              + aceClazzName + " objects...");
                    nameIter = fetchedAceObjects.nameIterator();
                }
            }
            if ((nameIter == null) || (!nameIter.hasNext())) {
                return null;
            }
            return new WorkItem((String) nameIter.next(), fetchedAceObjects);
        }
    }

    private static class WorkItem
    {
        private String name;
        private AceSet fetchedAceObjects;

        public WorkItem(String name, AceSet fetchedAceObjects) {
            this.name = name;
            this.fetchedAceObjects = fetchedAceObjects;
        }

        public String getName() {
            return name;
        }

        public AceSet getFetched() {
            return fetchedAceObjects;
        }
    }

    private static class ServiceThread extends Thread
    {
        private WorkSource workSource;
        private AceDataLoader loader;

        public ServiceThread(WorkSource workSource, AceDataLoader loader) {
            super();
            this.workSource = workSource;
            this.loader = loader;
        }
        
        public void run() {
            try {
                WorkItem todo = workSource.next();
                while (todo != null) {
                    try {
                        AceObject aceObj = (AceObject) todo.getFetched().retrieve(todo.getName());
                        synchronized(loader) {
                            loader.store(loader.processAceObject(aceObj));
                        }
                    } catch (Exception e) {
                        LOG.error("Object not retrievable: " + e.getMessage());
                    }
                    todo = workSource.next();
                }
            } catch (Exception e) {
                LOG.error("Ace data loader sub-thread died because of " + e);
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.08918132705360854, 'fraction_numerical': 0.0011358913977005125, 'mean_word_length': 1.7541584007324889, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '677061', 'n_tokens_mistral': 9199, 'n_tokens_neox': 8780, 'n_words': 2603}