<commit_before>/*
 * The Gemma project.
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.model.common.auditAndSecurity;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.Hibernate;
import org.hibernate.Query;
import org.hibernate.SessionFactory;
import org.hibernate.persister.entity.SingleTableEntityPersister;
import org.hibernate.proxy.HibernateProxy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.orm.hibernate3.HibernateTemplate;
import org.springframework.stereotype.Repository;

import ubic.gemma.model.common.Auditable;
import ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType;
import ubic.gemma.model.common.auditAndSecurity.eventType.OKStatusFlagEvent;
import ubic.gemma.model.common.auditAndSecurity.eventType.TroubleStatusFlagEvent;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.util.CommonQueries;
import ubic.gemma.util.EntityUtils;

/**
 * @see ubic.gemma.model.common.auditAndSecurity.AuditEvent
 * @author pavlidis
 * @version $Id$
 */
@Repository
public class AuditEventDaoImpl extends AuditEventDaoBase {

    private static Log log = LogFactory.getLog( AuditEventDaoImpl.class.getName() );

    /**
     * Classes that we track for 'updated since'. This is used for "What's new" functionality.
     */
    private static String[] AUDITABLES_TO_TRACK_FOR_WHATSNEW = {
            // "ubic.gemma.model.expression.analysis.ExpressionAnalysisImpl",
            "ubic.gemma.model.expression.arrayDesign.ArrayDesignImpl",
            // "ubic.gemma.model.common.description.BibliographicReferenceImpl",
            // "ubic.gemma.model.common.auditAndSecurity.ContactImpl",
            "ubic.gemma.model.expression.experiment.ExpressionExperimentImpl" };

    @Autowired
    public AuditEventDaoImpl( SessionFactory sessionFactory ) {
        super.setSessionFactory( sessionFactory );
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastEvents(java.util.Collection,
     * java.util.Collection)
     */
    @Override
    public Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> getLastEvents(
            Collection<? extends Auditable> auditables, Collection<Class<? extends AuditEventType>> types ) {
        StopWatch timer = new StopWatch();
        timer.start();

        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> results = new HashMap<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>>();
        if ( auditables.size() == 0 ) return results;

        for ( Class<? extends AuditEventType> t : types ) {
            results.put( t, new HashMap<Auditable, AuditEvent>() );
        }

        final Map<AuditTrail, Auditable> atmap = getAuditTrailMap( auditables );

        List<String> classes = getClassHierarchy( types );

        final String queryString = "select et, trail, event from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join trail.events event inner join event.eventType et inner join fetch event.performer where trail in (:trails) "
                + "and et.class in (" + StringUtils.join( classes, "," ) + ") order by event.date,event.id desc ";

        Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
        queryObject.setParameterList( "trails", atmap.keySet() );

        List<?> qr = queryObject.list();

        for ( Object o : qr ) {
            Object[] ar = ( Object[] ) o;
            AuditEventType ty = ( AuditEventType ) ar[0];
            AuditTrail t = ( AuditTrail ) ar[1];
            AuditEvent e = ( AuditEvent ) ar[2];

            /*
             * This is a bit inefficient. Loop needed because returned type is Impl (and probably a proxy). But probably
             * query is the bottleneck.
             */
            for ( Class<? extends AuditEventType> ti : types ) {
                if ( ti.isAssignableFrom( ty.getClass() ) ) {
                    // FIXME we need to distinguish subclasses of validation events, for example.
                    Map<Auditable, AuditEvent> innerMap = results.get( ti );

                    assert innerMap != null;

                    // only one event per type
                    if ( !innerMap.containsKey( atmap.get( t ) ) ) {
                        innerMap.put( atmap.get( t ), e );
                    }
                    break;
                }
            }
        }

        timer.stop();
        if ( timer.getTime() > 1000 ) {
            log.info( "Last events retrieved for  " + types.size() + " different types for " + auditables.size()
                    + " items in " + timer.getTime() + "ms" );
        }

        return results;
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastOutstandingTroubleEvent(java.util.Collection)
     */
    @Override
    public AuditEvent getLastOutstandingTroubleEvent( Collection<AuditEvent> events ) {
        return getLastOutstandingTroubleEventNoSort( events );
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastOutstandingTroubleEvents(java.util.Collection)
     */
    @Override
    public Map<Auditable, AuditEvent> getLastOutstandingTroubleEvents( Collection<? extends Auditable> auditables ) {
        Collection<Class<? extends AuditEventType>> types = new HashSet<Class<? extends AuditEventType>>();
        types.add( TroubleStatusFlagEvent.class );
        types.add( OKStatusFlagEvent.class );
        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> lastEvents = this.getLastEvents( auditables,
                types );

        Map<Auditable, AuditEvent> results = new HashMap<Auditable, AuditEvent>();

        if ( !lastEvents.containsKey( TroubleStatusFlagEvent.class ) ) {
            return results;
        }

        /*
         * Common case: check if auditables are EEs.
         */

        Collection<ExpressionExperiment> ees = new HashSet<ExpressionExperiment>();

        for ( Auditable a : auditables ) {

            Map<Auditable, AuditEvent> trouble = lastEvents.get( TroubleStatusFlagEvent.class );

            if ( !trouble.containsKey( a ) ) {
                // no trouble! but we should check the array designs later.
                if ( a instanceof ExpressionExperiment ) {
                    ees.add( ( ExpressionExperiment ) a );
                }
                continue;
            }

            AuditEvent t = trouble.get( a );

            Map<Auditable, AuditEvent> ok = lastEvents.get( OKStatusFlagEvent.class );

            if ( ok.containsKey( a ) ) {
                // do we have a more recent ok event?
                AuditEvent o = ok.get( a );
                if ( o.getDate().after( t.getDate() ) ) {
                    continue;
                }

                results.put( a, t );

            } else {
                results.put( a, t );
            }

        }

        if ( !ees.isEmpty() ) {
            Map<Long, ExpressionExperiment> eemap = EntityUtils.getIdMap( ees );
            Map<ArrayDesign, Collection<Long>> ads = CommonQueries.getArrayDesignsUsed( eemap.keySet(), this
                    .getSessionFactory().getCurrentSession() );

            Map<Auditable, AuditEvent> arrayDesignTrouble = getLastOutstandingTroubleEvents( ads.keySet() );

            for ( Entry<Auditable, AuditEvent> e : arrayDesignTrouble.entrySet() ) {
                for ( Long ee : ads.get( e.getKey() ) ) {
                    results.put( eemap.get( ee ), e.getValue() );
                }
            }
            results.putAll( arrayDesignTrouble );

        }

        return results;

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#hasEvent(ubic.gemma.model.common.Auditable,
     * java.lang.Class)
     */
    @Override
    public boolean hasEvent( Auditable a, Class<? extends AuditEventType> type ) {
        return this.getLastEvent( a, type ) != null;
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#retainHavingEvent(java.util.Collection,
     * java.lang.Class)
     */
    @Override
    public void retainHavingEvent( final Collection<? extends Auditable> a, final Class<? extends AuditEventType> type ) {

        final Map<Auditable, AuditEvent> events = this.getLastEvent( a, type );

        CollectionUtils.filter( a, new Predicate() {
            @Override
            public boolean evaluate( Object arg0 ) {
                return events.containsKey( arg0 );
            }
        } );

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#retainLackingEvent(java.util.Collection,
     * java.lang.Class)
     */
    @Override
    public void retainLackingEvent( final Collection<? extends Auditable> a, final Class<? extends AuditEventType> type ) {
        StopWatch timer = new StopWatch();
        timer.start();
        final Map<Auditable, AuditEvent> events = this.getLastEvent( a, type );
        log.info( "Phase I: " + timer.getTime() + "ms" );

        CollectionUtils.filter( a, new Predicate() {
            @Override
            public boolean evaluate( Object arg0 ) {
                return !events.containsKey( arg0 );
            }
        } );

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.model.common.auditAndSecurity.AuditEventDaoBase#handleGetEvents(ubic.gemma.model.common.Auditable)
     */
    @Override
    protected List<AuditEvent> handleGetEvents( final Auditable auditable ) {
        if ( auditable == null ) throw new IllegalArgumentException( "Auditable cannot be null" );

        if ( auditable.getAuditTrail() == null ) {
            throw new IllegalStateException( "Auditable did not have an audit trail: " + auditable );
        }

        Long id = auditable.getAuditTrail().getId();
        return this.getHibernateTemplate().findByNamedParam(
                "select e from AuditTrailImpl t join t.events e where t.id = :id order by e.date,e.id ", "id", id );

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.model.common.auditAndSecurity.AuditEventDaoBase#handleGetLastEvent(ubic.gemma.model.common.Auditable,
     * java.lang.Class)
     */
    @Override
    protected AuditEvent handleGetLastEvent( Auditable auditable, Class<? extends AuditEventType> type ) {
        return this.handleGetLastEvent( auditable.getAuditTrail(), type );
    }

    /**
     * @param auditTrail
     * @param type
     * @return
     * @throws java.lang.Exception
     */
    protected ubic.gemma.model.common.auditAndSecurity.AuditEvent handleGetLastEvent( final AuditTrail auditTrail,
            Class<? extends AuditEventType> type ) {

        /*
         * For the = operator to work in hibernate the class or class name can't be passed in as a parameter :type -
         * also queryObject.setParameter("type", type.getClass()); doesn't work. Although technically this is now
         * vulnerable to an sql injection attack, it seems moot as an attacker would have to have access to the JVM to
         * inject a malformed AuditEventType class name and if they had access to the JVM then sql injection is the
         * least of our worries. The real annoyance here is dealing with subclasses of event types.
         */

        List<String> classes = getClassHierarchy( type );

        if ( classes.size() == 0 ) {
            return null;
        }

        final String queryString = "select event from AuditTrailImpl trail "
                + "inner join trail.events event inner join event.eventType et inner join fetch event.performer "
                + "fetch all properties where trail = :trail " + "and et.class in (" + StringUtils.join( classes, "," )
                + ") order by event.date,event.id desc ";

        org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
        queryObject.setCacheable( true );
        queryObject.setReadOnly( true );
        queryObject.setParameter( "trail", auditTrail );
        queryObject.setMaxResults( 1 );

        Collection<AuditEvent> results = queryObject.list();

        if ( results == null || results.isEmpty() ) return null;

        AuditEvent result = results.iterator().next();
        Hibernate.initialize( result.getPerformer() ); // Hit performer to make hibernate initialize it.
        return result;

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.AuditableDaoBase#handleGetLastAuditEvent(java.util.Collection,
     * ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType)
     */
    @Override
    protected Map<Auditable, AuditEvent> handleGetLastEvent( final Collection<? extends Auditable> auditables,
            Class<? extends AuditEventType> type ) {

        Map<Auditable, AuditEvent> result = new HashMap<Auditable, AuditEvent>();
        if ( auditables.size() == 0 ) return result;

        final Map<AuditTrail, Auditable> atmap = getAuditTrailMap( auditables );

        List<String> classes = getClassHierarchy( type );

        final String queryString = "select trail, ae from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join trail.events ae inner join ae.eventType et inner join fetch ae.performer where trail in (:trails) "
                + "and et.class in (" + StringUtils.join( classes, "," ) + ") order by ae.date,ae.id desc ";

        StopWatch timer = new StopWatch();
        timer.start();

        Collection<AuditTrail> batch = new ArrayList<AuditTrail>();
        int BATCHSIZE = 100;

        for ( AuditTrail at : atmap.keySet() ) {
            batch.add( at );

            if ( batch.size() == BATCHSIZE ) {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameterList( "trails", batch );
                queryObject.setReadOnly( true );

                List<?> qr = queryObject.list();
                if ( qr == null || qr.isEmpty() ) {
                    batch.clear();
                    continue;
                }

                for ( Object o : qr ) {
                    Object[] ar = ( Object[] ) o;
                    AuditTrail t = ( AuditTrail ) ar[0];
                    AuditEvent e = ( AuditEvent ) ar[1];

                    // only one event per object, please - the most recent.
                    if ( result.containsKey( atmap.get( t ) ) ) continue;

                    result.put( atmap.get( t ), e );
                }
                batch.clear();
            }
        }

        if ( !batch.isEmpty() ) {
            org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
            queryObject.setParameterList( "trails", batch ); // if too many will fail.
            queryObject.setReadOnly( true );

            List<?> qr = queryObject.list();
            if ( qr == null || qr.isEmpty() ) return result;

            for ( Object o : qr ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[0];
                AuditEvent e = ( AuditEvent ) ar[1];

                // only one event per object, please - the most recent.
                if ( result.containsKey( atmap.get( t ) ) ) continue;

                result.put( atmap.get( t ), e );
            }
        }

        timer.stop();
        if ( timer.getTime() > 500 ) {
            log.info( "Last event of type " + type.getSimpleName() + " retrieved for " + auditables.size()
                    + " items in " + timer.getTime() + "ms" );
        }

        return result;
    }

    @SuppressWarnings("unchecked")
    @Override
    protected Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> handleGetLastTypedAuditEvents(
            Collection<? extends Auditable> auditables ) {

        Map<AuditTrail, Auditable> atmap = new HashMap<AuditTrail, Auditable>();
        for ( Auditable a : ( Collection<Auditable> ) auditables ) {
            atmap.put( a.getAuditTrail(), a );
        }

        final String queryString = "select trail,event,et from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join fetch trail.events event inner join event.eventType et inner join fetch event.performer "
                + "where trail in (:trails) order by event.date,event.id desc ";

        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> result = new HashMap<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>>();
        try {
            org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
            queryObject.setCacheable( true );
            queryObject.setParameter( "trails", atmap.keySet() );

            List<?> qr = queryObject.list();
            if ( qr == null || qr.isEmpty() ) return result;

            for ( Object o : qr ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[0];
                AuditEvent e = ( AuditEvent ) ar[1];
                AuditEventType ty = ( AuditEventType ) ar[2];

                /*
                 * Careful with subclasses. The key in the hashtable should really only be the superclass.
                 */
                if ( !result.containsKey( ty.getClass() ) ) {
                    result.put( ty.getClass(), new HashMap<Auditable, AuditEvent>() );
                }
                Map<Auditable, AuditEvent> amap = result.get( ty.getClass() );

                // only one event per object, please - the most recent.
                if ( amap.containsKey( atmap.get( t ) ) ) continue;

                amap.put( atmap.get( t ), e );
            }

        } catch ( org.hibernate.HibernateException ex ) {
            throw super.convertHibernateAccessException( ex );
        }

        return result;
    }

    /**
     * Note that this only returns selected classes of auditables.
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getNewSinceDate(java.util.Date)
     * @return Collection of Auditables
     */
    @Override
    protected java.util.Collection<Auditable> handleGetNewSinceDate( java.util.Date date ) {
        Collection<Auditable> result = new HashSet<Auditable>();
        for ( String clazz : AUDITABLES_TO_TRACK_FOR_WHATSNEW ) {
            String queryString = "select distinct adb from "
                    + clazz
                    + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='C'";
            try {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameter( "date", date );
                result.addAll( queryObject.list() );
            } catch ( org.hibernate.HibernateException ex ) {
                throw super.convertHibernateAccessException( ex );
            }
        }
        return result;
    }

    /**
     * Note that this only returns selected classes of auditables.
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getUpdatedSinceDate(java.util.Date)
     * @return Collection of Auditables
     */
    @Override
    protected java.util.Collection<Auditable> handleGetUpdatedSinceDate( java.util.Date date ) {
        Collection<Auditable> result = new HashSet<Auditable>();
        for ( String clazz : AUDITABLES_TO_TRACK_FOR_WHATSNEW ) {
            String queryString = "select distinct adb from "
                    + clazz
                    + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='U'";
            try {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameter( "date", date );
                result.addAll( queryObject.list() );
            } catch ( org.hibernate.HibernateException ex ) {
                throw super.convertHibernateAccessException( ex );
            }
        }
        return result;
    }

    @Deprecated
    @Override
    protected void handleThaw( AuditEvent auditEvent ) throws Exception {
        if ( auditEvent == null ) return;

        auditEvent = ( AuditEvent ) this
                .getHibernateTemplate()
                .findByNamedParam(
                        "select a from AuditEventImpl a fetch all properties join fetch a.performer fetch all properties where a = :ae ",
                        "ae", auditEvent ).iterator().next();

    }

    /**
     * Essential thaw the auditables to the point we get the AuditTrail proxies for them.
     * 
     * @param auditables
     * @return
     */
    @SuppressWarnings("unchecked")
    private Map<AuditTrail, Auditable> getAuditTrailMap( final Collection<? extends Auditable> auditables ) {

        /*
         * This is the fastest way I've found to thaw the audit trails of a whole bunch of auditables. Because Auditable
         * is not mapped, we have to query for each class separately ... just in case the user has passed a
         * heterogeneous collection.
         */
        final Map<AuditTrail, Auditable> atmap = new HashMap<AuditTrail, Auditable>();
        Map<String, Collection<Auditable>> clazzmap = new HashMap<String, Collection<Auditable>>();
        for ( Auditable a : ( Collection<Auditable> ) auditables ) {
            Class<? extends Auditable> clazz = a.getClass();

            /*
             * proxy?
             */
            String clazzName = clazz.getName();
            if ( a instanceof HibernateProxy ) {
                clazzName = ( ( HibernateProxy ) a ).getHibernateLazyInitializer().getEntityName();
            }

            if ( !clazzmap.containsKey( clazzName ) ) {
                clazzmap.put( clazzName, new HashSet<Auditable>() );
            }
            clazzmap.get( clazzName ).add( a );
        }

        StopWatch timer = new StopWatch();
        timer.start();

        HibernateTemplate template = new HibernateTemplate( this.getSessionFactory() );
        template.setCacheQueries( true );
        template.setQueryCacheRegion( "org.hibernate.cache.StandardQueryCache" );

        for ( String clazz : clazzmap.keySet() ) {
            final String trailQuery = "select a, a.auditTrail from " + clazz + " a where a in (:auditables) ";
            List<?> res = template.findByNamedParam( trailQuery, "auditables", clazzmap.get( clazz ) );
            for ( Object o : res ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[1];
                Auditable a = ( Auditable ) ar[0];
                atmap.put( t, a );
            }

            timer.stop();
            if ( timer.getTime() > 1000 ) {
                log.info( "Audit trails retrieved for " + auditables.size() + " " + clazz + " items in "
                        + timer.getTime() + "ms" );
            }
            timer.reset();
            timer.start();

        }
        return atmap;
    }

    /**
     * Determine the full set of AuditEventTypes that are needed (that is, subclasses of the given class)
     * 
     * @param type Class
     * @return A List of class names, including the given type.
     */
    private List<String> getClassHierarchy( Class<? extends AuditEventType> type ) {
        List<String> classes = new ArrayList<String>();
        classes.add( getImplClass( type ) );

        // how to determine subclasses? There is no way to do this but the hibernate way.
        SingleTableEntityPersister classMetadata = ( SingleTableEntityPersister ) this.getSessionFactory()
                .getClassMetadata( getImplClass( type ) );
        if ( classMetadata == null ) return classes;

        if ( classMetadata.hasSubclasses() ) {
            String[] subclasses = classMetadata.getSubclassClosure(); // this includes the superclass, fully qualified
            // names.
            classes.clear();
            for ( String string : subclasses ) {
                // strip qualification to leave BlabablImpl
                string = string.replaceFirst( ".+\\.", "" );
                classes.add( string );
            }
        }
        return classes;
    }

    /**
     * Determine the full set of AuditEventTypes that are needed (that is, subclasses of the given classes)
     * 
     * @param types
     * @return
     */
    private List<String> getClassHierarchy( Collection<Class<? extends AuditEventType>> types ) {
        List<String> classes = new ArrayList<String>();
        for ( Class<? extends AuditEventType> t : types ) {
            classes.addAll( getClassHierarchy( t ) );
        }
        return classes;
    }

    /**
     * @param type
     * @return
     */
    private String getImplClass( Class<? extends AuditEventType> type ) {
        String canonicalName = type.getName();
        return canonicalName.endsWith( "Impl" ) ? type.getName() : type.getName() + "Impl";
    }

    /**
     * @param events
     * @return
     */
    private AuditEvent getLastOutstandingTroubleEventNoSort( Collection<AuditEvent> events ) {
        AuditEvent lastTroubleEvent = null;
        AuditEvent lastOKEvent = null;
        for ( AuditEvent event : events ) {
            if ( event.getEventType() == null ) {
                continue;
            } else if ( OKStatusFlagEvent.class.isAssignableFrom( event.getEventType().getClass() ) ) {
                if ( lastOKEvent == null || lastOKEvent.getDate().before( event.getDate() ) ) lastOKEvent = event;
            } else if ( TroubleStatusFlagEvent.class.isAssignableFrom( event.getEventType().getClass() ) ) {
                if ( lastTroubleEvent == null || lastTroubleEvent.getDate().before( event.getDate() ) )
                    lastTroubleEvent = event;
            }
        }
        if ( lastTroubleEvent != null )
            if ( lastOKEvent == null || lastOKEvent.getDate().before( lastTroubleEvent.getDate() ) )
                return lastTroubleEvent;
        return null;
    }

}<commit_msg>Fixes bug 4448 - date columns in platforms not current <commit_after>/*
 * The Gemma project.
 * 
 * Copyright (c) 2006 University of British Columbia
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */
package ubic.gemma.model.common.auditAndSecurity;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.StopWatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.Hibernate;
import org.hibernate.Query;
import org.hibernate.SessionFactory;
import org.hibernate.persister.entity.SingleTableEntityPersister;
import org.hibernate.proxy.HibernateProxy;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.orm.hibernate3.HibernateTemplate;
import org.springframework.stereotype.Repository;

import ubic.gemma.model.common.Auditable;
import ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType;
import ubic.gemma.model.common.auditAndSecurity.eventType.OKStatusFlagEvent;
import ubic.gemma.model.common.auditAndSecurity.eventType.TroubleStatusFlagEvent;
import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
import ubic.gemma.model.expression.experiment.ExpressionExperiment;
import ubic.gemma.util.CommonQueries;
import ubic.gemma.util.EntityUtils;

/**
 * @see ubic.gemma.model.common.auditAndSecurity.AuditEvent
 * @author pavlidis
 * @version $Id$
 */
@Repository
public class AuditEventDaoImpl extends AuditEventDaoBase {

    private static Log log = LogFactory.getLog( AuditEventDaoImpl.class.getName() );

    /**
     * Classes that we track for 'updated since'. This is used for "What's new" functionality.
     */
    private static String[] AUDITABLES_TO_TRACK_FOR_WHATSNEW = {
            // "ubic.gemma.model.expression.analysis.ExpressionAnalysisImpl",
            "ubic.gemma.model.expression.arrayDesign.ArrayDesignImpl",
            // "ubic.gemma.model.common.description.BibliographicReferenceImpl",
            // "ubic.gemma.model.common.auditAndSecurity.ContactImpl",
            "ubic.gemma.model.expression.experiment.ExpressionExperimentImpl" };

    @Autowired
    public AuditEventDaoImpl( SessionFactory sessionFactory ) {
        super.setSessionFactory( sessionFactory );
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastEvents(java.util.Collection,
     * java.util.Collection)
     */
    @Override
    public Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> getLastEvents(
            Collection<? extends Auditable> auditables, Collection<Class<? extends AuditEventType>> types ) {
        StopWatch timer = new StopWatch();
        timer.start();

        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> results = new HashMap<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>>();
        if ( auditables.size() == 0 ) return results;

        for ( Class<? extends AuditEventType> t : types ) {
            results.put( t, new HashMap<Auditable, AuditEvent>() );
        }

        final Map<AuditTrail, Auditable> atmap = getAuditTrailMap( auditables );

        List<String> classes = getClassHierarchy( types );

        final String queryString = "select et, trail, event from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join trail.events event inner join event.eventType et inner join fetch event.performer where trail in (:trails) "
                + "and et.class in (" + StringUtils.join( classes, "," ) + ") order by event.date,event.id desc ";

        Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
        queryObject.setParameterList( "trails", atmap.keySet() );

        List<?> qr = queryObject.list();

        for ( Object o : qr ) {
            Object[] ar = ( Object[] ) o;
            AuditEventType ty = ( AuditEventType ) ar[0];
            AuditTrail t = ( AuditTrail ) ar[1];
            AuditEvent e = ( AuditEvent ) ar[2];

            /*
             * This is a bit inefficient. Loop needed because returned type is Impl (and probably a proxy). But probably
             * query is the bottleneck.
             */
            for ( Class<? extends AuditEventType> ti : types ) {
                if ( ti.isAssignableFrom( ty.getClass() ) ) {
                    // FIXME we need to distinguish subclasses of validation events, for example.
                    Map<Auditable, AuditEvent> innerMap = results.get( ti );

                    assert innerMap != null;

                    // only replace event if its date is more recent.
                    Auditable ae = atmap.get( t );
                    if ( !innerMap.containsKey( ae ) || innerMap.get( ae ).getDate().compareTo( e.getDate() ) < 0 ) {
                        innerMap.put( atmap.get( t ), e );
                    }
                    break;
                }
            }
        }

        timer.stop();
        if ( timer.getTime() > 1000 ) {
            log.info( "Last events retrieved for  " + types.size() + " different types for " + auditables.size()
                    + " items in " + timer.getTime() + "ms" );
        }

        return results;
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastOutstandingTroubleEvent(java.util.Collection)
     */
    @Override
    public AuditEvent getLastOutstandingTroubleEvent( Collection<AuditEvent> events ) {
        return getLastOutstandingTroubleEventNoSort( events );
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getLastOutstandingTroubleEvents(java.util.Collection)
     */
    @Override
    public Map<Auditable, AuditEvent> getLastOutstandingTroubleEvents( Collection<? extends Auditable> auditables ) {
        Collection<Class<? extends AuditEventType>> types = new HashSet<Class<? extends AuditEventType>>();
        types.add( TroubleStatusFlagEvent.class );
        types.add( OKStatusFlagEvent.class );
        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> lastEvents = this.getLastEvents( auditables,
                types );

        Map<Auditable, AuditEvent> results = new HashMap<Auditable, AuditEvent>();

        if ( !lastEvents.containsKey( TroubleStatusFlagEvent.class ) ) {
            return results;
        }

        /*
         * Common case: check if auditables are EEs.
         */

        Collection<ExpressionExperiment> ees = new HashSet<ExpressionExperiment>();

        for ( Auditable a : auditables ) {

            Map<Auditable, AuditEvent> trouble = lastEvents.get( TroubleStatusFlagEvent.class );

            if ( !trouble.containsKey( a ) ) {
                // no trouble! but we should check the array designs later.
                if ( a instanceof ExpressionExperiment ) {
                    ees.add( ( ExpressionExperiment ) a );
                }
                continue;
            }

            AuditEvent t = trouble.get( a );

            Map<Auditable, AuditEvent> ok = lastEvents.get( OKStatusFlagEvent.class );

            if ( ok.containsKey( a ) ) {
                // do we have a more recent ok event?
                AuditEvent o = ok.get( a );
                if ( o.getDate().after( t.getDate() ) ) {
                    continue;
                }

                results.put( a, t );

            } else {
                results.put( a, t );
            }

        }

        if ( !ees.isEmpty() ) {
            Map<Long, ExpressionExperiment> eemap = EntityUtils.getIdMap( ees );
            Map<ArrayDesign, Collection<Long>> ads = CommonQueries.getArrayDesignsUsed( eemap.keySet(), this
                    .getSessionFactory().getCurrentSession() );

            Map<Auditable, AuditEvent> arrayDesignTrouble = getLastOutstandingTroubleEvents( ads.keySet() );

            for ( Entry<Auditable, AuditEvent> e : arrayDesignTrouble.entrySet() ) {
                for ( Long ee : ads.get( e.getKey() ) ) {
                    results.put( eemap.get( ee ), e.getValue() );
                }
            }
            results.putAll( arrayDesignTrouble );

        }

        return results;

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#hasEvent(ubic.gemma.model.common.Auditable,
     * java.lang.Class)
     */
    @Override
    public boolean hasEvent( Auditable a, Class<? extends AuditEventType> type ) {
        return this.getLastEvent( a, type ) != null;
    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#retainHavingEvent(java.util.Collection,
     * java.lang.Class)
     */
    @Override
    public void retainHavingEvent( final Collection<? extends Auditable> a, final Class<? extends AuditEventType> type ) {

        final Map<Auditable, AuditEvent> events = this.getLastEvent( a, type );

        CollectionUtils.filter( a, new Predicate() {
            @Override
            public boolean evaluate( Object arg0 ) {
                return events.containsKey( arg0 );
            }
        } );

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#retainLackingEvent(java.util.Collection,
     * java.lang.Class)
     */
    @Override
    public void retainLackingEvent( final Collection<? extends Auditable> a, final Class<? extends AuditEventType> type ) {
        StopWatch timer = new StopWatch();
        timer.start();
        final Map<Auditable, AuditEvent> events = this.getLastEvent( a, type );
        log.info( "Phase I: " + timer.getTime() + "ms" );

        CollectionUtils.filter( a, new Predicate() {
            @Override
            public boolean evaluate( Object arg0 ) {
                return !events.containsKey( arg0 );
            }
        } );

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.model.common.auditAndSecurity.AuditEventDaoBase#handleGetEvents(ubic.gemma.model.common.Auditable)
     */
    @Override
    protected List<AuditEvent> handleGetEvents( final Auditable auditable ) {
        if ( auditable == null ) throw new IllegalArgumentException( "Auditable cannot be null" );

        if ( auditable.getAuditTrail() == null ) {
            throw new IllegalStateException( "Auditable did not have an audit trail: " + auditable );
        }

        Long id = auditable.getAuditTrail().getId();
        return this.getHibernateTemplate().findByNamedParam(
                "select e from AuditTrailImpl t join t.events e where t.id = :id order by e.date,e.id ", "id", id );

    }

    /*
     * (non-Javadoc)
     * 
     * @see
     * ubic.gemma.model.common.auditAndSecurity.AuditEventDaoBase#handleGetLastEvent(ubic.gemma.model.common.Auditable,
     * java.lang.Class)
     */
    @Override
    protected AuditEvent handleGetLastEvent( Auditable auditable, Class<? extends AuditEventType> type ) {
        return this.handleGetLastEvent( auditable.getAuditTrail(), type );
    }

    /**
     * @param auditTrail
     * @param type
     * @return
     * @throws java.lang.Exception
     */
    protected ubic.gemma.model.common.auditAndSecurity.AuditEvent handleGetLastEvent( final AuditTrail auditTrail,
            Class<? extends AuditEventType> type ) {

        /*
         * For the = operator to work in hibernate the class or class name can't be passed in as a parameter :type -
         * also queryObject.setParameter("type", type.getClass()); doesn't work. Although technically this is now
         * vulnerable to an sql injection attack, it seems moot as an attacker would have to have access to the JVM to
         * inject a malformed AuditEventType class name and if they had access to the JVM then sql injection is the
         * least of our worries. The real annoyance here is dealing with subclasses of event types.
         */

        List<String> classes = getClassHierarchy( type );

        if ( classes.size() == 0 ) {
            return null;
        }

        final String queryString = "select event from AuditTrailImpl trail "
                + "inner join trail.events event inner join event.eventType et inner join fetch event.performer "
                + "fetch all properties where trail = :trail " + "and et.class in (" + StringUtils.join( classes, "," )
                + ") order by event.date,event.id desc ";

        org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
        queryObject.setCacheable( true );
        queryObject.setReadOnly( true );
        queryObject.setParameter( "trail", auditTrail );
        queryObject.setMaxResults( 1 );

        Collection<AuditEvent> results = queryObject.list();

        if ( results == null || results.isEmpty() ) return null;

        AuditEvent result = results.iterator().next();
        Hibernate.initialize( result.getPerformer() ); // Hit performer to make hibernate initialize it.
        return result;

    }

    /*
     * (non-Javadoc)
     * 
     * @see ubic.gemma.model.common.AuditableDaoBase#handleGetLastAuditEvent(java.util.Collection,
     * ubic.gemma.model.common.auditAndSecurity.eventType.AuditEventType)
     */
    @Override
    protected Map<Auditable, AuditEvent> handleGetLastEvent( final Collection<? extends Auditable> auditables,
            Class<? extends AuditEventType> type ) {

        Map<Auditable, AuditEvent> result = new HashMap<Auditable, AuditEvent>();
        if ( auditables.size() == 0 ) return result;

        final Map<AuditTrail, Auditable> atmap = getAuditTrailMap( auditables );

        List<String> classes = getClassHierarchy( type );

        final String queryString = "select trail, ae from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join trail.events ae inner join ae.eventType et inner join fetch ae.performer where trail in (:trails) "
                + "and et.class in (" + StringUtils.join( classes, "," ) + ") order by ae.date,ae.id desc ";

        StopWatch timer = new StopWatch();
        timer.start();

        Collection<AuditTrail> batch = new ArrayList<AuditTrail>();
        int BATCHSIZE = 100;

        for ( AuditTrail at : atmap.keySet() ) {
            batch.add( at );

            if ( batch.size() == BATCHSIZE ) {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameterList( "trails", batch );
                queryObject.setReadOnly( true );

                List<?> qr = queryObject.list();
                if ( qr == null || qr.isEmpty() ) {
                    batch.clear();
                    continue;
                }

                for ( Object o : qr ) {
                    Object[] ar = ( Object[] ) o;
                    AuditTrail t = ( AuditTrail ) ar[0];
                    AuditEvent e = ( AuditEvent ) ar[1];

                    // only one event per object, please - the most recent.
                    if ( result.containsKey( atmap.get( t ) ) ) continue;

                    result.put( atmap.get( t ), e );
                }
                batch.clear();
            }
        }

        if ( !batch.isEmpty() ) {
            org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
            queryObject.setParameterList( "trails", batch ); // if too many will fail.
            queryObject.setReadOnly( true );

            List<?> qr = queryObject.list();
            if ( qr == null || qr.isEmpty() ) return result;

            for ( Object o : qr ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[0];
                AuditEvent e = ( AuditEvent ) ar[1];

                // only one event per object, please - the most recent.
                if ( result.containsKey( atmap.get( t ) ) ) continue;

                result.put( atmap.get( t ), e );
            }
        }

        timer.stop();
        if ( timer.getTime() > 500 ) {
            log.info( "Last event of type " + type.getSimpleName() + " retrieved for " + auditables.size()
                    + " items in " + timer.getTime() + "ms" );
        }

        return result;
    }

    @SuppressWarnings("unchecked")
    @Override
    protected Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> handleGetLastTypedAuditEvents(
            Collection<? extends Auditable> auditables ) {

        Map<AuditTrail, Auditable> atmap = new HashMap<AuditTrail, Auditable>();
        for ( Auditable a : ( Collection<Auditable> ) auditables ) {
            atmap.put( a.getAuditTrail(), a );
        }

        final String queryString = "select trail,event,et from ubic.gemma.model.common.auditAndSecurity.AuditTrail trail "
                + "inner join fetch trail.events event inner join event.eventType et inner join fetch event.performer "
                + "where trail in (:trails) order by event.date,event.id desc ";

        Map<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>> result = new HashMap<Class<? extends AuditEventType>, Map<Auditable, AuditEvent>>();
        try {
            org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession().createQuery( queryString );
            queryObject.setCacheable( true );
            queryObject.setParameter( "trails", atmap.keySet() );

            List<?> qr = queryObject.list();
            if ( qr == null || qr.isEmpty() ) return result;

            for ( Object o : qr ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[0];
                AuditEvent e = ( AuditEvent ) ar[1];
                AuditEventType ty = ( AuditEventType ) ar[2];

                /*
                 * Careful with subclasses. The key in the hashtable should really only be the superclass.
                 */
                if ( !result.containsKey( ty.getClass() ) ) {
                    result.put( ty.getClass(), new HashMap<Auditable, AuditEvent>() );
                }
                Map<Auditable, AuditEvent> amap = result.get( ty.getClass() );

                // only one event per object, please - the most recent.
                if ( amap.containsKey( atmap.get( t ) ) ) continue;

                amap.put( atmap.get( t ), e );
            }

        } catch ( org.hibernate.HibernateException ex ) {
            throw super.convertHibernateAccessException( ex );
        }

        return result;
    }

    /**
     * Note that this only returns selected classes of auditables.
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getNewSinceDate(java.util.Date)
     * @return Collection of Auditables
     */
    @Override
    protected java.util.Collection<Auditable> handleGetNewSinceDate( java.util.Date date ) {
        Collection<Auditable> result = new HashSet<Auditable>();
        for ( String clazz : AUDITABLES_TO_TRACK_FOR_WHATSNEW ) {
            String queryString = "select distinct adb from "
                    + clazz
                    + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='C'";
            try {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameter( "date", date );
                result.addAll( queryObject.list() );
            } catch ( org.hibernate.HibernateException ex ) {
                throw super.convertHibernateAccessException( ex );
            }
        }
        return result;
    }

    /**
     * Note that this only returns selected classes of auditables.
     * 
     * @see ubic.gemma.model.common.auditAndSecurity.AuditEventDao#getUpdatedSinceDate(java.util.Date)
     * @return Collection of Auditables
     */
    @Override
    protected java.util.Collection<Auditable> handleGetUpdatedSinceDate( java.util.Date date ) {
        Collection<Auditable> result = new HashSet<Auditable>();
        for ( String clazz : AUDITABLES_TO_TRACK_FOR_WHATSNEW ) {
            String queryString = "select distinct adb from "
                    + clazz
                    + " adb inner join adb.auditTrail atr inner join atr.events as ae where ae.date > :date and ae.action='U'";
            try {
                org.hibernate.Query queryObject = super.getSessionFactory().getCurrentSession()
                        .createQuery( queryString );
                queryObject.setParameter( "date", date );
                result.addAll( queryObject.list() );
            } catch ( org.hibernate.HibernateException ex ) {
                throw super.convertHibernateAccessException( ex );
            }
        }
        return result;
    }

    @Deprecated
    @Override
    protected void handleThaw( AuditEvent auditEvent ) throws Exception {
        if ( auditEvent == null ) return;

        auditEvent = ( AuditEvent ) this
                .getHibernateTemplate()
                .findByNamedParam(
                        "select a from AuditEventImpl a fetch all properties join fetch a.performer fetch all properties where a = :ae ",
                        "ae", auditEvent ).iterator().next();

    }

    /**
     * Essential thaw the auditables to the point we get the AuditTrail proxies for them.
     * 
     * @param auditables
     * @return
     */
    @SuppressWarnings("unchecked")
    private Map<AuditTrail, Auditable> getAuditTrailMap( final Collection<? extends Auditable> auditables ) {

        /*
         * This is the fastest way I've found to thaw the audit trails of a whole bunch of auditables. Because Auditable
         * is not mapped, we have to query for each class separately ... just in case the user has passed a
         * heterogeneous collection.
         */
        final Map<AuditTrail, Auditable> atmap = new HashMap<AuditTrail, Auditable>();
        Map<String, Collection<Auditable>> clazzmap = new HashMap<String, Collection<Auditable>>();
        for ( Auditable a : ( Collection<Auditable> ) auditables ) {
            Class<? extends Auditable> clazz = a.getClass();

            /*
             * proxy?
             */
            String clazzName = clazz.getName();
            if ( a instanceof HibernateProxy ) {
                clazzName = ( ( HibernateProxy ) a ).getHibernateLazyInitializer().getEntityName();
            }

            if ( !clazzmap.containsKey( clazzName ) ) {
                clazzmap.put( clazzName, new HashSet<Auditable>() );
            }
            clazzmap.get( clazzName ).add( a );
        }

        StopWatch timer = new StopWatch();
        timer.start();

        HibernateTemplate template = new HibernateTemplate( this.getSessionFactory() );
        template.setCacheQueries( true );
        template.setQueryCacheRegion( "org.hibernate.cache.StandardQueryCache" );

        for ( String clazz : clazzmap.keySet() ) {
            final String trailQuery = "select a, a.auditTrail from " + clazz + " a where a in (:auditables) ";
            List<?> res = template.findByNamedParam( trailQuery, "auditables", clazzmap.get( clazz ) );
            for ( Object o : res ) {
                Object[] ar = ( Object[] ) o;
                AuditTrail t = ( AuditTrail ) ar[1];
                Auditable a = ( Auditable ) ar[0];
                atmap.put( t, a );
            }

            timer.stop();
            if ( timer.getTime() > 1000 ) {
                log.info( "Audit trails retrieved for " + auditables.size() + " " + clazz + " items in "
                        + timer.getTime() + "ms" );
            }
            timer.reset();
            timer.start();

        }
        return atmap;
    }

    /**
     * Determine the full set of AuditEventTypes that are needed (that is, subclasses of the given class)
     * 
     * @param type Class
     * @return A List of class names, including the given type.
     */
    private List<String> getClassHierarchy( Class<? extends AuditEventType> type ) {
        List<String> classes = new ArrayList<String>();
        classes.add( getImplClass( type ) );

        // how to determine subclasses? There is no way to do this but the hibernate way.
        SingleTableEntityPersister classMetadata = ( SingleTableEntityPersister ) this.getSessionFactory()
                .getClassMetadata( getImplClass( type ) );
        if ( classMetadata == null ) return classes;

        if ( classMetadata.hasSubclasses() ) {
            String[] subclasses = classMetadata.getSubclassClosure(); // this includes the superclass, fully qualified
            // names.
            classes.clear();
            for ( String string : subclasses ) {
                // strip qualification to leave BlabablImpl
                string = string.replaceFirst( ".+\\.", "" );
                classes.add( string );
            }
        }
        return classes;
    }

    /**
     * Determine the full set of AuditEventTypes that are needed (that is, subclasses of the given classes)
     * 
     * @param types
     * @return
     */
    private List<String> getClassHierarchy( Collection<Class<? extends AuditEventType>> types ) {
        List<String> classes = new ArrayList<String>();
        for ( Class<? extends AuditEventType> t : types ) {
            classes.addAll( getClassHierarchy( t ) );
        }
        return classes;
    }

    /**
     * @param type
     * @return
     */
    private String getImplClass( Class<? extends AuditEventType> type ) {
        String canonicalName = type.getName();
        return canonicalName.endsWith( "Impl" ) ? type.getName() : type.getName() + "Impl";
    }

    /**
     * @param events
     * @return
     */
    private AuditEvent getLastOutstandingTroubleEventNoSort( Collection<AuditEvent> events ) {
        AuditEvent lastTroubleEvent = null;
        AuditEvent lastOKEvent = null;
        for ( AuditEvent event : events ) {
            if ( event.getEventType() == null ) {
                continue;
            } else if ( OKStatusFlagEvent.class.isAssignableFrom( event.getEventType().getClass() ) ) {
                if ( lastOKEvent == null || lastOKEvent.getDate().before( event.getDate() ) ) lastOKEvent = event;
            } else if ( TroubleStatusFlagEvent.class.isAssignableFrom( event.getEventType().getClass() ) ) {
                if ( lastTroubleEvent == null || lastTroubleEvent.getDate().before( event.getDate() ) )
                    lastTroubleEvent = event;
            }
        }
        if ( lastTroubleEvent != null )
            if ( lastOKEvent == null || lastOKEvent.getDate().before( lastTroubleEvent.getDate() ) )
                return lastTroubleEvent;
        return null;
    }

}
{'fraction_non_alphanumeric': 0.1023232038355778, 'fraction_numerical': 0.0016745399421843052, 'mean_word_length': 1.9982559983088468, 'pattern_counts': {'":': 0, '<': 222, '<?xml version=': 0, '>': 231, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6350296', 'n_tokens_mistral': 16037, 'n_tokens_neox': 15085, 'n_words': 3835}