<commit_before>/**
 * Licensed to Jasig under one or more contributor license
 * agreements. See the NOTICE file distributed with this work
 * for additional information regarding copyright ownership.
 * Jasig licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a
 * copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.jasig.portal.layout.dlm;

import java.io.StringWriter;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.danann.cernunnos.Attributes;
import org.danann.cernunnos.ReturnValueImpl;
import org.danann.cernunnos.Task;
import org.danann.cernunnos.runtime.RuntimeRequestResponse;
import org.dom4j.io.DOMReader;
import org.dom4j.io.DOMWriter;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import org.jasig.portal.EntityIdentifier;
import org.jasig.portal.IUserIdentityStore;
import org.jasig.portal.IUserProfile;
import org.jasig.portal.PortalException;
import org.jasig.portal.RDBMServices;
import org.jasig.portal.layout.LayoutStructure;
import org.jasig.portal.layout.StructureParameter;
import org.jasig.portal.layout.StylesheetUserPreferencesImpl;
import org.jasig.portal.layout.dao.IStylesheetUserPreferencesDao;
import org.jasig.portal.layout.om.IStylesheetDescriptor;
import org.jasig.portal.layout.om.IStylesheetUserPreferences;
import org.jasig.portal.layout.simple.RDBMUserLayoutStore;
import org.jasig.portal.portlet.om.IPortletDefinition;
import org.jasig.portal.portlet.om.IPortletDefinitionId;
import org.jasig.portal.portlet.om.IPortletDefinitionParameter;
import org.jasig.portal.portlet.om.IPortletDescriptorKey;
import org.jasig.portal.portlet.om.IPortletPreference;
import org.jasig.portal.portlet.om.IPortletPreferences;
import org.jasig.portal.portlet.om.IPortletType;
import org.jasig.portal.portlet.om.PortletLifecycleState;
import org.jasig.portal.properties.PropertiesManager;
import org.jasig.portal.security.IPerson;
import org.jasig.portal.security.provider.BrokenSecurityContext;
import org.jasig.portal.security.provider.PersonImpl;
import org.jasig.portal.utils.DocumentFactory;
import org.jasig.portal.xml.XmlUtilitiesImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.w3c.dom.Attr;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

/**
 * This class extends RDBMUserLayoutStore and implements instantiating and
 * storing layouts that conform to the design of the distribute layout
 * management system. These layouts consist of two types: layout fragments
 * that are the layouts owned by a user specified in dlm.xml, and composite
 * view layouts which represent regular users with zero or more UI elements
 * incorporated from layout fragments. Only a user's personal layout fragment
 * is
 *
 * @version $Revision$ $Date$
 * @since uPortal 2.5
 */
public class RDBMDistributedLayoutStore
    extends RDBMUserLayoutStore
{
    public static final String RCS_ID = "@(#) $Header$";
    private static final Log LOG = LogFactory.getLog(RDBMDistributedLayoutStore.class);
    
    private final static Pattern USER_NODE_PATTERN = Pattern.compile("\\A([a-zA-Z]\\d*)\\z");

    private String systemDefaultUser = null;
    private boolean systemDefaultUserLoaded = false;
    
    private ConfigurationLoader configurationLoader;
    private FragmentActivator fragmentActivator;

    private final Map<String, FragmentNodeInfo> fragmentInfoCache = new ConcurrentHashMap<String, FragmentNodeInfo>();
    
    static final String TEMPLATE_USER_NAME
        = "org.jasig.portal.services.Authentication.defaultTemplateUserName";
    
    final static String DELETE_FROM_UP_SS_USER_ATTS_SQL = "DELETE FROM UP_SS_USER_ATTS WHERE USER_ID = ? AND PROFILE_ID = ? AND SS_ID = ? AND SS_TYPE = ?";
    final static String DELETE_FROM_UP_USER_PARM = "DELETE FROM UP_SS_USER_PARM WHERE USER_ID=?  AND PROFILE_ID=? AND SS_ID=? AND SS_TYPE=?";
    
    // Used in Import/Export operations
    private final org.dom4j.DocumentFactory fac = new org.dom4j.DocumentFactory();
    private final DOMReader reader = new DOMReader();
    private final DOMWriter writer = new DOMWriter();
    private Task lookupNoderefTask;
    private Task lookupPathrefTask;
    private IUserIdentityStore identityStore;
    private IStylesheetUserPreferencesDao stylesheetUserPreferencesDao;
    
    @Autowired
    public void setIdentityStore(IUserIdentityStore identityStore) {
        this.identityStore = identityStore;
    }

    @Autowired
    public void setStylesheetUserPreferencesDao(IStylesheetUserPreferencesDao stylesheetUserPreferencesDao) {
        this.stylesheetUserPreferencesDao = stylesheetUserPreferencesDao;
    }

    public void setLookupNoderefTask(Task k) {
        this.lookupNoderefTask = k;
    }

    public void setLookupPathrefTask(Task k) {
        this.lookupPathrefTask = k;
    }

    /**
     * Method for acquiring copies of fragment layouts to assist in debugging.
     * No infrastructure code calls this but channels designed to expose the
     * structure of the cached fragments use this to obtain copies.
     * @return Map
     */
    public Map<String, Document> getFragmentLayoutCopies()
    
    {

        FragmentActivator activator = this.getFragmentActivator();

        Map<String, Document> layouts = new HashMap<String, Document>();

        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        for (final FragmentDefinition fragmentDefinition : definitions) {
            Document layout = DocumentFactory.getNewDocument();
            Node copy = layout.importNode(activator.getUserView(fragmentDefinition)
                        .layout.getDocumentElement(), true);
            layout.appendChild(copy);
            layouts.put(fragmentDefinition.getOwnerId(), layout);
        }
        return layouts;
    }
    
    @Autowired
    public void setConfigurationLoader(ConfigurationLoader configurationLoader) {
        this.configurationLoader = configurationLoader;
    }
    
    @Autowired
    public void setFragmentActivator(FragmentActivator fragmentActivator) {
        this.fragmentActivator = fragmentActivator;
    }

    private FragmentActivator getFragmentActivator() {
        return this.fragmentActivator;
    }

    @Override
    public void cleanFragments() {
        
        FragmentActivator activator = this.getFragmentActivator();

        //get each layout owner
        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        if ( null != definitions ) {
            
            final Map<IPerson, FragmentDefinition> owners = new HashMap<IPerson, FragmentDefinition>();
            for (final FragmentDefinition fragmentDefinition : definitions) {
                String ownerId = fragmentDefinition.getOwnerId();
                final UserView userView = activator.getUserView(fragmentDefinition);
                if (userView != null) {
                    int userId  = userView.getUserId();
    
                    if ( null != ownerId )
                    {
                        IPerson p = new PersonImpl();
                        p.setID( userId );
                        p.setAttribute( "username", ownerId );
                        owners.put(p, fragmentDefinition);
                    }
                }
            }

            // cycle through each layout owner and clear out their
            // respective layouts so users fragments will be cleared
            for (final Map.Entry<IPerson, FragmentDefinition> ownerEntry : owners.entrySet()) {
                final IPerson person = ownerEntry.getKey();
                final IUserProfile profile;
                try {
                    profile = getUserProfileByFname(person, "default");
                }
                catch (Exception e) {
                    this.log.error("Failed to retrieve UserProfile for person " + person + " while cleaning fragment cache, person will be skipped", e);
                    continue;
                }
                
                // TODO fix hard coded "default" later for profiling
                profile.setProfileFname("default");
                
                final Document layout;
                try {
                    layout = getFragmentLayout(person, profile);
                }
                catch (Exception e) {
                    this.log.error("Failed to retrieve layout for person " + person + " and profile " + profile + " while cleaning fragment cache, person will be skipped", e);
                    continue;
                }
                
                FragmentDefinition fragment = ownerEntry.getValue();
                updateCachedLayout( layout, profile, fragment );
            }
        }
        fragmentInfoCache.clear();
    }
    
    
    protected IStylesheetUserPreferences loadDistributedStylesheetUserPreferences(IPerson person, IUserProfile profile, long stylesheetDescriptorId, Set<String> fragmentNames) {
        if (this.isFragmentOwner(person)) {
            return null;
        }
        
        final IStylesheetDescriptor stylesheetDescriptor = this.stylesheetDescriptorDao.getStylesheetDescriptor(stylesheetDescriptorId);
        final IStylesheetUserPreferences stylesheetUserPreferences = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(stylesheetDescriptor, person, profile);
        
        final IStylesheetUserPreferences distributedStylesheetUserPreferences = new StylesheetUserPreferencesImpl(stylesheetDescriptorId);
    
        final FragmentActivator fragmentActivator = this.getFragmentActivator();
        
        for (final String fragmentOwnerId : fragmentNames) {
            final FragmentDefinition fragmentDefinition = this.configurationLoader.getFragmentByName(fragmentOwnerId);
        
            //UserView may be missing if the fragment isn't defined correctly
            final UserView userView = fragmentActivator.getUserView(fragmentDefinition);
            if (userView == null) {
                log.warn("No UserView is present for fragment " + fragmentDefinition.getName() + " it will be skipped when loading distributed stylesheet user preferences");
                continue;
            }
            
            //IStylesheetUserPreferences only exist if something was actually set
            final IStylesheetUserPreferences fragmentStylesheetUserPreferences = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(stylesheetDescriptor, userView.getUserId(), userView.profileId);
            if (fragmentStylesheetUserPreferences == null) {
                continue;
            }
            
            //Get the info needed to DLMify node IDs
            final Element root = userView.layout.getDocumentElement();
            final String labelBase = root.getAttribute( Constants.ATT_ID );
            
            boolean modified = false;
        
            //Copy all of the fragment preferences into the distributed preferences
            final Map<String, Map<String, String>> allLayoutAttributes = fragmentStylesheetUserPreferences.getAllLayoutAttributes();
            for (final Map.Entry<String, Map<String, String>> layoutNodeAttributesEntry : allLayoutAttributes.entrySet()) {
                String nodeId = layoutNodeAttributesEntry.getKey();
                
                if (!nodeId.startsWith(Constants.FRAGMENT_ID_USER_PREFIX)) {
                    nodeId = labelBase + nodeId;
                }
                
                final Map<String, String> layoutAttributes = layoutNodeAttributesEntry.getValue();
                for (final Map.Entry<String, String> layoutAttributesEntry : layoutAttributes.entrySet()) {
                    final String name = layoutAttributesEntry.getKey();
                    final String value = layoutAttributesEntry.getValue();
                    
                    //Fragmentize the nodeId here
                    distributedStylesheetUserPreferences.setLayoutAttribute(nodeId, name, value);
                    
                    //Clean out user preferences data that matches data from the fragment.
                    if (stylesheetUserPreferences != null) {
                        final String userValue = stylesheetUserPreferences.getLayoutAttribute(nodeId, name);
                        if (userValue != null && userValue.equals(value)) {
                            stylesheetUserPreferences.removeLayoutAttribute(nodeId, name);
                            EditManager.removePreferenceDirective(person, nodeId, name);
                            modified = true;
                        }
                    }
                }
            }
            
            if (modified) {
                this.stylesheetUserPreferencesDao.storeStylesheetUserPreferences(stylesheetUserPreferences);
            }
        }
        
        return distributedStylesheetUserPreferences;
    }
    
    @Override
    public double getFragmentPrecedence( int index )
    {
        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        if ( index < 0 ||
             index > definitions.size()-1 )
            return 0;

        // must pass through the array looking for the fragment with this
        // index since the array was sorted by precedence and then index
        // within precedence.
        for (final FragmentDefinition fragmentDefinition : definitions) {
            if ( fragmentDefinition.getIndex() == index ) {
                return fragmentDefinition.getPrecedence();
            }
        }
        return 0; // should never get here.
    }

    /**
       Returns the layout for a user decorated with any specified decorator.
       The layout returned is a composite layout for non fragment owners
       and a regular layout for layout owners. A composite layout is made up
       of layout pieces from potentially multiple incorporated layouts. If
       no layouts are defined then the composite layout will be the same as
       the user's personal layout fragment or PLF, the one holding only those
       UI elements that they own or incorporated elements that they have been
       allowed to changed.
     */
    @Override
    public DistributedUserLayout getUserLayout (IPerson person,
                                   IUserProfile profile)
        
    {

        DistributedUserLayout layout = _getUserLayout( person, profile );

        return layout;
    }
    
    private boolean layoutExistsForUser(IPerson person) {
        
        // Assertions.
        if (person == null) {
            String msg = "Argument 'person' cannot be null.";
            throw new IllegalArgumentException(msg);
        }
        
        final int struct_count = jdbcTemplate.queryForInt("SELECT COUNT(*) FROM up_layout_struct WHERE user_id = ?", person.getID());
        return struct_count == 0 ? false : true;
        
    }
    
    @SuppressWarnings("unchecked")
    public org.dom4j.Element exportLayout(IPerson person, IUserProfile profile) {
        
        if (!layoutExistsForUser(person)) {
            return null;
        }
        
        org.dom4j.Document layoutDoc = null;
        try {
            Document layoutDom = _safeGetUserLayout(person, profile);
            person.setAttribute(Constants.PLF, layoutDom);
            layoutDoc = reader.read(layoutDom);
        } catch (Throwable t) {
            String msg = "Unable to obtain layout & profile for user '" 
                            + person.getUserName() + "', profileId " 
                            + profile.getProfileId();
            throw new RuntimeException(msg, t);
        }
        
        if (log.isDebugEnabled()) {
            // Write out this version of the layout to the log for dev purposes...
            StringWriter str = new StringWriter();
            XMLWriter xml = new XMLWriter(str, new OutputFormat("  ", true));
            try {
                xml.write(layoutDoc);
                xml.close();
            } catch (Throwable t) {
                throw new RuntimeException("Failed to write the layout for user '" 
                            + person.getUserName() + "' to the DEBUG log", t);
            }
            log.debug("Layout for user:  " + person.getUserName() 
                        + "\n" + str.getBuffer().toString());
        }

        /*
         * Clean up the DOM for export.
         */
        
        // (1) Add structure & theme attributes...
        final int structureStylesheetId = profile.getStructureStylesheetId();
        addStylesheetUserPreferencesAttributes(person,
                profile,
                layoutDoc,
                structureStylesheetId,
                "structure");
        
        final int themeStylesheetId = profile.getThemeStylesheetId();
        addStylesheetUserPreferencesAttributes(person,
                profile,
                layoutDoc,
                themeStylesheetId,
                "theme");
                
        // (2) Remove locale info...
        Iterator<org.dom4j.Attribute> locale = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@locale").iterator();
        while (locale.hasNext()) {
            org.dom4j.Attribute loc = locale.next();
            loc.getParent().remove(loc);
        }

        // (3) Scrub unnecessary channel information...
        for (Iterator<org.dom4j.Element> orphanedChannels = (Iterator<org.dom4j.Element>) layoutDoc.selectNodes("//channel[@fname = '']").iterator(); orphanedChannels.hasNext();) {
            // These elements represent UP_LAYOUT_STRUCT rows where the 
            // CHAN_ID field was not recognized by ChannelRegistryStore;  
            // best thing to do is remove the elements...
            org.dom4j.Element ch = orphanedChannels.next();
            ch.getParent().remove(ch);
        }
        List<String> channelAttributeWhitelist = Arrays.asList(new String[] { 
                        "fname", 
                        "unremovable", 
                        "hidden", 
                        "immutable",
                        "ID",
                        "dlm:plfID",
                        "dlm:moveAllowed",
                        "dlm:deleteAllowed"
                    });
        Iterator<org.dom4j.Element> channels = (Iterator<org.dom4j.Element>) layoutDoc.selectNodes("//channel").iterator();
        while (channels.hasNext()) {
            org.dom4j.Element oldCh = channels.next();
            org.dom4j.Element parent = oldCh.getParent();
            org.dom4j.Element newCh = fac.createElement("channel");
            for (String aName : channelAttributeWhitelist) {
                org.dom4j.Attribute a = (org.dom4j.Attribute) oldCh.selectSingleNode("@" + aName);
                if (a != null) {
                    newCh.addAttribute(a.getQName(), a.getValue());
                }
            }
            parent.elements().add(parent.elements().indexOf(oldCh), newCh);
            parent.remove(oldCh);
        }
                
        // (4) Convert internal DLM noderefs to external form (pathrefs)...
        for (Iterator<org.dom4j.Attribute> origins = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:origin").iterator(); origins.hasNext();) {
            org.dom4j.Attribute org = origins.next();
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), org.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                org.setValue(pathTokens[0] + ":" + pathTokens[1]);
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + org.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + org.getValue() + "'");
                }
            }
        }
        for (Iterator<org.dom4j.Attribute> it = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:target").iterator(); it.hasNext();) {
            org.dom4j.Attribute target = it.next();
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), target.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                target.setValue(pathTokens[0] + ":" + pathTokens[1]);
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + target.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + target.getValue() + "'");
                }
            }
        }
        for (Iterator<org.dom4j.Attribute> names = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//dlm:*/@name").iterator(); names.hasNext();) {
            org.dom4j.Attribute n = names.next();
            if (n.getValue() == null || n.getValue().trim().length() == 0) {
                // Outer <dlm:positionSet> elements don't seem to use the name 
                // attribute, though their childern do.  Just skip these so we 
                // don't send a false WARNING.
                continue;
            }
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), n.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                n.setValue(pathTokens[0] + ":" + pathTokens[1]);
                // These *may* have fnames...
                if (pathTokens[2] != null && pathTokens[2].trim().length() != 0) {
                    n.getParent().addAttribute("fname", pathTokens[2]);                
                } 
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + n.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + n.getValue() + "'");
                }
            }
        }

        // Remove synthetic Ids, but from non-fragment owners only...
        if (!isFragmentOwner(person)) {
            
            /*
             * In the case of fragment owners, the original database Ids allow 
             * us keep (not break) the associations that subscribers have with 
             * nodes on the fragment layout.
             */ 
            
            // (5) Remove dlm:plfID...
            for (Iterator<org.dom4j.Attribute> plfid = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:plfID").iterator(); plfid.hasNext();) {
                org.dom4j.Attribute plf = plfid.next();
                plf.getParent().remove(plf);
            }

            // (6) Remove database Ids...
            for (Iterator<org.dom4j.Attribute> ids = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@ID").iterator(); ids.hasNext();) {
                org.dom4j.Attribute a = ids.next();
                a.getParent().remove(a);
            }
        }

        return layoutDoc.getRootElement();

    }

    protected void addStylesheetUserPreferencesAttributes(IPerson person, IUserProfile profile,
            org.dom4j.Document layoutDoc, int stylesheetId, String attributeType) {
        final IStylesheetDescriptor structureStylesheetDescriptor = this.stylesheetDescriptorDao.getStylesheetDescriptor(stylesheetId);
        
        final IStylesheetUserPreferences ssup = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(structureStylesheetDescriptor, person, profile);
        
        final Map<String, Map<String, String>> allLayoutAttributes = ssup.getAllLayoutAttributes();
        for (final Entry<String, Map<String, String>> nodeEntry : allLayoutAttributes.entrySet()) {
            final String nodeId = nodeEntry.getKey();
            final Map<String, String> attributes = nodeEntry.getValue();
            
            final org.dom4j.Element element = layoutDoc.elementByID(nodeId);
            if (element == null) {
                this.log.warn("No layout node with id '" + nodeId + "' found attributes will be ignored: " + attributes);
                continue;
            }
            
            for (final Entry<String, String> attributeEntry : attributes.entrySet()) {
                final String name = attributeEntry.getKey();
                final String value = attributeEntry.getValue();
                
                if (log.isDebugEnabled()) {
                    log.debug("Adding structure folder attribute:  name=" + name + ", value=" + value);
                }
                org.dom4j.Element structAttrElement = fac.createElement(attributeType + "-attribute");
                org.dom4j.Element nameAttribute = structAttrElement.addElement("name");
                nameAttribute.setText(name);
                org.dom4j.Element valueAttribute = structAttrElement.addElement("value");
                valueAttribute.setText(value);
                element.elements().add(0, structAttrElement);
            }
        }
    }
        
    @Override
    @SuppressWarnings("unchecked")
    @Transactional
    public void importLayout(org.dom4j.Element layout) {
        
        String ownerUsername = layout.valueOf("@username");
        IPerson person = null;
        IUserProfile profile = null;
        try {
            person = new PersonImpl();
            person.setUserName(ownerUsername);
            int ownerId = identityStore.getPortalUID(person);
            if (ownerId == -1) {
                String msg = "No userId for username=" + ownerUsername;
                throw new RuntimeException(msg);
            }
            person.setID(ownerId);
            person.setSecurityContext(new BrokenSecurityContext());
            profile = this.getUserProfileByFname(person, "default");
        } catch (Throwable t) {
            String msg = "Unrecognized user " + person.getUserName() + "; you must import users before their layouts.";
            throw new RuntimeException(msg, t);
        }
        
        // (6) Add database Ids & (5) Add dlm:plfID ...<commit_msg>NOJIRA guard against possible null return value from getStylesheetUserPreferences<commit_after>/**
 * Licensed to Jasig under one or more contributor license
 * agreements. See the NOTICE file distributed with this work
 * for additional information regarding copyright ownership.
 * Jasig licenses this file to you under the Apache License,
 * Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a
 * copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package org.jasig.portal.layout.dlm;

import java.io.StringWriter;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.danann.cernunnos.Attributes;
import org.danann.cernunnos.ReturnValueImpl;
import org.danann.cernunnos.Task;
import org.danann.cernunnos.runtime.RuntimeRequestResponse;
import org.dom4j.io.DOMReader;
import org.dom4j.io.DOMWriter;
import org.dom4j.io.OutputFormat;
import org.dom4j.io.XMLWriter;
import org.jasig.portal.EntityIdentifier;
import org.jasig.portal.IUserIdentityStore;
import org.jasig.portal.IUserProfile;
import org.jasig.portal.PortalException;
import org.jasig.portal.RDBMServices;
import org.jasig.portal.layout.LayoutStructure;
import org.jasig.portal.layout.StructureParameter;
import org.jasig.portal.layout.StylesheetUserPreferencesImpl;
import org.jasig.portal.layout.dao.IStylesheetUserPreferencesDao;
import org.jasig.portal.layout.om.IStylesheetDescriptor;
import org.jasig.portal.layout.om.IStylesheetUserPreferences;
import org.jasig.portal.layout.simple.RDBMUserLayoutStore;
import org.jasig.portal.portlet.om.IPortletDefinition;
import org.jasig.portal.portlet.om.IPortletDefinitionId;
import org.jasig.portal.portlet.om.IPortletDefinitionParameter;
import org.jasig.portal.portlet.om.IPortletDescriptorKey;
import org.jasig.portal.portlet.om.IPortletPreference;
import org.jasig.portal.portlet.om.IPortletPreferences;
import org.jasig.portal.portlet.om.IPortletType;
import org.jasig.portal.portlet.om.PortletLifecycleState;
import org.jasig.portal.properties.PropertiesManager;
import org.jasig.portal.security.IPerson;
import org.jasig.portal.security.provider.BrokenSecurityContext;
import org.jasig.portal.security.provider.PersonImpl;
import org.jasig.portal.utils.DocumentFactory;
import org.jasig.portal.xml.XmlUtilitiesImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.w3c.dom.Attr;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

/**
 * This class extends RDBMUserLayoutStore and implements instantiating and
 * storing layouts that conform to the design of the distribute layout
 * management system. These layouts consist of two types: layout fragments
 * that are the layouts owned by a user specified in dlm.xml, and composite
 * view layouts which represent regular users with zero or more UI elements
 * incorporated from layout fragments. Only a user's personal layout fragment
 * is
 *
 * @version $Revision$ $Date$
 * @since uPortal 2.5
 */
public class RDBMDistributedLayoutStore
    extends RDBMUserLayoutStore
{
    public static final String RCS_ID = "@(#) $Header$";
    private static final Log LOG = LogFactory.getLog(RDBMDistributedLayoutStore.class);
    
    private final static Pattern USER_NODE_PATTERN = Pattern.compile("\\A([a-zA-Z]\\d*)\\z");

    private String systemDefaultUser = null;
    private boolean systemDefaultUserLoaded = false;
    
    private ConfigurationLoader configurationLoader;
    private FragmentActivator fragmentActivator;

    private final Map<String, FragmentNodeInfo> fragmentInfoCache = new ConcurrentHashMap<String, FragmentNodeInfo>();
    
    static final String TEMPLATE_USER_NAME
        = "org.jasig.portal.services.Authentication.defaultTemplateUserName";
    
    final static String DELETE_FROM_UP_SS_USER_ATTS_SQL = "DELETE FROM UP_SS_USER_ATTS WHERE USER_ID = ? AND PROFILE_ID = ? AND SS_ID = ? AND SS_TYPE = ?";
    final static String DELETE_FROM_UP_USER_PARM = "DELETE FROM UP_SS_USER_PARM WHERE USER_ID=?  AND PROFILE_ID=? AND SS_ID=? AND SS_TYPE=?";
    
    // Used in Import/Export operations
    private final org.dom4j.DocumentFactory fac = new org.dom4j.DocumentFactory();
    private final DOMReader reader = new DOMReader();
    private final DOMWriter writer = new DOMWriter();
    private Task lookupNoderefTask;
    private Task lookupPathrefTask;
    private IUserIdentityStore identityStore;
    private IStylesheetUserPreferencesDao stylesheetUserPreferencesDao;
    
    @Autowired
    public void setIdentityStore(IUserIdentityStore identityStore) {
        this.identityStore = identityStore;
    }

    @Autowired
    public void setStylesheetUserPreferencesDao(IStylesheetUserPreferencesDao stylesheetUserPreferencesDao) {
        this.stylesheetUserPreferencesDao = stylesheetUserPreferencesDao;
    }

    public void setLookupNoderefTask(Task k) {
        this.lookupNoderefTask = k;
    }

    public void setLookupPathrefTask(Task k) {
        this.lookupPathrefTask = k;
    }

    /**
     * Method for acquiring copies of fragment layouts to assist in debugging.
     * No infrastructure code calls this but channels designed to expose the
     * structure of the cached fragments use this to obtain copies.
     * @return Map
     */
    public Map<String, Document> getFragmentLayoutCopies()
    
    {

        FragmentActivator activator = this.getFragmentActivator();

        Map<String, Document> layouts = new HashMap<String, Document>();

        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        for (final FragmentDefinition fragmentDefinition : definitions) {
            Document layout = DocumentFactory.getNewDocument();
            Node copy = layout.importNode(activator.getUserView(fragmentDefinition)
                        .layout.getDocumentElement(), true);
            layout.appendChild(copy);
            layouts.put(fragmentDefinition.getOwnerId(), layout);
        }
        return layouts;
    }
    
    @Autowired
    public void setConfigurationLoader(ConfigurationLoader configurationLoader) {
        this.configurationLoader = configurationLoader;
    }
    
    @Autowired
    public void setFragmentActivator(FragmentActivator fragmentActivator) {
        this.fragmentActivator = fragmentActivator;
    }

    private FragmentActivator getFragmentActivator() {
        return this.fragmentActivator;
    }

    @Override
    public void cleanFragments() {
        
        FragmentActivator activator = this.getFragmentActivator();

        //get each layout owner
        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        if ( null != definitions ) {
            
            final Map<IPerson, FragmentDefinition> owners = new HashMap<IPerson, FragmentDefinition>();
            for (final FragmentDefinition fragmentDefinition : definitions) {
                String ownerId = fragmentDefinition.getOwnerId();
                final UserView userView = activator.getUserView(fragmentDefinition);
                if (userView != null) {
                    int userId  = userView.getUserId();
    
                    if ( null != ownerId )
                    {
                        IPerson p = new PersonImpl();
                        p.setID( userId );
                        p.setAttribute( "username", ownerId );
                        owners.put(p, fragmentDefinition);
                    }
                }
            }

            // cycle through each layout owner and clear out their
            // respective layouts so users fragments will be cleared
            for (final Map.Entry<IPerson, FragmentDefinition> ownerEntry : owners.entrySet()) {
                final IPerson person = ownerEntry.getKey();
                final IUserProfile profile;
                try {
                    profile = getUserProfileByFname(person, "default");
                }
                catch (Exception e) {
                    this.log.error("Failed to retrieve UserProfile for person " + person + " while cleaning fragment cache, person will be skipped", e);
                    continue;
                }
                
                // TODO fix hard coded "default" later for profiling
                profile.setProfileFname("default");
                
                final Document layout;
                try {
                    layout = getFragmentLayout(person, profile);
                }
                catch (Exception e) {
                    this.log.error("Failed to retrieve layout for person " + person + " and profile " + profile + " while cleaning fragment cache, person will be skipped", e);
                    continue;
                }
                
                FragmentDefinition fragment = ownerEntry.getValue();
                updateCachedLayout( layout, profile, fragment );
            }
        }
        fragmentInfoCache.clear();
    }
    
    
    protected IStylesheetUserPreferences loadDistributedStylesheetUserPreferences(IPerson person, IUserProfile profile, long stylesheetDescriptorId, Set<String> fragmentNames) {
        if (this.isFragmentOwner(person)) {
            return null;
        }
        
        final IStylesheetDescriptor stylesheetDescriptor = this.stylesheetDescriptorDao.getStylesheetDescriptor(stylesheetDescriptorId);
        final IStylesheetUserPreferences stylesheetUserPreferences = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(stylesheetDescriptor, person, profile);
        
        final IStylesheetUserPreferences distributedStylesheetUserPreferences = new StylesheetUserPreferencesImpl(stylesheetDescriptorId);
    
        final FragmentActivator fragmentActivator = this.getFragmentActivator();
        
        for (final String fragmentOwnerId : fragmentNames) {
            final FragmentDefinition fragmentDefinition = this.configurationLoader.getFragmentByName(fragmentOwnerId);
        
            //UserView may be missing if the fragment isn't defined correctly
            final UserView userView = fragmentActivator.getUserView(fragmentDefinition);
            if (userView == null) {
                log.warn("No UserView is present for fragment " + fragmentDefinition.getName() + " it will be skipped when loading distributed stylesheet user preferences");
                continue;
            }
            
            //IStylesheetUserPreferences only exist if something was actually set
            final IStylesheetUserPreferences fragmentStylesheetUserPreferences = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(stylesheetDescriptor, userView.getUserId(), userView.profileId);
            if (fragmentStylesheetUserPreferences == null) {
                continue;
            }
            
            //Get the info needed to DLMify node IDs
            final Element root = userView.layout.getDocumentElement();
            final String labelBase = root.getAttribute( Constants.ATT_ID );
            
            boolean modified = false;
        
            //Copy all of the fragment preferences into the distributed preferences
            final Map<String, Map<String, String>> allLayoutAttributes = fragmentStylesheetUserPreferences.getAllLayoutAttributes();
            for (final Map.Entry<String, Map<String, String>> layoutNodeAttributesEntry : allLayoutAttributes.entrySet()) {
                String nodeId = layoutNodeAttributesEntry.getKey();
                
                if (!nodeId.startsWith(Constants.FRAGMENT_ID_USER_PREFIX)) {
                    nodeId = labelBase + nodeId;
                }
                
                final Map<String, String> layoutAttributes = layoutNodeAttributesEntry.getValue();
                for (final Map.Entry<String, String> layoutAttributesEntry : layoutAttributes.entrySet()) {
                    final String name = layoutAttributesEntry.getKey();
                    final String value = layoutAttributesEntry.getValue();
                    
                    //Fragmentize the nodeId here
                    distributedStylesheetUserPreferences.setLayoutAttribute(nodeId, name, value);
                    
                    //Clean out user preferences data that matches data from the fragment.
                    if (stylesheetUserPreferences != null) {
                        final String userValue = stylesheetUserPreferences.getLayoutAttribute(nodeId, name);
                        if (userValue != null && userValue.equals(value)) {
                            stylesheetUserPreferences.removeLayoutAttribute(nodeId, name);
                            EditManager.removePreferenceDirective(person, nodeId, name);
                            modified = true;
                        }
                    }
                }
            }
            
            if (modified) {
                this.stylesheetUserPreferencesDao.storeStylesheetUserPreferences(stylesheetUserPreferences);
            }
        }
        
        return distributedStylesheetUserPreferences;
    }
    
    @Override
    public double getFragmentPrecedence( int index )
    {
        final List<FragmentDefinition> definitions = configurationLoader.getFragments();
        if ( index < 0 ||
             index > definitions.size()-1 )
            return 0;

        // must pass through the array looking for the fragment with this
        // index since the array was sorted by precedence and then index
        // within precedence.
        for (final FragmentDefinition fragmentDefinition : definitions) {
            if ( fragmentDefinition.getIndex() == index ) {
                return fragmentDefinition.getPrecedence();
            }
        }
        return 0; // should never get here.
    }

    /**
       Returns the layout for a user decorated with any specified decorator.
       The layout returned is a composite layout for non fragment owners
       and a regular layout for layout owners. A composite layout is made up
       of layout pieces from potentially multiple incorporated layouts. If
       no layouts are defined then the composite layout will be the same as
       the user's personal layout fragment or PLF, the one holding only those
       UI elements that they own or incorporated elements that they have been
       allowed to changed.
     */
    @Override
    public DistributedUserLayout getUserLayout (IPerson person,
                                   IUserProfile profile)
        
    {

        DistributedUserLayout layout = _getUserLayout( person, profile );

        return layout;
    }
    
    private boolean layoutExistsForUser(IPerson person) {
        
        // Assertions.
        if (person == null) {
            String msg = "Argument 'person' cannot be null.";
            throw new IllegalArgumentException(msg);
        }
        
        final int struct_count = jdbcTemplate.queryForInt("SELECT COUNT(*) FROM up_layout_struct WHERE user_id = ?", person.getID());
        return struct_count == 0 ? false : true;
        
    }
    
    @SuppressWarnings("unchecked")
    public org.dom4j.Element exportLayout(IPerson person, IUserProfile profile) {
        
        if (!layoutExistsForUser(person)) {
            return null;
        }
        
        org.dom4j.Document layoutDoc = null;
        try {
            Document layoutDom = _safeGetUserLayout(person, profile);
            person.setAttribute(Constants.PLF, layoutDom);
            layoutDoc = reader.read(layoutDom);
        } catch (Throwable t) {
            String msg = "Unable to obtain layout & profile for user '" 
                            + person.getUserName() + "', profileId " 
                            + profile.getProfileId();
            throw new RuntimeException(msg, t);
        }
        
        if (log.isDebugEnabled()) {
            // Write out this version of the layout to the log for dev purposes...
            StringWriter str = new StringWriter();
            XMLWriter xml = new XMLWriter(str, new OutputFormat("  ", true));
            try {
                xml.write(layoutDoc);
                xml.close();
            } catch (Throwable t) {
                throw new RuntimeException("Failed to write the layout for user '" 
                            + person.getUserName() + "' to the DEBUG log", t);
            }
            log.debug("Layout for user:  " + person.getUserName() 
                        + "\n" + str.getBuffer().toString());
        }

        /*
         * Clean up the DOM for export.
         */
        
        // (1) Add structure & theme attributes...
        final int structureStylesheetId = profile.getStructureStylesheetId();
        addStylesheetUserPreferencesAttributes(person,
                profile,
                layoutDoc,
                structureStylesheetId,
                "structure");
        
        final int themeStylesheetId = profile.getThemeStylesheetId();
        addStylesheetUserPreferencesAttributes(person,
                profile,
                layoutDoc,
                themeStylesheetId,
                "theme");
                
        // (2) Remove locale info...
        Iterator<org.dom4j.Attribute> locale = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@locale").iterator();
        while (locale.hasNext()) {
            org.dom4j.Attribute loc = locale.next();
            loc.getParent().remove(loc);
        }

        // (3) Scrub unnecessary channel information...
        for (Iterator<org.dom4j.Element> orphanedChannels = (Iterator<org.dom4j.Element>) layoutDoc.selectNodes("//channel[@fname = '']").iterator(); orphanedChannels.hasNext();) {
            // These elements represent UP_LAYOUT_STRUCT rows where the 
            // CHAN_ID field was not recognized by ChannelRegistryStore;  
            // best thing to do is remove the elements...
            org.dom4j.Element ch = orphanedChannels.next();
            ch.getParent().remove(ch);
        }
        List<String> channelAttributeWhitelist = Arrays.asList(new String[] { 
                        "fname", 
                        "unremovable", 
                        "hidden", 
                        "immutable",
                        "ID",
                        "dlm:plfID",
                        "dlm:moveAllowed",
                        "dlm:deleteAllowed"
                    });
        Iterator<org.dom4j.Element> channels = (Iterator<org.dom4j.Element>) layoutDoc.selectNodes("//channel").iterator();
        while (channels.hasNext()) {
            org.dom4j.Element oldCh = channels.next();
            org.dom4j.Element parent = oldCh.getParent();
            org.dom4j.Element newCh = fac.createElement("channel");
            for (String aName : channelAttributeWhitelist) {
                org.dom4j.Attribute a = (org.dom4j.Attribute) oldCh.selectSingleNode("@" + aName);
                if (a != null) {
                    newCh.addAttribute(a.getQName(), a.getValue());
                }
            }
            parent.elements().add(parent.elements().indexOf(oldCh), newCh);
            parent.remove(oldCh);
        }
                
        // (4) Convert internal DLM noderefs to external form (pathrefs)...
        for (Iterator<org.dom4j.Attribute> origins = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:origin").iterator(); origins.hasNext();) {
            org.dom4j.Attribute org = origins.next();
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), org.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                org.setValue(pathTokens[0] + ":" + pathTokens[1]);
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + org.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + org.getValue() + "'");
                }
            }
        }
        for (Iterator<org.dom4j.Attribute> it = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:target").iterator(); it.hasNext();) {
            org.dom4j.Attribute target = it.next();
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), target.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                target.setValue(pathTokens[0] + ":" + pathTokens[1]);
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + target.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + target.getValue() + "'");
                }
            }
        }
        for (Iterator<org.dom4j.Attribute> names = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//dlm:*/@name").iterator(); names.hasNext();) {
            org.dom4j.Attribute n = names.next();
            if (n.getValue() == null || n.getValue().trim().length() == 0) {
                // Outer <dlm:positionSet> elements don't seem to use the name 
                // attribute, though their childern do.  Just skip these so we 
                // don't send a false WARNING.
                continue;
            }
            String[] pathTokens = getDlmPathref((String) person.getAttribute(IPerson.USERNAME), person.getID(), n.getValue(), layoutDoc.getRootElement());
            if (pathTokens != null) {
                // Change the value only if we have a valid pathref...
                n.setValue(pathTokens[0] + ":" + pathTokens[1]);
                // These *may* have fnames...
                if (pathTokens[2] != null && pathTokens[2].trim().length() != 0) {
                    n.getParent().addAttribute("fname", pathTokens[2]);                
                } 
            } else {
                if (log.isWarnEnabled()) {
                    log.warn("Layout element '" + n.getUniquePath() 
                            + "' from user '" + person.getAttribute(IPerson.USERNAME) 
                            + "' failed to match noderef '" + n.getValue() + "'");
                }
            }
        }

        // Remove synthetic Ids, but from non-fragment owners only...
        if (!isFragmentOwner(person)) {
            
            /*
             * In the case of fragment owners, the original database Ids allow 
             * us keep (not break) the associations that subscribers have with 
             * nodes on the fragment layout.
             */ 
            
            // (5) Remove dlm:plfID...
            for (Iterator<org.dom4j.Attribute> plfid = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@dlm:plfID").iterator(); plfid.hasNext();) {
                org.dom4j.Attribute plf = plfid.next();
                plf.getParent().remove(plf);
            }

            // (6) Remove database Ids...
            for (Iterator<org.dom4j.Attribute> ids = (Iterator<org.dom4j.Attribute>) layoutDoc.selectNodes("//@ID").iterator(); ids.hasNext();) {
                org.dom4j.Attribute a = ids.next();
                a.getParent().remove(a);
            }
        }

        return layoutDoc.getRootElement();

    }

    protected void addStylesheetUserPreferencesAttributes(IPerson person, IUserProfile profile,
            org.dom4j.Document layoutDoc, int stylesheetId, String attributeType) {
        final IStylesheetDescriptor structureStylesheetDescriptor = this.stylesheetDescriptorDao.getStylesheetDescriptor(stylesheetId);
        
        final IStylesheetUserPreferences ssup = this.stylesheetUserPreferencesDao.getStylesheetUserPreferences(structureStylesheetDescriptor, person, profile);
        
        if(ssup != null) {
        	final Map<String, Map<String, String>> allLayoutAttributes = ssup.getAllLayoutAttributes();
        	for (final Entry<String, Map<String, String>> nodeEntry : allLayoutAttributes.entrySet()) {
        		final String nodeId = nodeEntry.getKey();
        		final Map<String, String> attributes = nodeEntry.getValue();

        		final org.dom4j.Element element = layoutDoc.elementByID(nodeId);
        		if (element == null) {
        			this.log.warn("No layout node with id '" + nodeId + "' found attributes will be ignored: " + attributes);
        			continue;
        		}

        		for (final Entry<String, String> attributeEntry : attributes.entrySet()) {
        			final String name = attributeEntry.getKey();
        			final String value = attributeEntry.getValue();

        			if (log.isDebugEnabled()) {
        				log.debug("Adding structure folder attribute:  name=" + name + ", value=" + value);
        			}
        			org.dom4j.Element structAttrElement = fac.createElement(attributeType + "-attribute");
        			org.dom4j.Element nameAttribute = structAttrElement.addElement("name");
        			nameAttribute.setText(name);
        			org.dom4j.Element valueAttribute = structAttrElement.addElement("value");
        			valueAttribute.setText(value);
        			element.elements().add(0, structAttrElement);
        		}
        	}
        } else {
        	LOG.debug("no StylesheetUserPreferences found for " + person + ", " + profile);
        }
    }
        
    @Override
    @SuppressWarnings("unchecked")
    @Transactional
    public void importLayout(org.dom4j.Element layout) {
        
        String ownerUsername = layout.valueOf("@username");
        IPerson person = null;
        IUserProfile profile = null;
        try {
            person = new PersonImpl();
            person.setUserName(ownerUsername);
            int ownerId = identityStore.getPortalUID(person);
            if (ownerId == -1) {
                String msg = "No userId for username=" + ownerUsername;
                throw new RuntimeException(msg);
            }
            person.setID(ownerId);
            person.setSecurityContext(new BrokenSecurityContext());
            profile = this.getUserProfileByFname(person, "default");
        } catch (Throwable t) {
            String msg = "Unrecognized user " + person.getUserName() + "; you must import users before their layouts.";
            throw new RuntimeException(msg, t);
        }
        
        // (6) Add database Ids & (5) Add dlm:plfID ...
{'fraction_non_alphanumeric': 0.09120086548864047, 'fraction_numerical': 0.00288496213487198, 'mean_word_length': 2.2252267969295185, 'pattern_counts': {'":': 6, '<': 89, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7594000', 'n_tokens_mistral': 13989, 'n_tokens_neox': 13757, 'n_words': 3495}