<commit_before>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package com.xpn.xwiki.plugin.lucene.internal;

import java.util.Date;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.xwiki.model.reference.EntityReferenceSerializer;

import com.xpn.xwiki.XWikiContext;
import com.xpn.xwiki.doc.XWikiDocument;
import com.xpn.xwiki.objects.BaseObject;
import com.xpn.xwiki.objects.BaseProperty;
import com.xpn.xwiki.objects.PropertyInterface;
import com.xpn.xwiki.objects.classes.BaseClass;
import com.xpn.xwiki.objects.classes.DateClass;
import com.xpn.xwiki.objects.classes.ListItem;
import com.xpn.xwiki.objects.classes.PasswordClass;
import com.xpn.xwiki.objects.classes.StaticListClass;
import com.xpn.xwiki.plugin.lucene.LucenePlugin;
import com.xpn.xwiki.web.Utils;

/**
 * Holds all data but the content of a wiki page to be indexed. The content is retrieved at indexing time, which should
 * save us some memory especially when rebuilding an index for a big wiki.
 * 
 * @version $Id$
 */
public class DocumentData extends AbstractDocumentData
{
    /** The importance of an object classname. **/
    private static final float CLASSNAME_BOOST = 0.5f;

    /** The importance of an object property. **/
    private static final float OBJECT_PROPERTY_BOOST = 0.75f;

    /** Reference serializer which removes the wiki prefix. */
    @SuppressWarnings("unchecked")
    private EntityReferenceSerializer<String> localEntityReferenceSerializer = Utils.getComponent(
        EntityReferenceSerializer.TYPE_STRING, "local");

    public DocumentData(final XWikiDocument doc, final XWikiContext context, final boolean deleted)
    {
        super(LucenePlugin.DOCTYPE_WIKIPAGE, doc, context, deleted);

        setAuthor(doc.getAuthor());
        setCreator(doc.getCreator());
        setModificationDate(doc.getDate());
        setCreationDate(doc.getCreationDate());
    }

    /**
     * Append a string containing the result of {@link AbstractIndexData#getFullText} plus the full text content of this
     * document (in the given language)
     */
    @Override
    protected void getFullText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        super.getFullText(sb, doc, context);

        sb.append(" ");
        sb.append(StringUtils.lowerCase(doc.getContent()));
        sb.append(" ");

        getObjectFullText(sb, doc, context);
    }

    /**
     * Add to the string builder, the result of {@link AbstractIndexData#getFullText(XWikiDocument,XWikiContext)}plus
     * the full text content (values of title,category,content and extract ) XWiki.ArticleClass Object, as far as it
     * could be extracted.
     */
    private void getObjectFullText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        getObjectContentAsText(sb, doc, context);
    }

    /**
     * Add to the string builder the value of title,category,content and extract of XWiki.ArticleClass
     */
    private void getObjectContentAsText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        for (List<BaseObject> objects : doc.getXObjects().values()) {
            for (BaseObject obj : objects) {
                extractObjectContent(sb, obj, context);
            }
        }
    }

    private void getObjectContentAsText(StringBuilder contentText, BaseObject baseObject, String property,
        XWikiContext context)
    {
        BaseProperty baseProperty = (BaseProperty) baseObject.getField(property);
        // FIXME Can baseProperty really be null?
        if (baseProperty != null && baseProperty.getValue() != null) {
            if (!(baseObject.getXClass(context).getField(property) instanceof PasswordClass)) {
                contentText.append(StringUtils.lowerCase(baseProperty.getValue().toString()));
            }
        }
    }

    private void extractObjectContent(StringBuilder contentText, BaseObject baseObject, XWikiContext context)
    {
        if (baseObject != null) {
            String[] propertyNames = baseObject.getPropertyNames();
            for (String propertyName : propertyNames) {
                getObjectContentAsText(contentText, baseObject, propertyName, context);
                contentText.append(" ");
            }
        }
    }

    @Override
    public void addDocumentDataToLuceneDocument(Document luceneDoc, XWikiDocument doc, XWikiContext context)
    {
        super.addDocumentDataToLuceneDocument(luceneDoc, doc, context);

        for (List<BaseObject> objects : doc.getXObjects().values()) {
            for (BaseObject obj : objects) {
                if (obj != null) {
                    addFieldToDocument(IndexFields.OBJECT,
                        this.localEntityReferenceSerializer.serialize(obj.getXClassReference()).toLowerCase(),
                        Field.Store.YES, Field.Index.NOT_ANALYZED, CLASSNAME_BOOST, luceneDoc);
                    Object[] propertyNames = obj.getPropertyNames();
                    for (int i = 0; i < propertyNames.length; i++) {
                        indexProperty(luceneDoc, obj, (String) propertyNames[i], context);
                    }
                }
            }
        }
    }

    private void indexProperty(Document luceneDoc, BaseObject baseObject, String propertyName, XWikiContext context)
    {
        String fieldFullName = baseObject.getClassName() + "." + propertyName;
        BaseClass bClass = baseObject.getXClass(context);
        PropertyInterface prop = bClass.getField(propertyName);

        if (prop instanceof PasswordClass) {
            // Do not index passwords
        } else if (prop instanceof StaticListClass && ((StaticListClass) prop).isMultiSelect()) {
            indexStaticList(luceneDoc, baseObject, (StaticListClass) prop, propertyName, context);
        } else if (prop instanceof DateClass) {
            // Date properties are indexed the same as document dates: formatted through IndexFields.dateToString() and
            // untokenized, to be able to sort by their values.
            addFieldToDocument(fieldFullName, getContentAsDate(baseObject, propertyName), Field.Store.YES,
                Field.Index.NOT_ANALYZED, OBJECT_PROPERTY_BOOST, luceneDoc);
        } else {
            StringBuilder sb = new StringBuilder();
            getObjectContentAsText(sb, baseObject, propertyName, context);
            addFieldToDocument(fieldFullName, sb.toString(), Field.Store.YES, Field.Index.ANALYZED,
                OBJECT_PROPERTY_BOOST, luceneDoc);
        }
    }

    private void indexStaticList(Document luceneDoc, BaseObject baseObject, StaticListClass prop, String propertyName,
        XWikiContext context)
    {
        Map<String, ListItem> possibleValues = prop.getMap(context);
        String fieldFullName = baseObject.getClassName() + "." + propertyName;<commit_msg>XWIKI-9067 : Improve the way list property values are indexed by Lucene<commit_after>/*
 * See the NOTICE file distributed with this work for additional
 * information regarding copyright ownership.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package com.xpn.xwiki.plugin.lucene.internal;

import java.util.Date;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.xwiki.model.reference.EntityReference;
import org.xwiki.model.reference.EntityReferenceSerializer;

import com.xpn.xwiki.XWikiContext;
import com.xpn.xwiki.doc.XWikiDocument;
import com.xpn.xwiki.objects.BaseObject;
import com.xpn.xwiki.objects.BaseProperty;
import com.xpn.xwiki.objects.PropertyInterface;
import com.xpn.xwiki.objects.classes.BaseClass;
import com.xpn.xwiki.objects.classes.DateClass;
import com.xpn.xwiki.objects.classes.ListItem;
import com.xpn.xwiki.objects.classes.PasswordClass;
import com.xpn.xwiki.objects.classes.PropertyClass;
import com.xpn.xwiki.objects.classes.StaticListClass;
import com.xpn.xwiki.plugin.lucene.LucenePlugin;
import com.xpn.xwiki.web.Utils;

/**
 * Holds all data but the content of a wiki page to be indexed. The content is retrieved at indexing time, which should
 * save us some memory especially when rebuilding an index for a big wiki.
 * 
 * @version $Id$
 */
public class DocumentData extends AbstractDocumentData
{
    /** The importance of an object classname. **/
    private static final float CLASSNAME_BOOST = 0.5f;

    /** The importance of an object property. **/
    private static final float OBJECT_PROPERTY_BOOST = 0.75f;

    /** Reference serializer which removes the wiki prefix. */
    @SuppressWarnings("unchecked")
    private EntityReferenceSerializer<String> localEntityReferenceSerializer = Utils.getComponent(
        EntityReferenceSerializer.TYPE_STRING, "local");

    public DocumentData(final XWikiDocument doc, final XWikiContext context, final boolean deleted)
    {
        super(LucenePlugin.DOCTYPE_WIKIPAGE, doc, context, deleted);

        setAuthor(doc.getAuthor());
        setCreator(doc.getCreator());
        setModificationDate(doc.getDate());
        setCreationDate(doc.getCreationDate());
    }

    /**
     * Append a string containing the result of {@link AbstractIndexData#getFullText} plus the full text content of this
     * document (in the given language)
     */
    @Override
    protected void getFullText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        super.getFullText(sb, doc, context);

        sb.append(" ");
        sb.append(StringUtils.lowerCase(doc.getContent()));
        sb.append(" ");

        getObjectFullText(sb, doc, context);
    }

    /**
     * Add to the string builder, the result of {@link AbstractIndexData#getFullText(XWikiDocument,XWikiContext)}plus
     * the full text content (values of title,category,content and extract ) XWiki.ArticleClass Object, as far as it
     * could be extracted.
     */
    private void getObjectFullText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        getObjectContentAsText(sb, doc, context);
    }

    /**
     * Add to the string builder the value of title,category,content and extract of XWiki.ArticleClass
     */
    private void getObjectContentAsText(StringBuilder sb, XWikiDocument doc, XWikiContext context)
    {
        for (List<BaseObject> objects : doc.getXObjects().values()) {
            for (BaseObject obj : objects) {
                extractObjectContent(sb, obj, context);
            }
        }
    }

    private void getObjectContentAsText(StringBuilder contentText, BaseObject baseObject, String property,
        XWikiContext context)
    {
        BaseProperty baseProperty = (BaseProperty) baseObject.getField(property);
        // FIXME Can baseProperty really be null?
        if (baseProperty != null && baseProperty.getValue() != null) {
            if (!(baseObject.getXClass(context).getField(property) instanceof PasswordClass)) {
                contentText.append(StringUtils.lowerCase(baseProperty.getValue().toString()));
            }
        }
    }

    private void extractObjectContent(StringBuilder contentText, BaseObject baseObject, XWikiContext context)
    {
        if (baseObject != null) {
            String[] propertyNames = baseObject.getPropertyNames();
            for (String propertyName : propertyNames) {
                getObjectContentAsText(contentText, baseObject, propertyName, context);
                contentText.append(" ");
            }
        }
    }

    @Override
    public void addDocumentDataToLuceneDocument(Document luceneDoc, XWikiDocument doc, XWikiContext context)
    {
        super.addDocumentDataToLuceneDocument(luceneDoc, doc, context);

        for (List<BaseObject> objects : doc.getXObjects().values()) {
            for (BaseObject obj : objects) {
                if (obj != null) {
                    addFieldToDocument(IndexFields.OBJECT,
                        this.localEntityReferenceSerializer.serialize(obj.getXClassReference()).toLowerCase(),
                        Field.Store.YES, Field.Index.NOT_ANALYZED, CLASSNAME_BOOST, luceneDoc);
                    Object[] propertyNames = obj.getPropertyNames();
                    for (int i = 0; i < propertyNames.length; i++) {
                        indexProperty(luceneDoc, obj, (String) propertyNames[i], context);
                    }
                }
            }
        }
    }

    private void indexProperty(Document luceneDoc, BaseObject baseObject, String propertyName, XWikiContext context)
    {
        String fieldFullName = baseObject.getClassName() + "." + propertyName;

        BaseClass xClass = baseObject.getXClass(context);
        BaseProperty<EntityReference> property = (BaseProperty<EntityReference>) baseObject.getField(propertyName);
        PropertyClass propertyClass = (PropertyClass) xClass.get(property.getName());

        Object propertyValue = property.getValue();
        if (propertyValue != null) {
            if (propertyClass instanceof PasswordClass) {
                // Do not index passwords
            } else if (propertyClass instanceof StaticListClass && ((StaticListClass) propertyClass).isMultiSelect()) {
                indexStaticList(luceneDoc, baseObject, (StaticListClass) propertyClass, propertyName, context);
            } else if (propertyClass instanceof DateClass) {
                // Date properties are indexed the same as document dates: formatted through IndexFields.dateToString()
                // and untokenized, to be able to sort by their values.
                addFieldToDocument(fieldFullName, getContentAsDate(baseObject, propertyName), Field.Store.YES,
                    Field.Index.NOT_ANALYZED, OBJECT_PROPERTY_BOOST, luceneDoc);
            } else if (propertyValue instanceof List) {
                // Handle list property values, by adding each list entry.
                List propertyListValues = (List) propertyValue;
                for (Object propertyListValue : propertyListValues) {
                    if (propertyListValue != null) {
                        addFieldToDocument(fieldFullName, propertyListValue.toString(), Field.Store.YES,
                            Field.Index.ANALYZED, OBJECT_PROPERTY_BOOST, luceneDoc);
                    }
                }
            } else {
                StringBuilder sb = new StringBuilder();
                getObjectContentAsText(sb, baseObject, propertyName, context);
                addFieldToDocument(fieldFullName, sb.toString(), Field.Store.YES, Field.Index.ANALYZED,
                    OBJECT_PROPERTY_BOOST, luceneDoc);
            }
        }
    }

    private void indexStaticList(Document luceneDoc, BaseObject baseObject, StaticListClass prop, String propertyName,
        XWikiContext context)
    {
        Map<String, ListItem> possibleValues = prop.getMap(context);
        String fieldFullName = baseObject.getClassName() + "." + propertyName;
{'fraction_non_alphanumeric': 0.08776259555769578, 'fraction_numerical': 0.002648528261000421, 'mean_word_length': 2.9851283281362435, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '383414', 'n_tokens_mistral': 4456, 'n_tokens_neox': 4280, 'n_words': 1237}