<commit_before>package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser;
import edu.harvard.iq.dataverse.api.SearchFields;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.search.IndexableDataset;
import edu.harvard.iq.dataverse.search.IndexableObject;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.io.IOException;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.inject.Named;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;

@Stateless
@Named
public class IndexServiceBean {

    private static final Logger logger = Logger.getLogger(IndexServiceBean.class.getCanonicalName());

    @EJB
    DataverseServiceBean dataverseService;
    @EJB
    DatasetServiceBean datasetService;
    @EJB
    BuiltinUserServiceBean dataverseUserServiceBean;
    @EJB
    PermissionServiceBean permissionService;
    @EJB
    AuthenticationServiceBean userServiceBean;

    private final String solrDocIdentifierDataverse = "dataverse_";
    public static final String solrDocIdentifierFile = "datafile_";
    public static final String solrDocIdentifierDataset = "dataset_";
    public static final String draftSuffix = "_draft";
    public static final String deaccessionedSuffix = "_deaccessioned";
    private static final String groupPrefix = "group_";
    private static final String groupPerUserPrefix = "group_user";
    private static final Long publicGroupId = 1L;
    private static final String publicGroupString = groupPrefix + "public";
    /**
     * @todo: remove this fake "has access to all data" group
     */
    private static final Long tmpNsaGroupId = 2L;
    private static final String PUBLISHED_STRING = "Published";
    private static final String UNPUBLISHED_STRING = "Unpublished";
    private static final String DRAFT_STRING = "Draft";
    private static final String DEACCESSIONED_STRING = "Deaccessioned";
    private Dataverse rootDataverseCached;

    public String indexAll() {
        /**
         * @todo allow for configuration of hostname and port
         */
        SolrServer server = new HttpSolrServer("http://localhost:8983/solr/");
        logger.info("deleting all Solr documents before a complete re-index");
        try {
            server.deleteByQuery("*:*");// CAUTION: deletes everything!
        } catch (SolrServerException | IOException ex) {
            return ex.toString();
        }
        try {
            server.commit();
        } catch (SolrServerException | IOException ex) {
            return ex.toString();
        }

        /**
         * @todo: replace hard-coded groups with real groups
         */
        Map<Long, String> groups = new HashMap<>();
        groups.put(publicGroupId, publicGroupString);
        groups.put(tmpNsaGroupId, "nsa");
        groups.put(tmpNsaGroupId + 1, "flappybird");
        groups.put(tmpNsaGroupId + 2, "2048");

        int groupIndexCount = 0;
        for (Map.Entry<Long, String> group : groups.entrySet()) {
            groupIndexCount++;
            logger.info("indexing group " + groupIndexCount + " of " + groups.size() + ": " + indexGroup(group));
        }

        int userIndexCount = 0;
        for (AuthenticatedUser user : userServiceBean.findAllAuthenticatedUsers()) {
            userIndexCount++;
            logger.info("indexing user " + userIndexCount + " of several: " + indexUser(user));
        }
        List<Dataverse> dataverses = dataverseService.findAll();
        int dataverseIndexCount = 0;
        for (Dataverse dataverse : dataverses) {
            logger.info("indexing dataverse " + dataverseIndexCount + " of " + dataverses.size() + ": " + indexDataverse(dataverse));
            dataverseIndexCount++;
        }

        int datasetIndexCount = 0;
        List<Dataset> datasets = datasetService.findAll();
        for (Dataset dataset : datasets) {
            datasetIndexCount++;
            logger.info("indexing dataset " + datasetIndexCount + " of " + datasets.size() + ": " + indexDataset(dataset));
        }
//        logger.info("advanced search fields: " + advancedSearchFields);
//        logger.info("not advanced search fields: " + notAdvancedSearchFields);
        logger.info("done iterating through all datasets");

        return dataverseIndexCount + " dataverses, " + datasetIndexCount + " datasets, " + groupIndexCount + " groups, and " + userIndexCount + " users indexed\n";
    }

    public String indexDataverse(Dataverse dataverse) {
        logger.info("indexDataverse called on dataverse id " + dataverse.getId() + "(" + dataverse.getAlias() + ")");
        if (dataverse.getId() == null) {
            String msg = "unable to index dataverse. id was null (alias: " + dataverse.getAlias() + ")";
            logger.info(msg);
            return msg;
        }
        Dataverse rootDataverse = findRootDataverseCached();
        if (dataverse.getId() == rootDataverse.getId()) {
            /**
             * @todo: replace hard-coded groups with real groups
             */
            Map<Long, String> groups = new HashMap<>();
            groups.put(publicGroupId, publicGroupString);
            groups.put(tmpNsaGroupId, "nsa");
            groups.put(tmpNsaGroupId + 1, "flappybird");
            groups.put(tmpNsaGroupId + 2, "2048");

            int groupIndexCount = 0;
            for (Map.Entry<Long, String> group : groups.entrySet()) {
                groupIndexCount++;
                logger.info("indexing group " + groupIndexCount + " of " + groups.size() + ": " + indexGroup(group));
            }
            return "The root dataverse shoud not be indexed. Indexed temporary groups instead.";
        }
        Collection<SolrInputDocument> docs = new ArrayList<>();
        SolrInputDocument solrInputDocument = new SolrInputDocument();
        solrInputDocument.addField(SearchFields.ID, solrDocIdentifierDataverse + dataverse.getId());
        solrInputDocument.addField(SearchFields.ENTITY_ID, dataverse.getId());
        solrInputDocument.addField(SearchFields.TYPE, "dataverses");
        solrInputDocument.addField(SearchFields.NAME, dataverse.getName());
        solrInputDocument.addField(SearchFields.NAME_SORT, dataverse.getName());
        solrInputDocument.addField(SearchFields.DATAVERSE_NAME, dataverse.getName());
        if (dataverse.isReleased()) {
            solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING);
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getPublicationDate());
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE_SEARCHABLE_TEXT, convertToFriendlyDate(dataverse.getPublicationDate()));
            solrInputDocument.addField(SearchFields.PERMS, publicGroupString);
        } else if (dataverse.getCreator() != null) { //@todo: do we need this check still
            solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getCreateDate());
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE_SEARCHABLE_TEXT, convertToFriendlyDate(dataverse.getCreateDate()));
        }

        /**
         * @todo check if you are at a permission root. If not, go up the
         * dataverse hierarchy until you reach a permission root (may be the
         * root dataverse).
         *
         * Below are approaches toward going beyond non-creator indexing
         * permissions for https://github.com/IQSS/dataverse/issues/734
         */
        // this is the dumb way... iterate through every user each time
//        for (DataverseUser user : dataverseUserServiceBean.findAll()) {
//            if (permissionService.isUserAllowedOn(user, UpdateDataverseCommand.class, dataverse)) {
//                solrInputDocument.addField(SearchFields.PERMS, groupPerUserPrefix + user.getId());
//            }
//        }
        // this should be the more performant way... given a dataverse, figure out who has the access in question
        List<RoleAssignment> assignmentsOn = permissionService.assignmentsOn(dataverse);
        for (RoleAssignment roleAssignment : assignmentsOn) {<commit_msg>show dates and unpublished label on dataverses #909<commit_after>package edu.harvard.iq.dataverse;

import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUserServiceBean;
import edu.harvard.iq.dataverse.authorization.providers.builtin.BuiltinUser;
import edu.harvard.iq.dataverse.api.SearchFields;
import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.User;
import edu.harvard.iq.dataverse.datavariable.DataVariable;
import edu.harvard.iq.dataverse.search.IndexableDataset;
import edu.harvard.iq.dataverse.search.IndexableObject;
import edu.harvard.iq.dataverse.util.FileUtil;
import java.io.IOException;
import java.sql.Timestamp;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import javax.ejb.EJB;
import javax.ejb.Stateless;
import javax.inject.Named;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;

@Stateless
@Named
public class IndexServiceBean {

    private static final Logger logger = Logger.getLogger(IndexServiceBean.class.getCanonicalName());

    @EJB
    DataverseServiceBean dataverseService;
    @EJB
    DatasetServiceBean datasetService;
    @EJB
    BuiltinUserServiceBean dataverseUserServiceBean;
    @EJB
    PermissionServiceBean permissionService;
    @EJB
    AuthenticationServiceBean userServiceBean;

    private final String solrDocIdentifierDataverse = "dataverse_";
    public static final String solrDocIdentifierFile = "datafile_";
    public static final String solrDocIdentifierDataset = "dataset_";
    public static final String draftSuffix = "_draft";
    public static final String deaccessionedSuffix = "_deaccessioned";
    private static final String groupPrefix = "group_";
    private static final String groupPerUserPrefix = "group_user";
    private static final Long publicGroupId = 1L;
    private static final String publicGroupString = groupPrefix + "public";
    /**
     * @todo: remove this fake "has access to all data" group
     */
    private static final Long tmpNsaGroupId = 2L;
    private static final String PUBLISHED_STRING = "Published";
    private static final String UNPUBLISHED_STRING = "Unpublished";
    private static final String DRAFT_STRING = "Draft";
    private static final String DEACCESSIONED_STRING = "Deaccessioned";
    private Dataverse rootDataverseCached;

    public String indexAll() {
        /**
         * @todo allow for configuration of hostname and port
         */
        SolrServer server = new HttpSolrServer("http://localhost:8983/solr/");
        logger.info("deleting all Solr documents before a complete re-index");
        try {
            server.deleteByQuery("*:*");// CAUTION: deletes everything!
        } catch (SolrServerException | IOException ex) {
            return ex.toString();
        }
        try {
            server.commit();
        } catch (SolrServerException | IOException ex) {
            return ex.toString();
        }

        /**
         * @todo: replace hard-coded groups with real groups
         */
        Map<Long, String> groups = new HashMap<>();
        groups.put(publicGroupId, publicGroupString);
        groups.put(tmpNsaGroupId, "nsa");
        groups.put(tmpNsaGroupId + 1, "flappybird");
        groups.put(tmpNsaGroupId + 2, "2048");

        int groupIndexCount = 0;
        for (Map.Entry<Long, String> group : groups.entrySet()) {
            groupIndexCount++;
            logger.info("indexing group " + groupIndexCount + " of " + groups.size() + ": " + indexGroup(group));
        }

        int userIndexCount = 0;
        for (AuthenticatedUser user : userServiceBean.findAllAuthenticatedUsers()) {
            userIndexCount++;
            logger.info("indexing user " + userIndexCount + " of several: " + indexUser(user));
        }
        List<Dataverse> dataverses = dataverseService.findAll();
        int dataverseIndexCount = 0;
        for (Dataverse dataverse : dataverses) {
            logger.info("indexing dataverse " + dataverseIndexCount + " of " + dataverses.size() + ": " + indexDataverse(dataverse));
            dataverseIndexCount++;
        }

        int datasetIndexCount = 0;
        List<Dataset> datasets = datasetService.findAll();
        for (Dataset dataset : datasets) {
            datasetIndexCount++;
            logger.info("indexing dataset " + datasetIndexCount + " of " + datasets.size() + ": " + indexDataset(dataset));
        }
//        logger.info("advanced search fields: " + advancedSearchFields);
//        logger.info("not advanced search fields: " + notAdvancedSearchFields);
        logger.info("done iterating through all datasets");

        return dataverseIndexCount + " dataverses, " + datasetIndexCount + " datasets, " + groupIndexCount + " groups, and " + userIndexCount + " users indexed\n";
    }

    public String indexDataverse(Dataverse dataverse) {
        logger.info("indexDataverse called on dataverse id " + dataverse.getId() + "(" + dataverse.getAlias() + ")");
        if (dataverse.getId() == null) {
            String msg = "unable to index dataverse. id was null (alias: " + dataverse.getAlias() + ")";
            logger.info(msg);
            return msg;
        }
        Dataverse rootDataverse = findRootDataverseCached();
        if (dataverse.getId() == rootDataverse.getId()) {
            /**
             * @todo: replace hard-coded groups with real groups
             */
            Map<Long, String> groups = new HashMap<>();
            groups.put(publicGroupId, publicGroupString);
            groups.put(tmpNsaGroupId, "nsa");
            groups.put(tmpNsaGroupId + 1, "flappybird");
            groups.put(tmpNsaGroupId + 2, "2048");

            int groupIndexCount = 0;
            for (Map.Entry<Long, String> group : groups.entrySet()) {
                groupIndexCount++;
                logger.info("indexing group " + groupIndexCount + " of " + groups.size() + ": " + indexGroup(group));
            }
            return "The root dataverse shoud not be indexed. Indexed temporary groups instead.";
        }
        Collection<SolrInputDocument> docs = new ArrayList<>();
        SolrInputDocument solrInputDocument = new SolrInputDocument();
        solrInputDocument.addField(SearchFields.ID, solrDocIdentifierDataverse + dataverse.getId());
        solrInputDocument.addField(SearchFields.ENTITY_ID, dataverse.getId());
        solrInputDocument.addField(SearchFields.TYPE, "dataverses");
        solrInputDocument.addField(SearchFields.NAME, dataverse.getName());
        solrInputDocument.addField(SearchFields.NAME_SORT, dataverse.getName());
        solrInputDocument.addField(SearchFields.DATAVERSE_NAME, dataverse.getName());
        if (dataverse.isReleased()) {
            solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, PUBLISHED_STRING);
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getPublicationDate());
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE_SEARCHABLE_TEXT, convertToFriendlyDate(dataverse.getPublicationDate()));
            solrInputDocument.addField(SearchFields.PERMS, publicGroupString);
        } else {
            solrInputDocument.addField(SearchFields.PUBLICATION_STATUS, UNPUBLISHED_STRING);
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE, dataverse.getCreateDate());
            solrInputDocument.addField(SearchFields.RELEASE_OR_CREATE_DATE_SEARCHABLE_TEXT, convertToFriendlyDate(dataverse.getCreateDate()));
        }

        /**
         * @todo check if you are at a permission root. If not, go up the
         * dataverse hierarchy until you reach a permission root (may be the
         * root dataverse).
         *
         * Below are approaches toward going beyond non-creator indexing
         * permissions for https://github.com/IQSS/dataverse/issues/734
         */
        // this is the dumb way... iterate through every user each time
//        for (DataverseUser user : dataverseUserServiceBean.findAll()) {
//            if (permissionService.isUserAllowedOn(user, UpdateDataverseCommand.class, dataverse)) {
//                solrInputDocument.addField(SearchFields.PERMS, groupPerUserPrefix + user.getId());
//            }
//        }
        // this should be the more performant way... given a dataverse, figure out who has the access in question
        List<RoleAssignment> assignmentsOn = permissionService.assignmentsOn(dataverse);
        for (RoleAssignment roleAssignment : assignmentsOn) {
{'fraction_non_alphanumeric': 0.10226729676631445, 'fraction_numerical': 0.00292040567089683, 'mean_word_length': 3.2361673414304994, 'pattern_counts': {'":': 8, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5394281', 'n_tokens_mistral': 5327, 'n_tokens_neox': 5195, 'n_words': 1139}