<commit_before>package uk.ac.ebi.ddi.annotation.service.dataset;

import com.mongodb.BasicDBObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.aggregation.*;
import org.springframework.data.mongodb.core.query.Criteria;
import uk.ac.ebi.ddi.annotation.utils.Constants;
import uk.ac.ebi.ddi.annotation.utils.DatasetUtils;
import uk.ac.ebi.ddi.annotation.utils.Utils;
import uk.ac.ebi.ddi.service.db.model.dataset.Dataset;
import uk.ac.ebi.ddi.service.db.model.dataset.DatasetSimilars;
import uk.ac.ebi.ddi.service.db.model.dataset.DatasetStatus;
import uk.ac.ebi.ddi.service.db.model.dataset.SimilarDataset;
import uk.ac.ebi.ddi.service.db.model.publication.PublicationDataset;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetService;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetSimilarsService;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetStatusService;
import uk.ac.ebi.ddi.service.db.service.logger.IHttpEventService;
import uk.ac.ebi.ddi.service.db.service.publication.IPublicationDatasetService;
import uk.ac.ebi.ddi.service.db.utils.DatasetCategory;
import uk.ac.ebi.ddi.service.db.utils.DatasetSimilarsType;
import uk.ac.ebi.ddi.xml.validator.parser.model.Entry;
import uk.ac.ebi.ddi.xml.validator.utils.Field;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author <NAME> (<EMAIL>)
 * @date 05/05/2016
 */
public class DDIDatasetAnnotationService {

    @Autowired
    IDatasetService datasetService;

    @Autowired
    IDatasetStatusService statusService;

    @Autowired
    IPublicationDatasetService publicationService;

    @Autowired
    IDatasetSimilarsService similarsService;

    @Autowired
    IHttpEventService httpEventService;

    private static final Logger LOGGER = LoggerFactory.getLogger(DDIDatasetAnnotationService.class);

    /**
     * This function looks for individual datasets and check if they are in the database and if they needs to
     * be updated.
     *
     * @param dataset
     */
    @Deprecated
    public void insertDataset(Entry dataset) {
        Dataset dbDataset = DatasetUtils.transformEntryDataset(dataset);
        dbDataset = Utils.replaceTextCase(dbDataset);
        Dataset currentDataset = datasetService.read(dbDataset.getAccession(), dbDataset.getDatabase());
        if (currentDataset == null) {
            insertDataset(dbDataset);
        } else if (currentDataset.getInitHashCode() != dbDataset.getInitHashCode()) {
            updateDataset(currentDataset, dbDataset);
        }
    }

    /**
     * THis insert use a fixed database Name and not the one provided by the user
     * @param dataset Dataset Entry from the XML
     * @param databaseName database name provided by the users
     */
    public void insertDataset(Entry dataset, String databaseName) {
        Dataset dbDataset = DatasetUtils.transformEntryDataset(dataset, databaseName);
        dbDataset = Utils.replaceTextCase(dbDataset);
        Dataset currentDataset = datasetService.read(dbDataset.getAccession(), dbDataset.getDatabase());
        if (currentDataset == null) {
            insertDataset(dbDataset);
        } else if (currentDataset.getInitHashCode() != dbDataset.getInitHashCode()) {
            updateDataset(currentDataset, dbDataset);
        }
    }

    private void updateDataset(Dataset currentDataset, Dataset dbDataset) {
        dbDataset = datasetService.update(currentDataset.getId(), dbDataset);
        if (dbDataset.getId() != null) {
            statusService.save(new DatasetStatus(dbDataset.getAccession(), dbDataset.getDatabase(),
                    dbDataset.getInitHashCode(), getDate(), DatasetCategory.INSERTED.getType()));
        }
    }

    public void annotateDataset(Dataset exitingDataset) {<commit_msg>added logger to get to know dates<commit_after>package uk.ac.ebi.ddi.annotation.service.dataset;

import com.mongodb.BasicDBObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.mongodb.core.aggregation.*;
import org.springframework.data.mongodb.core.query.Criteria;
import uk.ac.ebi.ddi.annotation.utils.Constants;
import uk.ac.ebi.ddi.annotation.utils.DatasetUtils;
import uk.ac.ebi.ddi.annotation.utils.Utils;
import uk.ac.ebi.ddi.service.db.model.dataset.Dataset;
import uk.ac.ebi.ddi.service.db.model.dataset.DatasetSimilars;
import uk.ac.ebi.ddi.service.db.model.dataset.DatasetStatus;
import uk.ac.ebi.ddi.service.db.model.dataset.SimilarDataset;
import uk.ac.ebi.ddi.service.db.model.publication.PublicationDataset;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetService;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetSimilarsService;
import uk.ac.ebi.ddi.service.db.service.dataset.IDatasetStatusService;
import uk.ac.ebi.ddi.service.db.service.logger.IHttpEventService;
import uk.ac.ebi.ddi.service.db.service.publication.IPublicationDatasetService;
import uk.ac.ebi.ddi.service.db.utils.DatasetCategory;
import uk.ac.ebi.ddi.service.db.utils.DatasetSimilarsType;
import uk.ac.ebi.ddi.xml.validator.parser.model.Entry;
import uk.ac.ebi.ddi.xml.validator.utils.Field;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

/**
 * @author <NAME> (<EMAIL>)
 * @date 05/05/2016
 */
public class DDIDatasetAnnotationService {

    @Autowired
    IDatasetService datasetService;

    @Autowired
    IDatasetStatusService statusService;

    @Autowired
    IPublicationDatasetService publicationService;

    @Autowired
    IDatasetSimilarsService similarsService;

    @Autowired
    IHttpEventService httpEventService;

    private static final Logger LOGGER = LoggerFactory.getLogger(DDIDatasetAnnotationService.class);

    /**
     * This function looks for individual datasets and check if they are in the database and if they needs to
     * be updated.
     *
     * @param dataset
     */
    @Deprecated
    public void insertDataset(Entry dataset) {
        Dataset dbDataset = DatasetUtils.transformEntryDataset(dataset);
        dbDataset = Utils.replaceTextCase(dbDataset);
        Dataset currentDataset = datasetService.read(dbDataset.getAccession(), dbDataset.getDatabase());
        /*LOGGER.info("dates during insertion of "+ currentDataset.getAccession()+"are " +
                currentDataset.getDates().keySet()
                .stream().map(dt -> {
            LOGGER.info("dates during insertion of " + currentDataset.getAccession() +
                    currentDataset.getDates().get(dt).iterator().next());
            return currentDataset.getDates().get(dt);
        }));*/
        if (currentDataset == null) {
            insertDataset(dbDataset);
        } else if (currentDataset.getInitHashCode() != dbDataset.getInitHashCode()) {
            updateDataset(currentDataset, dbDataset);
        }
    }

    /**
     * THis insert use a fixed database Name and not the one provided by the user
     * @param dataset Dataset Entry from the XML
     * @param databaseName database name provided by the users
     */
    public void insertDataset(Entry dataset, String databaseName) {
        Dataset dbDataset = DatasetUtils.transformEntryDataset(dataset, databaseName);
        dbDataset = Utils.replaceTextCase(dbDataset);
        Dataset currentDataset = datasetService.read(dbDataset.getAccession(), dbDataset.getDatabase());

        if (currentDataset != null) {
            for (String date : currentDataset.getDates().keySet()) {
                LOGGER.info("dates during insertion of " + currentDataset.getAccession() + "with key" + date
                        + "are " + currentDataset.getDates().get(date));
            }
            if (!currentDataset.getDates().isEmpty()) {
                LOGGER.info("dates of " + currentDataset.getId() + "are " +
                        currentDataset.getDates().get("publication").toString());


                currentDataset.getDates().keySet()
                        .stream().map(dt -> {
                    LOGGER.info("dates during insertion of " + currentDataset.getAccession() +
                            currentDataset.getDates().get(dt).iterator().next());
                    return currentDataset.getDates().get(dt);
                });
            }
        }
        if (currentDataset == null) {
            LOGGER.info("inserting dataset as dataset is not available");
            LOGGER.info("dataset is " + dbDataset.toString());
            insertDataset(dbDataset);
        } else if (currentDataset.getInitHashCode() != dbDataset.getInitHashCode() ||
                (currentDataset.getDates().get("publication").iterator().next() !=
                        dbDataset.getDates().get("publication").iterator().next())) {
            updateDataset(currentDataset, dbDataset);
        }
    }

    private void updateDataset(Dataset currentDataset, Dataset dbDataset) {
        dbDataset = datasetService.update(currentDataset.getId(), dbDataset);
        if (dbDataset.getId() != null) {
            statusService.save(new DatasetStatus(dbDataset.getAccession(), dbDataset.getDatabase(),
                    dbDataset.getInitHashCode(), getDate(), DatasetCategory.INSERTED.getType()));
        }
    }

    public void annotateDataset(Dataset exitingDataset) {
{'fraction_non_alphanumeric': 0.11129568106312292, 'fraction_numerical': 0.0020764119601328905, 'mean_word_length': 3.3588235294117648, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1797742', 'n_tokens_mistral': 2796, 'n_tokens_neox': 2898, 'n_words': 499}