<commit_before>/*
 * Copyright 2012
 * Ubiquitous Knowledge Processing (UKP) Lab and FG Language Technology
 * Technische Universität Darmstadt
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.tudarmstadt.ukp.clarin.webanno.ui.curation.component.model;

import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.TypeUtil.getAdapter;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.getAddr;
import static org.apache.uima.fit.util.CasUtil.selectCovered;
import static org.apache.uima.fit.util.JCasUtil.selectCovered;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.uima.UIMAException;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.jcas.JCas;
import org.springframework.security.core.context.SecurityContextHolder;

import de.tudarmstadt.ukp.clarin.webanno.api.AnnotationService;
import de.tudarmstadt.ukp.clarin.webanno.api.RepositoryService;
import de.tudarmstadt.ukp.clarin.webanno.api.UserDao;
import de.tudarmstadt.ukp.clarin.webanno.api.WebAnnoConst;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.exception.AnnotationException;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.model.AnnotatorState;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocument;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
import de.tudarmstadt.ukp.clarin.webanno.model.Mode;
import de.tudarmstadt.ukp.clarin.webanno.model.Project;
import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
import de.tudarmstadt.ukp.clarin.webanno.model.User;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.component.CurationPanel;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.Configuration;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.ConfigurationSet;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.DiffResult;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.LinkCompareBehavior;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.MergeCas;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;

/**
 * This class is responsible for two things. Firstly, it creates a pre-merged cas, which contains
 * all annotations, where all annotators agree on. This is done by copying a random cas and removing
 * all differing annotations.
 *
 * Secondly, the class creates an instance of {@link CurationContainer}, which is the wicket model
 * for the curation panel. The {@link CurationContainer} contains the text for all sentences, which
 * are displayed at a specific page.
 *
 */
public class SuggestionBuilder
{
    private final AnnotationService annotationService;
    private final RepositoryService repository;
    private final UserDao userRepository;

    int diffRangeBegin, diffRangeEnd;
    boolean firstload = true;
    public static Map<Integer, Set<Integer>> crossSentenceLists;
    //
    Map<Integer, Integer> segmentBeginEnd = new HashMap<Integer, Integer>();

    public SuggestionBuilder(RepositoryService repository, AnnotationService aAnnotationService,
            UserDao aUserDao)
    {
        this.repository = repository;
        this.annotationService = aAnnotationService;
        userRepository = aUserDao;
    }

    public CurationContainer buildCurationContainer(AnnotatorState aBModel)
        throws UIMAException, ClassNotFoundException, IOException, AnnotationException
    {
        CurationContainer curationContainer = new CurationContainer();
        // initialize Variables
        SourceDocument sourceDocument = aBModel.getDocument();
        Map<Integer, Integer> segmentBeginEnd = new HashMap<Integer, Integer>();
        Map<Integer, Integer> segmentNumber = new HashMap<Integer, Integer>();
        Map<String, Map<Integer, Integer>> segmentAdress = new HashMap<String, Map<Integer, Integer>>();
        // get annotation documents

        List<AnnotationDocument> finishedAnnotationDocuments = new ArrayList<AnnotationDocument>();

        for (AnnotationDocument annotationDocument : repository.listAnnotationDocuments(aBModel
                .getDocument())) {
            if (annotationDocument.getState().equals(AnnotationDocumentState.FINISHED)) {
                finishedAnnotationDocuments.add(annotationDocument);
            }
        }

        Map<String, JCas> jCases = new HashMap<String, JCas>();

        AnnotationDocument randomAnnotationDocument = null;
        JCas mergeJCas;

        // get the correction/automation JCas for the logged in user
        if (aBModel.getMode().equals(Mode.AUTOMATION) || aBModel.getMode().equals(Mode.CORRECTION)) {
            jCases = listJcasesforCorrection(randomAnnotationDocument, sourceDocument,
                    aBModel.getMode());
            mergeJCas = getMergeCas(aBModel, sourceDocument, jCases, randomAnnotationDocument);
            String username = jCases.keySet().iterator().next();
            updateSegment(aBModel, segmentBeginEnd, segmentNumber, segmentAdress,
                    jCases.get(username), username, aBModel.getWindowBeginOffset(),
                    aBModel.getWindowEndOffset());
        }
        else {
            jCases = listJcasesforCuration(finishedAnnotationDocuments, randomAnnotationDocument,
                    aBModel.getMode());
            mergeJCas = getMergeCas(aBModel, sourceDocument, jCases, randomAnnotationDocument);
            updateSegment(aBModel, segmentBeginEnd, segmentNumber, segmentAdress, mergeJCas,
                    CurationPanel.CURATION_USER, 0, mergeJCas.getDocumentText().length());

        }

        List<Type> entryTypes = null;

        segmentAdress.put(CurationPanel.CURATION_USER, new HashMap<Integer, Integer>());
        for (Sentence sentence : selectCovered(mergeJCas, Sentence.class, diffRangeBegin, diffRangeEnd)) {
            segmentAdress.get(CurationPanel.CURATION_USER).put(sentence.getBegin(),
                    getAddr(sentence));
        }

        if (entryTypes == null) {
            entryTypes = getEntryTypes(mergeJCas, aBModel.getAnnotationLayers(), annotationService);
        }

        // for cross-sentences annotation, update the end of the segment<commit_msg>#476 - Sentence number in sidebar incorrect if document does not start with a sentence - Start the curation process at the first sentence offset, not at 0.<commit_after>/*
 * Copyright 2012
 * Ubiquitous Knowledge Processing (UKP) Lab and FG Language Technology
 * Technische Universität Darmstadt
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.tudarmstadt.ukp.clarin.webanno.ui.curation.component.model;

import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.TypeUtil.getAdapter;
import static de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil.getAddr;
import static org.apache.uima.fit.util.CasUtil.selectCovered;
import static org.apache.uima.fit.util.JCasUtil.selectCovered;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.uima.UIMAException;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.jcas.JCas;
import org.springframework.security.core.context.SecurityContextHolder;

import de.tudarmstadt.ukp.clarin.webanno.api.AnnotationService;
import de.tudarmstadt.ukp.clarin.webanno.api.RepositoryService;
import de.tudarmstadt.ukp.clarin.webanno.api.UserDao;
import de.tudarmstadt.ukp.clarin.webanno.api.WebAnnoConst;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.exception.AnnotationException;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.model.AnnotatorState;
import de.tudarmstadt.ukp.clarin.webanno.api.annotation.util.WebAnnoCasUtil;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocument;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationDocumentState;
import de.tudarmstadt.ukp.clarin.webanno.model.AnnotationLayer;
import de.tudarmstadt.ukp.clarin.webanno.model.Mode;
import de.tudarmstadt.ukp.clarin.webanno.model.Project;
import de.tudarmstadt.ukp.clarin.webanno.model.SourceDocument;
import de.tudarmstadt.ukp.clarin.webanno.model.User;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.component.CurationPanel;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.Configuration;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.ConfigurationSet;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.DiffResult;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.CasDiff2.LinkCompareBehavior;
import de.tudarmstadt.ukp.clarin.webanno.ui.curation.service.MergeCas;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token;

/**
 * This class is responsible for two things. Firstly, it creates a pre-merged cas, which contains
 * all annotations, where all annotators agree on. This is done by copying a random cas and removing
 * all differing annotations.
 *
 * Secondly, the class creates an instance of {@link CurationContainer}, which is the wicket model
 * for the curation panel. The {@link CurationContainer} contains the text for all sentences, which
 * are displayed at a specific page.
 *
 */
public class SuggestionBuilder
{
    private final AnnotationService annotationService;
    private final RepositoryService repository;
    private final UserDao userRepository;

    int diffRangeBegin, diffRangeEnd;
    boolean firstload = true;
    public static Map<Integer, Set<Integer>> crossSentenceLists;
    //
    Map<Integer, Integer> segmentBeginEnd = new HashMap<Integer, Integer>();

    public SuggestionBuilder(RepositoryService repository, AnnotationService aAnnotationService,
            UserDao aUserDao)
    {
        this.repository = repository;
        this.annotationService = aAnnotationService;
        userRepository = aUserDao;
    }

    public CurationContainer buildCurationContainer(AnnotatorState aBModel)
        throws UIMAException, ClassNotFoundException, IOException, AnnotationException
    {
        CurationContainer curationContainer = new CurationContainer();
        // initialize Variables
        SourceDocument sourceDocument = aBModel.getDocument();
        Map<Integer, Integer> segmentBeginEnd = new HashMap<Integer, Integer>();
        Map<Integer, Integer> segmentNumber = new HashMap<Integer, Integer>();
        Map<String, Map<Integer, Integer>> segmentAdress = new HashMap<String, Map<Integer, Integer>>();
        // get annotation documents

        List<AnnotationDocument> finishedAnnotationDocuments = new ArrayList<AnnotationDocument>();

        for (AnnotationDocument annotationDocument : repository.listAnnotationDocuments(aBModel
                .getDocument())) {
            if (annotationDocument.getState().equals(AnnotationDocumentState.FINISHED)) {
                finishedAnnotationDocuments.add(annotationDocument);
            }
        }

        Map<String, JCas> jCases = new HashMap<String, JCas>();

        AnnotationDocument randomAnnotationDocument = null;
        JCas mergeJCas;

        // get the correction/automation JCas for the logged in user
        if (aBModel.getMode().equals(Mode.AUTOMATION) || aBModel.getMode().equals(Mode.CORRECTION)) {
            jCases = listJcasesforCorrection(randomAnnotationDocument, sourceDocument,
                    aBModel.getMode());
            mergeJCas = getMergeCas(aBModel, sourceDocument, jCases, randomAnnotationDocument);
            String username = jCases.keySet().iterator().next();
            updateSegment(aBModel, segmentBeginEnd, segmentNumber, segmentAdress,
                    jCases.get(username), username, aBModel.getWindowBeginOffset(),
                    aBModel.getWindowEndOffset());
        }
        else {
            jCases = listJcasesforCuration(finishedAnnotationDocuments, randomAnnotationDocument,
                    aBModel.getMode());
            mergeJCas = getMergeCas(aBModel, sourceDocument, jCases, randomAnnotationDocument);
            updateSegment(aBModel, segmentBeginEnd, segmentNumber, segmentAdress, mergeJCas,
                    CurationPanel.CURATION_USER,
                    WebAnnoCasUtil.getFirstSentence(mergeJCas).getBegin(),
                    mergeJCas.getDocumentText().length());

        }

        List<Type> entryTypes = null;

        segmentAdress.put(CurationPanel.CURATION_USER, new HashMap<Integer, Integer>());
        for (Sentence sentence : selectCovered(mergeJCas, Sentence.class, diffRangeBegin, diffRangeEnd)) {
            segmentAdress.get(CurationPanel.CURATION_USER).put(sentence.getBegin(),
                    getAddr(sentence));
        }

        if (entryTypes == null) {
            entryTypes = getEntryTypes(mergeJCas, aBModel.getAnnotationLayers(), annotationService);
        }

        // for cross-sentences annotation, update the end of the segment
{'fraction_non_alphanumeric': 0.0917725843925106, 'fraction_numerical': 0.002118354516878502, 'mean_word_length': 3.9492729117348664, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3442969', 'n_tokens_mistral': 4276, 'n_tokens_neox': 4117, 'n_words': 902}