<commit_before>/*
 * Copyright 2011-2017 B2i Healthcare Pte Ltd, http://b2i.sg
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.b2international.snowowl.snomed.api.impl.traceability;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Sets.newHashSet;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Set;

import org.eclipse.emf.cdo.CDOObject;
import org.eclipse.emf.cdo.common.revision.delta.CDOFeatureDelta;
import org.eclipse.emf.cdo.common.revision.delta.CDORevisionDelta;
import org.eclipse.emf.cdo.common.revision.delta.CDOSetFeatureDelta;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EStructuralFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.b2international.commons.http.ExtendedLocale;
import com.b2international.index.revision.RevisionIndex;
import com.b2international.index.revision.RevisionIndexRead;
import com.b2international.index.revision.RevisionSearcher;
import com.b2international.snowowl.core.ApplicationContext;
import com.b2international.snowowl.core.RepositoryManager;
import com.b2international.snowowl.core.api.IBranchPath;
import com.b2international.snowowl.core.api.SnowowlRuntimeException;
import com.b2international.snowowl.core.api.SnowowlServiceException;
import com.b2international.snowowl.core.events.metrics.MetricsThreadLocal;
import com.b2international.snowowl.core.events.metrics.Timer;
import com.b2international.snowowl.core.ft.FeatureToggles;
import com.b2international.snowowl.core.ft.Features;
import com.b2international.snowowl.datastore.ICDOChangeProcessor;
import com.b2international.snowowl.datastore.ICDOCommitChangeSet;
import com.b2international.snowowl.datastore.cdo.CDOIDUtils;
import com.b2international.snowowl.datastore.index.ImmutableIndexCommitChangeSet;
import com.b2international.snowowl.datastore.index.IndexCommitChangeSet;
import com.b2international.snowowl.eventbus.IEventBus;
import com.b2international.snowowl.identity.domain.User;
import com.b2international.snowowl.snomed.Component;
import com.b2international.snowowl.snomed.Concept;
import com.b2international.snowowl.snomed.Description;
import com.b2international.snowowl.snomed.Relationship;
import com.b2international.snowowl.snomed.SnomedConstants;
import com.b2international.snowowl.snomed.SnomedConstants.Concepts;
import com.b2international.snowowl.snomed.SnomedPackage;
import com.b2international.snowowl.snomed.api.browser.ISnomedBrowserAxiomService;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserConcept;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserDescription;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserRelationship;
import com.b2international.snowowl.snomed.api.domain.browser.SnomedBrowserDescriptionType;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserConcept;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserDescription;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationship;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationshipTarget;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationshipType;
import com.b2international.snowowl.snomed.common.ContentSubType;
import com.b2international.snowowl.snomed.common.SnomedTerminologyComponentConstants;
import com.b2international.snowowl.snomed.core.domain.SnomedConcept;
import com.b2international.snowowl.snomed.core.domain.SnomedConcepts;
import com.b2international.snowowl.snomed.core.domain.SnomedDescription;
import com.b2international.snowowl.snomed.core.domain.SnomedDescriptions;
import com.b2international.snowowl.snomed.core.domain.SnomedRelationship;
import com.b2international.snowowl.snomed.core.domain.SnomedRelationships;
import com.b2international.snowowl.snomed.core.lang.LanguageSetting;
import com.b2international.snowowl.snomed.datastore.SnomedDatastoreActivator;
import com.b2international.snowowl.snomed.datastore.SnomedDescriptionLookupService;
import com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration;
import com.b2international.snowowl.snomed.datastore.id.SnomedIdentifiers;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedConceptDocument;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedDescriptionIndexEntry;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRefSetMemberIndexEntry;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRelationshipIndexEntry;
import com.b2international.snowowl.snomed.datastore.request.SnomedRequests;
import com.b2international.snowowl.snomed.snomedrefset.SnomedAnnotationRefSetMember;
import com.b2international.snowowl.snomed.snomedrefset.SnomedRefSetMember;
import com.b2international.snowowl.snomed.snomedrefset.SnomedRefSetPackage;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;

/**
 * Change processor implementation that produces a log entry for committed transactions.
 */
@SuppressWarnings("deprecation")
public class SnomedTraceabilityChangeProcessor implements ICDOChangeProcessor {

	private static final Logger TRACE_LOGGER = LoggerFactory.getLogger("traceability");
	private static final Logger SYS_LOGGER = LoggerFactory.getLogger(SnomedTraceabilityChangeProcessor.class);

	private static final String OWL_AXIOM = "OwlAxiom";

	private static final Set<EClass> MRCM_REFSET_MEMBER_TYPES = ImmutableSet.<EClass>of(
			SnomedRefSetPackage.Literals.SNOMED_MRCM_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_RANGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_MODULE_SCOPE_REF_SET_MEMBER);
	
	private static final Set<EClass> DETACHED_REFSET_MEMBER_TYPES = ImmutableSet.<EClass>of(
			SnomedRefSetPackage.Literals.SNOMED_ANNOTATION_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_LANGUAGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_ASSOCIATION_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_ATTRIBUTE_VALUE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_RANGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_MODULE_SCOPE_REF_SET_MEMBER);
	
	private static final Set<EStructuralFeature> IGNORED_FEATURES = ImmutableSet.<EStructuralFeature>of(
			SnomedPackage.Literals.CONCEPT__DESCRIPTIONS,
			SnomedPackage.Literals.CONCEPT__OUTBOUND_RELATIONSHIPS,
			SnomedPackage.Literals.CONCEPT__FULLY_SPECIFIED_NAME,
			SnomedPackage.Literals.RELATIONSHIP__REFINABILITY_REF_SET_MEMBERS);
	
	private TraceabilityEntry entry;

	private final boolean collectSystemChanges;
	private final RevisionIndex index;
	private final IBranchPath branchPath;
	private ISnomedBrowserAxiomService axiomService;

	private ICDOCommitChangeSet commitChangeSet;
	private ObjectWriter objectWriter;
	private SnomedDescriptionLookupService descriptionLookupService;

	public SnomedTraceabilityChangeProcessor(final IBranchPath branchPath, ObjectWriter objectWriter) {
		this.branchPath = checkNotNull(branchPath);
		this.objectWriter = checkNotNull(objectWriter);
		this.index = ApplicationContext.getServiceForClass(RepositoryManager.class).get(SnomedDatastoreActivator.REPOSITORY_UUID).service(RevisionIndex.class);
		this.collectSystemChanges = collectSystemChanges(branchPath.getPath());
		this.descriptionLookupService = new SnomedDescriptionLookupService();
	}

	@Override
	public void process(ICDOCommitChangeSet commitChangeSet) throws SnowowlServiceException {
		
		final Timer traceabilityTimer = MetricsThreadLocal.get().timer("traceability");
		
		try {
			
			traceabilityTimer.start();
			
			this.entry = new TraceabilityEntry(commitChangeSet);
			
			if (isSystemCommit(commitChangeSet.getUserId()) && !collectSystemChanges) {
				return;
			}
			
			this.commitChangeSet = commitChangeSet;
			
			processNewComponents(commitChangeSet.getNewComponents());
			processComponentUpdates(commitChangeSet.getDirtyComponents());
			processComponentDeletions(commitChangeSet);
			
		} finally {
			traceabilityTimer.stop();
		}
	}

	@Override
	public IndexCommitChangeSet commit() throws SnowowlServiceException {
		SYS_LOGGER.info("Collected {} traceability entries on branch {}", entry.getChanges().size(), branchPath.getPath());
		return ImmutableIndexCommitChangeSet.builder().build();
	}

	@Override
	public void afterCommit() {
		
		final Timer traceabilityTimer = MetricsThreadLocal.get().timer("traceability");
		
		try {
			
			traceabilityTimer.start();
			
			if (commitChangeSet != null) {
				
				boolean isLightWeight = false;
				
				if (!entry.getChanges().isEmpty()) {
					<commit_msg>MAINT-483: Fix traceability logging of merge/rebase operations<commit_after>/*
 * Copyright 2011-2017 B2i Healthcare Pte Ltd, http://b2i.sg
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.b2international.snowowl.snomed.api.impl.traceability;

import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.Sets.newHashSet;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Set;

import org.eclipse.emf.cdo.CDOObject;
import org.eclipse.emf.cdo.common.revision.delta.CDOFeatureDelta;
import org.eclipse.emf.cdo.common.revision.delta.CDORevisionDelta;
import org.eclipse.emf.cdo.common.revision.delta.CDOSetFeatureDelta;
import org.eclipse.emf.ecore.EClass;
import org.eclipse.emf.ecore.EStructuralFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.b2international.commons.http.ExtendedLocale;
import com.b2international.index.revision.RevisionIndex;
import com.b2international.index.revision.RevisionIndexRead;
import com.b2international.index.revision.RevisionSearcher;
import com.b2international.snowowl.core.ApplicationContext;
import com.b2international.snowowl.core.RepositoryManager;
import com.b2international.snowowl.core.api.IBranchPath;
import com.b2international.snowowl.core.api.SnowowlRuntimeException;
import com.b2international.snowowl.core.api.SnowowlServiceException;
import com.b2international.snowowl.core.events.metrics.MetricsThreadLocal;
import com.b2international.snowowl.core.events.metrics.Timer;
import com.b2international.snowowl.core.ft.FeatureToggles;
import com.b2international.snowowl.core.ft.Features;
import com.b2international.snowowl.datastore.ICDOChangeProcessor;
import com.b2international.snowowl.datastore.ICDOCommitChangeSet;
import com.b2international.snowowl.datastore.cdo.CDOIDUtils;
import com.b2international.snowowl.datastore.index.ImmutableIndexCommitChangeSet;
import com.b2international.snowowl.datastore.index.IndexCommitChangeSet;
import com.b2international.snowowl.eventbus.IEventBus;
import com.b2international.snowowl.identity.domain.User;
import com.b2international.snowowl.snomed.Component;
import com.b2international.snowowl.snomed.Concept;
import com.b2international.snowowl.snomed.Description;
import com.b2international.snowowl.snomed.Relationship;
import com.b2international.snowowl.snomed.SnomedConstants;
import com.b2international.snowowl.snomed.SnomedConstants.Concepts;
import com.b2international.snowowl.snomed.SnomedPackage;
import com.b2international.snowowl.snomed.api.browser.ISnomedBrowserAxiomService;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserConcept;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserDescription;
import com.b2international.snowowl.snomed.api.domain.browser.ISnomedBrowserRelationship;
import com.b2international.snowowl.snomed.api.domain.browser.SnomedBrowserDescriptionType;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserConcept;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserDescription;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationship;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationshipTarget;
import com.b2international.snowowl.snomed.api.impl.domain.browser.SnomedBrowserRelationshipType;
import com.b2international.snowowl.snomed.common.ContentSubType;
import com.b2international.snowowl.snomed.common.SnomedTerminologyComponentConstants;
import com.b2international.snowowl.snomed.core.domain.SnomedConcept;
import com.b2international.snowowl.snomed.core.domain.SnomedConcepts;
import com.b2international.snowowl.snomed.core.domain.SnomedDescription;
import com.b2international.snowowl.snomed.core.domain.SnomedDescriptions;
import com.b2international.snowowl.snomed.core.domain.SnomedRelationship;
import com.b2international.snowowl.snomed.core.domain.SnomedRelationships;
import com.b2international.snowowl.snomed.core.lang.LanguageSetting;
import com.b2international.snowowl.snomed.datastore.SnomedDatastoreActivator;
import com.b2international.snowowl.snomed.datastore.SnomedDescriptionLookupService;
import com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration;
import com.b2international.snowowl.snomed.datastore.id.SnomedIdentifiers;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedConceptDocument;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedDescriptionIndexEntry;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRefSetMemberIndexEntry;
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRelationshipIndexEntry;
import com.b2international.snowowl.snomed.datastore.request.SnomedRequests;
import com.b2international.snowowl.snomed.snomedrefset.SnomedAnnotationRefSetMember;
import com.b2international.snowowl.snomed.snomedrefset.SnomedRefSetMember;
import com.b2international.snowowl.snomed.snomedrefset.SnomedRefSetPackage;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.base.Function;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableSet;

/**
 * Change processor implementation that produces a log entry for committed transactions.
 */
@SuppressWarnings("deprecation")
public class SnomedTraceabilityChangeProcessor implements ICDOChangeProcessor {

	private static final Logger TRACE_LOGGER = LoggerFactory.getLogger("traceability");
	private static final Logger SYS_LOGGER = LoggerFactory.getLogger(SnomedTraceabilityChangeProcessor.class);

	private static final String OWL_AXIOM = "OwlAxiom";

	private static final Set<EClass> MRCM_REFSET_MEMBER_TYPES = ImmutableSet.<EClass>of(
			SnomedRefSetPackage.Literals.SNOMED_MRCM_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_RANGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_MODULE_SCOPE_REF_SET_MEMBER);
	
	private static final Set<EClass> DETACHED_REFSET_MEMBER_TYPES = ImmutableSet.<EClass>of(
			SnomedRefSetPackage.Literals.SNOMED_ANNOTATION_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_LANGUAGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_ASSOCIATION_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_ATTRIBUTE_VALUE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_DOMAIN_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_ATTRIBUTE_RANGE_REF_SET_MEMBER,
			SnomedRefSetPackage.Literals.SNOMED_MRCM_MODULE_SCOPE_REF_SET_MEMBER);
	
	private static final Set<EStructuralFeature> IGNORED_FEATURES = ImmutableSet.<EStructuralFeature>of(
			SnomedPackage.Literals.CONCEPT__DESCRIPTIONS,
			SnomedPackage.Literals.CONCEPT__OUTBOUND_RELATIONSHIPS,
			SnomedPackage.Literals.CONCEPT__FULLY_SPECIFIED_NAME,
			SnomedPackage.Literals.RELATIONSHIP__REFINABILITY_REF_SET_MEMBERS);
	
	private TraceabilityEntry entry;

	private final boolean collectSystemChanges;
	private final RevisionIndex index;
	private final IBranchPath branchPath;
	private ISnomedBrowserAxiomService axiomService;

	private ICDOCommitChangeSet commitChangeSet;
	private ObjectWriter objectWriter;
	private SnomedDescriptionLookupService descriptionLookupService;

	public SnomedTraceabilityChangeProcessor(final IBranchPath branchPath, ObjectWriter objectWriter) {
		this.branchPath = checkNotNull(branchPath);
		this.objectWriter = checkNotNull(objectWriter);
		this.index = ApplicationContext.getServiceForClass(RepositoryManager.class).get(SnomedDatastoreActivator.REPOSITORY_UUID).service(RevisionIndex.class);
		this.collectSystemChanges = collectSystemChanges(branchPath.getPath());
		this.descriptionLookupService = new SnomedDescriptionLookupService();
	}

	@Override
	public void process(ICDOCommitChangeSet commitChangeSet) throws SnowowlServiceException {
		
		final Timer traceabilityTimer = MetricsThreadLocal.get().timer("traceability");
		
		try {
			
			traceabilityTimer.start();
			
			this.entry = new TraceabilityEntry(commitChangeSet);
			this.commitChangeSet = commitChangeSet;
			
			if (isSystemCommit(commitChangeSet.getUserId()) && !collectSystemChanges) {
				return;
			}
			
			processNewComponents(commitChangeSet.getNewComponents());
			processComponentUpdates(commitChangeSet.getDirtyComponents());
			processComponentDeletions(commitChangeSet);
			
		} finally {
			traceabilityTimer.stop();
		}
	}

	@Override
	public IndexCommitChangeSet commit() throws SnowowlServiceException {
		SYS_LOGGER.info("Collected {} traceability entries on branch {}", entry.getChanges().size(), branchPath.getPath());
		return ImmutableIndexCommitChangeSet.builder().build();
	}

	@Override
	public void afterCommit() {
		
		final Timer traceabilityTimer = MetricsThreadLocal.get().timer("traceability");
		
		try {
			
			traceabilityTimer.start();
			
			if (commitChangeSet != null) {
				
				boolean isLightWeight = false;
				
				if (!entry.getChanges().isEmpty()) {
					
{'fraction_non_alphanumeric': 0.0836130764860817, 'fraction_numerical': 0.007901668129938543, 'mean_word_length': 7.051559251559252, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4448028', 'n_tokens_mistral': 6599, 'n_tokens_neox': 6028, 'n_words': 836}