<commit_before>package scripts;

import com.b2international.index.Hits
import com.b2international.index.query.Expressions
import com.b2international.index.query.Query
import com.b2international.index.query.Expressions.ExpressionBuilder
import com.b2international.index.revision.RevisionSearcher
import com.b2international.snowowl.core.ComponentIdentifier
import com.b2international.snowowl.core.date.EffectiveTimes
import com.b2international.snowowl.snomed.common.SnomedTerminologyComponentConstants
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedConceptDocument
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRefSetMemberIndexEntry
import com.google.common.collect.Lists

/**
 *  OWL axiom relationships must not refer to inactive concepts as its type or destination.
 */
RevisionSearcher searcher = ctx.service(RevisionSearcher.class)

Iterable<Hits<String>> inactiveConceptBatches = searcher.scroll(Query.select(String.class)
		.from(SnomedConceptDocument.class)
		.fields(SnomedConceptDocument.Fields.ID)
		.where(SnomedConceptDocument.Expressions.inactive())
		.limit(10_000)
		.build())

List<ComponentIdentifier> issues = Lists.newArrayList()

inactiveConceptBatches.each({ conceptBatch ->
	List<String> inactiveConceptIds = conceptBatch.getHits()
	
	ExpressionBuilder invalidOWLAxiomExpression = Expressions.builder()
			.filter(SnomedRefSetMemberIndexEntry.Expressions.active())
			.should(SnomedRefSetMemberIndexEntry.Expressions.owlExpressionConcept(inactiveConceptIds))
			.should(SnomedRefSetMemberIndexEntry.Expressions.referencedComponentIds(inactiveConceptIds));
	
	if (params.isUnpublishedOnly) {
		invalidOWLAxiomExpression.filter(SnomedRefSetMemberIndexEntry.Expressions.effectiveTime(EffectiveTimes.UNSET_EFFECTIVE_TIME))
	}
	
	Iterable<Hits<String>> invalidAxiomMemberBatches = searcher.scroll(Query.select(String.class)
			.from(SnomedRefSetMemberIndexEntry.class)
			.fields(SnomedRefSetMemberIndexEntry.Fields.ID)
			.where(invalidOWLAxiomExpression.build())
			.limit(10_000)
			.build())

	invalidAxiomMemberBatches.each({ axiomMemberBatch ->
		axiomMemberBatch.each({ id ->
			issues.add(ComponentIdentifier.of(SnomedTerminologyComponentConstants.REFSET_MEMBER_NUMBER, id))
		})
	})
})

return issues<commit_msg>SO-4183: Ensure rule only returns owl refset members<commit_after>package scripts;

import com.b2international.index.Hits
import com.b2international.index.query.Expressions
import com.b2international.index.query.Query
import com.b2international.index.query.Expressions.ExpressionBuilder
import com.b2international.index.revision.RevisionSearcher
import com.b2international.snowowl.core.ComponentIdentifier
import com.b2international.snowowl.core.date.EffectiveTimes
import com.b2international.snowowl.snomed.common.SnomedConstants.Concepts
import com.b2international.snowowl.snomed.common.SnomedTerminologyComponentConstants
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedConceptDocument
import com.b2international.snowowl.snomed.datastore.index.entry.SnomedRefSetMemberIndexEntry
import com.google.common.collect.Lists

/**
 *  OWL axiom relationships must not refer to inactive concepts as its type or destination.
 */
RevisionSearcher searcher = ctx.service(RevisionSearcher.class)

Iterable<Hits<String>> inactiveConceptBatches = searcher.scroll(Query.select(String.class)
		.from(SnomedConceptDocument.class)
		.fields(SnomedConceptDocument.Fields.ID)
		.where(SnomedConceptDocument.Expressions.inactive())
		.limit(10_000)
		.build())

List<ComponentIdentifier> issues = Lists.newArrayList()

inactiveConceptBatches.each({ conceptBatch ->
	List<String> inactiveConceptIds = conceptBatch.getHits()
	
	ExpressionBuilder invalidOWLAxiomExpression = Expressions.builder()
			.filter(SnomedRefSetMemberIndexEntry.Expressions.active())
			.filter(SnomedRefSetMemberIndexEntry.Expressions.referenceSetId(Concepts.REFSET_OWL_AXIOM))
			.should(SnomedRefSetMemberIndexEntry.Expressions.owlExpressionConcept(inactiveConceptIds))
			.should(SnomedRefSetMemberIndexEntry.Expressions.referencedComponentIds(inactiveConceptIds));
	
	if (params.isUnpublishedOnly) {
		invalidOWLAxiomExpression.filter(SnomedRefSetMemberIndexEntry.Expressions.effectiveTime(EffectiveTimes.UNSET_EFFECTIVE_TIME))
	}
	
	Iterable<Hits<String>> invalidAxiomMemberBatches = searcher.scroll(Query.select(String.class)
			.from(SnomedRefSetMemberIndexEntry.class)
			.fields(SnomedRefSetMemberIndexEntry.Fields.ID)
			.where(invalidOWLAxiomExpression.build())
			.limit(10_000)
			.build())

	invalidAxiomMemberBatches.each({ axiomMemberBatch ->
		axiomMemberBatch.each({ id ->
			issues.add(ComponentIdentifier.of(SnomedTerminologyComponentConstants.REFSET_MEMBER_NUMBER, id))
		})
	})
})

return issues
{'fraction_non_alphanumeric': 0.09334449560485558, 'fraction_numerical': 0.009418166596902469, 'mean_word_length': 7.398945518453427, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6748697', 'n_tokens_mistral': 1600, 'n_tokens_neox': 1469, 'n_words': 173}