<commit_before>"
I define a data structure independent text iterator api.

Use me for text enumeration purposes
"
Class {
	#name : #BlTextIterator,
	#superclass : #Object,
	#traits : 'TBlDebug',
	#classTraits : 'TBlDebug classTrait',
	#category : #'Bloc-Text-Text'
}

{ #category : #'iterator - accessing' }
BlTextIterator >> attributes [
	"Return a collection of rope attributes at a current position without duplicates"
	<return: #Collection of: #BlTextAttribute>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> detectAttribute: aBlock ifFound: aFoundBlock ifNone: anExceptionBlock [
	"Evaluate aBlock with each of the receiver's elements as the argument.  
	If some element evaluates aBlock to true, then cull this element into 
	foundBlock and answer the result of this evaluation. 
	If none evaluate to true, then evaluate exceptionBlock."
	<return: #BlTextAttribute or: #Object>

	^ self subclassResponsibility
]

{ #category : #'iterator - testing' }
BlTextIterator >> doesAttributeEnd: anAttribute [
	"Return true if a given attribute ends at the current position.
	By `ends` we mean that this attribute is not present at the next position.
	
	Implementation detail:
	 - we should not rely on the iterator scope, otherwise it will give us false positives on the edge of the scope for attributes that don't really end there."
	<return: #Boolean>
	
	"we are at the last character, so the attribute definitely ends here"
	self text size = self position
		ifTrue: [ ^ true ].
	
	(self attributes includes: anAttribute)
		ifFalse: [ ^ true ].
	
	"check if this attribute exists at the next position, if not - it ends here"
	^ ((self text attributesAt: self position + 1) includes: anAttribute) not
]

{ #category : #'iterator - testing' }
BlTextIterator >> hasNext [
	"Return true if there is a next text item (character) after a current position"
	<return: #Boolean>

	^ self subclassResponsibility
]

{ #category : #'iterator - testing' }
BlTextIterator >> hasPrevious [
	"Return true if there is a previous text item (character) before the current position"
	<return: #Boolean>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> next [
	"Return an item (character) at current position and move one to the right"
	<return: #Character or: #Object>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> nextAttributeStartAndEnd: aBlTextAttribute indicesDo: anIndicesBlock [
	"Find and iterate over the start and end indices of the given attribute"	
	| startPosition endPosition isAttributeStartFound isAttributeEndFound |

	self hasNext
		ifFalse: [ self error: 'reached end of rope' ].
	
	startPosition := nil.
	endPosition := nil.
	
	isAttributeStartFound := false.
	isAttributeEndFound := false.

	[ self hasNext and: [
		| hasAttribute aSpan aCurrentPosition |

		aCurrentPosition := self position + 1.

		aSpan := self nextSpan.
		hasAttribute := aSpan attributes includes: aBlTextAttribute.
		
		"we found the start of the attribute"
		(hasAttribute and: [ isAttributeStartFound not ])
			ifTrue: [
				startPosition := aCurrentPosition.
				isAttributeStartFound := true ].
	
		"we found the end of the attribute"
		(hasAttribute not and: [ isAttributeStartFound ])
			ifTrue: [
				endPosition := aCurrentPosition - 1.
				isAttributeEndFound := true ].
		
		(isAttributeStartFound and: [ isAttributeEndFound ]) not
	] ] whileTrue.

	(isAttributeStartFound and: [ isAttributeEndFound not ])
		ifTrue: [
			endPosition := self position.<commit_msg>[feenkcom/gtoolkit#683] nextAttributeStartAndEnd: compare by identity<commit_after>"
I define a data structure independent text iterator api.

Use me for text enumeration purposes
"
Class {
	#name : #BlTextIterator,
	#superclass : #Object,
	#traits : 'TBlDebug',
	#classTraits : 'TBlDebug classTrait',
	#category : #'Bloc-Text-Text'
}

{ #category : #'iterator - accessing' }
BlTextIterator >> attributes [
	"Return a collection of rope attributes at a current position without duplicates"
	<return: #Collection of: #BlTextAttribute>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> detectAttribute: aBlock ifFound: aFoundBlock ifNone: anExceptionBlock [
	"Evaluate aBlock with each of the receiver's elements as the argument.  
	If some element evaluates aBlock to true, then cull this element into 
	foundBlock and answer the result of this evaluation. 
	If none evaluate to true, then evaluate exceptionBlock."
	<return: #BlTextAttribute or: #Object>

	^ self subclassResponsibility
]

{ #category : #'iterator - testing' }
BlTextIterator >> doesAttributeEnd: anAttribute [
	"Return true if a given attribute ends at the current position.
	By `ends` we mean that this attribute is not present at the next position.
	
	Implementation detail:
	 - we should not rely on the iterator scope, otherwise it will give us false positives on the edge of the scope for attributes that don't really end there."
	<return: #Boolean>
	
	"we are at the last character, so the attribute definitely ends here"
	self text size = self position
		ifTrue: [ ^ true ].
	
	(self attributes includes: anAttribute)
		ifFalse: [ ^ true ].
	
	"check if this attribute exists at the next position, if not - it ends here"
	^ ((self text attributesAt: self position + 1) includes: anAttribute) not
]

{ #category : #'iterator - testing' }
BlTextIterator >> hasNext [
	"Return true if there is a next text item (character) after a current position"
	<return: #Boolean>

	^ self subclassResponsibility
]

{ #category : #'iterator - testing' }
BlTextIterator >> hasPrevious [
	"Return true if there is a previous text item (character) before the current position"
	<return: #Boolean>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> next [
	"Return an item (character) at current position and move one to the right"
	<return: #Character or: #Object>

	^ self subclassResponsibility
]

{ #category : #'iterator - enumeration' }
BlTextIterator >> nextAttributeStartAndEnd: aBlTextAttribute indicesDo: anIndicesBlock [
	"Find and iterate over the start and end indices of the given attribute"	
	| startPosition endPosition isAttributeStartFound isAttributeEndFound |

	self hasNext
		ifFalse: [ self error: 'reached end of rope' ].
	
	startPosition := nil.
	endPosition := nil.
	
	isAttributeStartFound := false.
	isAttributeEndFound := false.

	[ self hasNext and: [
		| hasAttribute aSpan aCurrentPosition |

		aCurrentPosition := self position + 1.

		aSpan := self nextSpan.
		hasAttribute := aSpan attributes anySatisfy: [ :eachAttribute | eachAttribute == aBlTextAttribute ] .
		
		"we found the start of the attribute"
		(hasAttribute and: [ isAttributeStartFound not ])
			ifTrue: [
				startPosition := aCurrentPosition.
				isAttributeStartFound := true ].
	
		"we found the end of the attribute"
		(hasAttribute not and: [ isAttributeStartFound ])
			ifTrue: [
				endPosition := aCurrentPosition - 1.
				isAttributeEndFound := true ].
		
		(isAttributeStartFound and: [ isAttributeEndFound ]) not
	] ] whileTrue.

	(isAttributeStartFound and: [ isAttributeEndFound not ])
		ifTrue: [
			endPosition := self position.
{'fraction_non_alphanumeric': 0.0864367184279236, 'fraction_numerical': 0.0012367733956300674, 'mean_word_length': 4.347538574577516, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3000237', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2018, 'n_words': 791}