<commit_before>import sublime, sublime_plugin
import string

class MoveToBegOfContigBoundaryCommand(sublime_plugin.TextCommand):
	def run(self, edit, forward):
		view = self.view
		oldSelRegions = list(view.sel())
		view.sel().clear()
		for thisregion in oldSelRegions:
			if(forward): #forward
				caretPos = thisregion.b
				if(view.substr(caretPos) not in string.whitespace): #initially have char right of me, find whitespace
					while ((view.substr(caretPos) not in string.whitespace) and (caretPos < view.size())):
						caretPos += 1
				while((view.substr(caretPos) in string.whitespace) and (caretPos < view.size())): #now have whitespace right of me, find char beginning
					caretPos += 1
				else:
					view.sel().add(sublime.Region(caretPos))
					view.show(caretPos)
			else: #backward
				caretPos = thisregion.b - 1
				if(view.substr(caretPos) in string.whitespace): #initially have whitespace left of me, find char
					while ((view.substr(caretPos) in string.whitespace) and (caretPos >= 0)):
						caretPos -= 1
				while ((view.substr(caretPos) not in string.whitespace) and (caretPos >= 0)): #now have char left of me, find whitespace
					caretPos -= 1
				else:
					view.sel().add(sublime.Region(caretPos+1))
					view.show(caretPos+1)

class MoveToBegOfSubwordBoundaryCommand(sublime_plugin.TextCommand):
	def run(self, edit, forward):
		# chr(34) is doublequote
		subwordDelims = [" ", chr(34), ".", "+", "_", "<", ">", "[", "]", "{", "}", "-", "(", ")", "|", "\\"]
		view = self.view
		oldSelRegions = list(view.sel())
		view.sel().clear()
		for thisregion in oldSelRegions:
			if(forward): #forward
				caretPos = thisregion.b
				haveMovedAtleastOnce = False
				while ( (view.substr(caretPos) not in subwordDelims) and (caretPos < view.size()) ):
					if(not haveMovedAtleastOnce):
						haveMovedAtleastOnce = True
					caretPos += 1
				if(not haveMovedAtleastOnce):
					caretPos += 1
				view.sel().add(sublime.Region(caretPos))
				view.show(caretPos)
			else: #backward
				caretPos = thisregion.b - 1
				haveMovedAtleastOnce = False
				while ( (view.substr(caretPos) not in subwordDelims) and (caretPos >= 0) ):
					if(not haveMovedAtleastOnce):
						haveMovedAtleastOnce = True
					caretPos -= 1
				if(not haveMovedAtleastOnce):
					caretPos -= 1
				view.sel().add(sublime.Region(caretPos+1))
				view.show(caretPos+1)

class ExpandSelectionToDelimsCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		# beginDelims = ["\"", "\'", "(", "<", "[", "{"]
		# endDelims = ["\"", "\'", ")", ">", "]", "}"]
		universalDelims = [" ", "\"", "\'", "-", "(", ")", "<", ">", "[", "]", "{", "}", ":", ".", ",", "%", "@", "&", "\t", "\n"]
		view = self.view
		oldSelRegions = list(view.sel())
		for thisregion in oldSelRegions:
			thisRegionBegin = thisregion.begin() - 1
			thisRegionEnd = thisregion.end()
			if( (thisregion.begin() != thisRegionEnd) and (view.substr(thisRegionBegin) in universalDelims) ):<commit_msg>move_to_beg_of_subword_boundary add tab and newline and carriagereturn as character to stop on<commit_after>import sublime, sublime_plugin
import string

class MoveToBegOfContigBoundaryCommand(sublime_plugin.TextCommand):
	def run(self, edit, forward):
		view = self.view
		oldSelRegions = list(view.sel())
		view.sel().clear()
		for thisregion in oldSelRegions:
			if(forward): #forward
				caretPos = thisregion.b
				if(view.substr(caretPos) not in string.whitespace): #initially have char right of me, find whitespace
					while ((view.substr(caretPos) not in string.whitespace) and (caretPos < view.size())):
						caretPos += 1
				while((view.substr(caretPos) in string.whitespace) and (caretPos < view.size())): #now have whitespace right of me, find char beginning
					caretPos += 1
				else:
					view.sel().add(sublime.Region(caretPos))
					view.show(caretPos)
			else: #backward
				caretPos = thisregion.b - 1
				if(view.substr(caretPos) in string.whitespace): #initially have whitespace left of me, find char
					while ((view.substr(caretPos) in string.whitespace) and (caretPos >= 0)):
						caretPos -= 1
				while ((view.substr(caretPos) not in string.whitespace) and (caretPos >= 0)): #now have char left of me, find whitespace
					caretPos -= 1
				else:
					view.sel().add(sublime.Region(caretPos+1))
					view.show(caretPos+1)

class MoveToBegOfSubwordBoundaryCommand(sublime_plugin.TextCommand):
	def run(self, edit, forward):
		# chr(34) is doublequote
		# chr(9) is tab
		# chr(10) is new line
		# chr(13) is carriage return
		subwordDelims = [" ", chr(9), chr(10), chr(13), chr(34), ".", "+", "_", "<", ">", "[", "]", "{", "}", "-", "(", ")", "|", "\\"]
		view = self.view
		oldSelRegions = list(view.sel())
		view.sel().clear()
		for thisregion in oldSelRegions:
			if(forward): #forward
				caretPos = thisregion.b
				haveMovedAtleastOnce = False
				while ( (view.substr(caretPos) not in subwordDelims) and (caretPos < view.size()) ):
					if(not haveMovedAtleastOnce):
						haveMovedAtleastOnce = True
					caretPos += 1
				if(not haveMovedAtleastOnce):
					caretPos += 1
				view.sel().add(sublime.Region(caretPos))
				view.show(caretPos)
			else: #backward
				caretPos = thisregion.b - 1
				haveMovedAtleastOnce = False
				while ( (view.substr(caretPos) not in subwordDelims) and (caretPos >= 0) ):
					if(not haveMovedAtleastOnce):
						haveMovedAtleastOnce = True
					caretPos -= 1
				if(not haveMovedAtleastOnce):
					caretPos -= 1
				view.sel().add(sublime.Region(caretPos+1))
				view.show(caretPos+1)

class ExpandSelectionToDelimsCommand(sublime_plugin.TextCommand):
	def run(self, edit):
		# chr(9) is tab
		# chr(10) is new line
		# chr(13) is carriage return
		universalDelims = [" ", "\"", "\'", "-", "(", ")", "<", ">", "[", "]", "{", "}", ":", ".", ",", "%", "@", "&", chr(9), "\n"]
		view = self.view
		oldSelRegions = list(view.sel())
		for thisregion in oldSelRegions:
			thisRegionBegin = thisregion.begin() - 1
			thisRegionEnd = thisregion.end()
			if( (thisregion.begin() != thisRegionEnd) and (view.substr(thisRegionBegin) in universalDelims) ):
{'fraction_non_alphanumeric': 0.16351373581181114, 'fraction_numerical': 0.009870044415199869, 'mean_word_length': 3.833068362480127, 'pattern_counts': {'":': 2, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5245394', 'n_tokens_mistral': 2531, 'n_tokens_neox': 2120, 'n_words': 464}