<commit_before>##
# copyright 2009, <NAME>
# http://python.projects.postgresql.org
##
"""
String split and join operations for dealing with literals and identifiers.

Notably, the functions in this module are intended to be used for simple
use-cases. It attempts to stay away from "real" parsing and simply provides
functions for common needs, like the ability to identify unquoted portions of a
query string so that logic or transformations can be applied to only unquoted
portions. Scanning for statement terminators, or safely interpolating
identifiers.

All functions deal with strict quoting rules.
"""
import re

def escape_literal(text):
	"Replace every instance of ' with ''"
	return text.replace("'", "''")

def quote_literal(text):
	"Escape the literal and wrap it in [single] quotations"
	return "'" + text.replace("'", "''") + "'"

def escape_ident(text):
	'Replace every instance of " with ""'
	return text.replace('"', '""')

def quote_ident(text):
	"""
	If needed, replace every instance of '"' with '""' *and* place '"' on each
	end. Otherwise, just return the text.
	"""
	if not text or text[0].isdecimal() or (
		not text.replace('_', 'a').isalnum()
	):
		return '"' + text.replace('"', '""') + '"'
	return text

quote_re = re.compile(r"""(?xu)
	E'(?:''|\\.|[^'])*(?:'|$)          (?# Backslash escapes E'str')
|	'(?:''|[^'])*(?:'|$)               (?# Regular literals 'str')
|	"(?:""|[^"])*(?:"|$)               (?# Identifiers "str")
|	(\$(?:[^0-9$]\w*)?\$).*?(?:\1|$)   (?# Dollar quotes $$str$$)
""")

def split(text, re = quote_re):
	"""
	split the string up by into non-quoted and quoted portions. Zero and even
	numbered indexes are unquoted portions, while odd indexes are quoted
	portions. 

	Unquoted portions are regular strings, whereas quoted portions are
	pair-tuples specifying the quotation mechanism and the content thereof.

	>>> list(split("select $$foobar$$"))
	['select ', ('$$', 'foobar'), '']

	If the split ends on a quoted section, it means the string's quote was not
	terminated. Subsequently, there will be an even number of objects in the
	list.

	Quotation errors are detected, but never raised. Rather it's up to the user
	to identify the best course of action for the given split.
	"""
	lastend = 0
	scan = re.scanner(text)
	match = scan.search()
	while match is not None:
		# text preceding the quotation<commit_msg>Use the re from the globals.<commit_after>##
# copyright 2009, <NAME>
# http://python.projects.postgresql.org
##
"""
String split and join operations for dealing with literals and identifiers.

Notably, the functions in this module are intended to be used for simple
use-cases. It attempts to stay away from "real" parsing and simply provides
functions for common needs, like the ability to identify unquoted portions of a
query string so that logic or transformations can be applied to only unquoted
portions. Scanning for statement terminators, or safely interpolating
identifiers.

All functions deal with strict quoting rules.
"""
import re

def escape_literal(text):
	"Replace every instance of ' with ''"
	return text.replace("'", "''")

def quote_literal(text):
	"Escape the literal and wrap it in [single] quotations"
	return "'" + text.replace("'", "''") + "'"

def escape_ident(text):
	'Replace every instance of " with ""'
	return text.replace('"', '""')

def quote_ident(text):
	"""
	If needed, replace every instance of '"' with '""' *and* place '"' on each
	end. Otherwise, just return the text.
	"""
	if not text or text[0].isdecimal() or (
		not text.replace('_', 'a').isalnum()
	):
		return '"' + text.replace('"', '""') + '"'
	return text

quote_re = re.compile(r"""(?xu)
	E'(?:''|\\.|[^'])*(?:'|$)          (?# Backslash escapes E'str')
|	'(?:''|[^'])*(?:'|$)               (?# Regular literals 'str')
|	"(?:""|[^"])*(?:"|$)               (?# Identifiers "str")
|	(\$(?:[^0-9$]\w*)?\$).*?(?:\1|$)   (?# Dollar quotes $$str$$)
""")

def split(text):
	"""
	split the string up by into non-quoted and quoted portions. Zero and even
	numbered indexes are unquoted portions, while odd indexes are quoted
	portions. 

	Unquoted portions are regular strings, whereas quoted portions are
	pair-tuples specifying the quotation mechanism and the content thereof.

	>>> list(split("select $$foobar$$"))
	['select ', ('$$', 'foobar'), '']

	If the split ends on a quoted section, it means the string's quote was not
	terminated. Subsequently, there will be an even number of objects in the
	list.

	Quotation errors are detected, but never raised. Rather it's up to the user
	to identify the best course of action for the given split.
	"""
	lastend = 0
	re = quote_re
	scan = re.scanner(text)
	match = scan.search()
	while match is not None:
		# text preceding the quotation
{'fraction_non_alphanumeric': 0.1533600168527491, 'fraction_numerical': 0.003791868548556983, 'mean_word_length': 4.116379310344827, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1835444', 'n_tokens_mistral': 1520, 'n_tokens_neox': 1435, 'n_words': 564}