<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys,time,random,base64,re
from datetime import datetime, timedelta
from ACR.errors import *
from ACR.utils.hashcompat import md5_constructor
from ACR.utils.xmlextras import escapeQuotes
from ACR.utils import dicttree
#from ACR.utils.interpreter import make_tree
from ACR import acconfig

if hasattr(random, 'SystemRandom'):
	randrange=random.SystemRandom().randrange
else:
	randrange=random.randrange

PREFIX_DELIMITER="::"
RE_PATH=re.compile("{\$([^}]+)}") # {$ foobar}
RE_PATH_split=re.compile("{\$[^}]+}") # {$ foobar}

def getStorage(env,s):
	D=env.doDebug
	if D:
		env.debug("START getStorage with s= '%s'",s)
	s=s.lower()
	if s=="session" or s=="ss":
		if not env.sessionStorage:
			if D: env.debug("END getStorage with: session do not exists")
			return False
		if D: env.debug("END getStorage with: session storage")
		return env.sessionStorage.data
	#elif s=="app" or s=="as":
	#	return
	#TODO cookie storage
	#elif s=="cookie" or s=="cs":
	#	return
	elif s=="global" or s=="gs":
		return {}
	if D: env.debug("END getStorage with: request storage")
	return env.requestStorage

def replaceVars(env,l,fn=None):
	"""
	l - output of prepareVars
	fn - function that will be executed on each string
	"""
	D=env.doDebug
	if D: env.debug("START replaceVars with: path='%s' and fn='%s'", l, fn)
	try:
		l.__iter__
	except:
		return l
	ret=[]
	for i in l:
		if type(i) is tuple:
			if D: env.debug("computing '%s'",i)
			storage=getStorage(env,i[0])
			v=dicttree.get(storage,i[1],acenv=env)
			doFN=True
			if D: env.debug("adding '%s' to the end of string",v)
			ret.append(v)
		else:
			if D: env.debug("adding '%s' to the end of string",i)
			ret.append(i)
	if len(ret) is 1:
		if D: env.debug("END replaceVars with: %s",ret[0])
		return ret[0]
	try:
		if D: env.debug("END replaceVars with: %s","".join(ret))
		return "".join(ret)
	except TypeError,e:
		if D: env.error("END replaceVars with TypeError: %s; returning %s",e,ret)
		print type(ret[3])
		return ret

def prepareVars(s):
	if type(s) is not str:
		return s
	splitted=RE_PATH_split.split(s)
	vars=RE_PATH.findall(s)
	ret=[]
	try:
		while True:
			ret.append(splitted.pop(0))
			var=vars.pop(0)
			try:
				storageName,path=var.split(PREFIX_DELIMITER)
			except ValueError:
				storageName="rs"
				path=var
			path=path.split(".")
			ret.append((storageName,path))
	except:<commit_msg>minor fixes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys,time,random,base64,re
from datetime import datetime, timedelta
from ACR.errors import *
from ACR.utils.hashcompat import md5_constructor
from ACR.utils.xmlextras import escapeQuotes
from ACR.utils import dicttree
#from ACR.utils.interpreter import make_tree
from ACR import acconfig

if hasattr(random, 'SystemRandom'):
	randrange=random.SystemRandom().randrange
else:
	randrange=random.randrange

PREFIX_DELIMITER="::"
RE_PATH=re.compile("{\$([^}]+)}") # {$ foobar}
RE_PATH_split=re.compile("{\$[^}]+}") # {$ foobar}

def getStorage(env,s):
	D=env.doDebug
	if D:
		env.debug("START getStorage with s= '%s'",s)
	s=s.lower()
	if s=="session" or s=="ss":
		if not env.sessionStorage:
			if D: env.debug("END getStorage with: session do not exists")
			return False
		if D: env.debug("END getStorage with: session storage")
		return env.sessionStorage.data
	#elif s=="app" or s=="as":
	#	return
	#TODO cookie storage
	#elif s=="cookie" or s=="cs":
	#	return
	elif s=="global" or s=="gs":
		return {}
	if D: env.debug("END getStorage with: request storage")
	return env.requestStorage

def replaceVars(env,l,fn=None):
	"""
	l - output of prepareVars
	fn - function that will be executed on each string
	"""
	D=env.doDebug
	if D: env.debug("START replaceVars with: path='%s' and fn='%s'", l, fn)
	try:
		l.__iter__
	except:
		return l
	ret=[]
	for i in l:
		if type(i) is tuple:
			if D: env.debug("computing '%s'",i)
			storage=getStorage(env,i[0])
			v=dicttree.get(storage,i[1],acenv=env)
			doFN=True
			if D: env.debug("adding '%s' to the end of string",v)
			ret.append(v)
		else:
			if D: env.debug("adding '%s' to the end of string",i)
			ret.append(i)
	if len(ret) is 1:
		if D: env.debug("END replaceVars with: %s",ret[0])
		return ret[0]
	try:
		if D: env.debug("END replaceVars with: %s","".join(ret))
		return "".join(ret)
	except TypeError,e:
		if D: env.error("END replaceVars with TypeError: %s; returning %s",e,ret)
		return ret

def prepareVars(s):
	if type(s) is not str:
		return s
	splitted=RE_PATH_split.split(s)
	vars=RE_PATH.findall(s)
	ret=[]
	try:
		while True:
			ret.append(splitted.pop(0))
			var=vars.pop(0)
			try:
				storageName,path=var.split(PREFIX_DELIMITER)
			except ValueError:
				storageName="rs"
				path=var
			path=path.split(".")
			ret.append((storageName,path))
	except:
{'fraction_non_alphanumeric': 0.15915229586536464, 'fraction_numerical': 0.0047787242883856225, 'mean_word_length': 4.396860986547085, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1639248', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1852, 'n_words': 478}