<commit_before># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

"""build query for doclistview and return results"""

import frappe, json
import frappe.permissions
from frappe.model.db_query import DatabaseQuery
from frappe.model import default_fields, optional_fields
from frappe import _
from io import StringIO
from frappe.core.doctype.access_log.access_log import make_access_log
from frappe.utils import cstr, format_duration
from frappe.model.base_document import get_controller


@frappe.whitelist()
@frappe.read_only()
def get():
	args = get_form_params()
	# If virtual doctype get data from controller het_list method
	if frappe.db.get_value("DocType", filters={"name": args.doctype}, fieldname="is_virtual"):
		controller = get_controller(args.doctype)
		data = compress(controller(args.doctype).get_list(args))
	else:
		data = compress(execute(**args), args=args)
	return data

@frappe.whitelist()
@frappe.read_only()
def get_list():
	# uncompressed (refactored from frappe.model.db_query.get_list)
	return execute(**get_form_params())

@frappe.whitelist()
@frappe.read_only()
def get_count():
	args = get_form_params()

	distinct = 'distinct ' if args.distinct=='true' else ''
	args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"]
	return execute(**args)[0].get('total_count')

def execute(doctype, *args, **kwargs):
	return DatabaseQuery(doctype).execute(*args, **kwargs)

def get_form_params():
	"""Stringify GET request parameters."""
	data = frappe._dict(frappe.local.form_dict)
	clean_params(data)
	validate_args(data)
	return data

def validate_args(data):
	parse_json(data)
	setup_group_by(data)

	validate_fields(data)
	if data.filters:
		validate_filters(data, data.filters)
	if data.or_filters:
		validate_filters(data, data.or_filters)

	data.strict = None

	return data

def validate_fields(data):
	wildcard = update_wildcard_field_param(data)

	for field in data.fields or []:
		fieldname = extract_fieldname(field)
		if is_standard(fieldname):
			continue

		meta, df = get_meta_and_docfield(fieldname, data)

		if not df:
			if wildcard:
				continue
			else:
				raise_invalid_field(fieldname)

		# remove the field from the query if the report hide flag is set and current view is Report
		if df.report_hide and data.view == 'Report':
			data.fields.remove(field)
			continue

		if df.fieldname in [_df.fieldname for _df in meta.get_high_permlevel_fields()]:
			if df.get('permlevel') not in meta.get_permlevel_access(parenttype=data.doctype):
				data.fields.remove(field)

def validate_filters(data, filters):
	if isinstance(filters, list):
		# filters as list
		for condition in filters:
			if len(condition)==3:
				# [fieldname, condition, value]
				fieldname = condition[0]
				if is_standard(fieldname):
					continue
				meta, df = get_meta_and_docfield(fieldname, data)
				if not df:
					raise_invalid_field(condition[0])
			else:
				# [doctype, fieldname, condition, value]
				fieldname = condition[1]
				if is_standard(fieldname):
					continue
				meta = frappe.get_meta(condition[0])
				if not meta.get_field(fieldname):
					raise_invalid_field(fieldname)

	else:
		for fieldname in filters:
			if is_standard(fieldname):
				continue
			meta, df = get_meta_and_docfield(fieldname, data)
			if not df:
				raise_invalid_field(fieldname)

def setup_group_by(data):
	'''Add columns for aggregated values e.g. count(name)'''
	if data.group_by and data.aggregate_function:
		if data.aggregate_function.lower() not in ('count', 'sum', 'avg'):
			frappe.throw(_('Invalid aggregate function'))

		if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field):
			data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data))
			if data.aggregate_on_field:
				data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`")
		else:
			raise_invalid_field(data.aggregate_on_field)

		data.pop('aggregate_on_doctype')
		data.pop('aggregate_on_field')
		data.pop('aggregate_function')

def raise_invalid_field(fieldname):
	frappe.throw(_('Field not permitted in query') + ': {0}'.format(fieldname), frappe.DataError)

def is_standard(fieldname):
	if '.' in fieldname:
		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
	return fieldname in default_fields or fieldname in optional_fields

def extract_fieldname(field):
	for text in (',', '/*', '#'):
		if text in field:
			raise_invalid_field(field)

	fieldname = field
	for sep in (' as ', ' AS '):
		if sep in fieldname:
			fieldname = fieldname.split(sep)[0]

	# certain functions allowed, extract the fieldname from the function
	if (fieldname.startswith('count(')
		or fieldname.startswith('sum(')
		or fieldname.startswith('avg(')):
		if not fieldname.strip().endswith(')'):
			raise_invalid_field(field)
		fieldname = fieldname.split('(', 1)[1][:-1]

	return fieldname

def get_meta_and_docfield(fieldname, data):
	parenttype, fieldname = get_parenttype_and_fieldname(fieldname, data)
	meta = frappe.get_meta(parenttype)
	df = meta.get_field(fieldname)
	return meta, df

def update_wildcard_field_param(data):
	if ((isinstance(data.fields, str) and data.fields == "*")
		or (isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*")):
		data.fields = frappe.db.get_table_columns(data.doctype)
		return True

	return False


def clean_params(data):
	data.pop('cmd', None)
	data.pop('data', None)
	data.pop('ignore_permissions', None)
	data.pop('view', None)
	data.pop('user', None)

	if "csrf_token" in data:
		del data["csrf_token"]


def parse_json(data):
	if isinstance(data.get("filters"), str):
		data["filters"] = json.loads(data["filters"])
	if isinstance(data.get("or_filters"), str):
		data["or_filters"] = json.loads(data["or_filters"])
	if isinstance(data.get("fields"), str):
		data["fields"] = json.loads(data["fields"])
	if isinstance(data.get("docstatus"), str):
		data["docstatus"] = json.loads(data["docstatus"])
	if isinstance(data.get("save_user_settings"), str):
		data["save_user_settings"] = json.loads(data["save_user_settings"])<commit_msg>fix: clean `join` param when executing `reportview.get`<commit_after># Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# License: MIT. See LICENSE

"""build query for doclistview and return results"""

import frappe, json
import frappe.permissions
from frappe.model.db_query import DatabaseQuery
from frappe.model import default_fields, optional_fields
from frappe import _
from io import StringIO
from frappe.core.doctype.access_log.access_log import make_access_log
from frappe.utils import cstr, format_duration
from frappe.model.base_document import get_controller


@frappe.whitelist()
@frappe.read_only()
def get():
	args = get_form_params()
	# If virtual doctype get data from controller het_list method
	if frappe.db.get_value("DocType", filters={"name": args.doctype}, fieldname="is_virtual"):
		controller = get_controller(args.doctype)
		data = compress(controller(args.doctype).get_list(args))
	else:
		data = compress(execute(**args), args=args)
	return data

@frappe.whitelist()
@frappe.read_only()
def get_list():
	# uncompressed (refactored from frappe.model.db_query.get_list)
	return execute(**get_form_params())

@frappe.whitelist()
@frappe.read_only()
def get_count():
	args = get_form_params()

	distinct = 'distinct ' if args.distinct=='true' else ''
	args.fields = [f"count({distinct}`tab{args.doctype}`.name) as total_count"]
	return execute(**args)[0].get('total_count')

def execute(doctype, *args, **kwargs):
	return DatabaseQuery(doctype).execute(*args, **kwargs)

def get_form_params():
	"""Stringify GET request parameters."""
	data = frappe._dict(frappe.local.form_dict)
	clean_params(data)
	validate_args(data)
	return data

def validate_args(data):
	parse_json(data)
	setup_group_by(data)

	validate_fields(data)
	if data.filters:
		validate_filters(data, data.filters)
	if data.or_filters:
		validate_filters(data, data.or_filters)

	data.strict = None

	return data

def validate_fields(data):
	wildcard = update_wildcard_field_param(data)

	for field in data.fields or []:
		fieldname = extract_fieldname(field)
		if is_standard(fieldname):
			continue

		meta, df = get_meta_and_docfield(fieldname, data)

		if not df:
			if wildcard:
				continue
			else:
				raise_invalid_field(fieldname)

		# remove the field from the query if the report hide flag is set and current view is Report
		if df.report_hide and data.view == 'Report':
			data.fields.remove(field)
			continue

		if df.fieldname in [_df.fieldname for _df in meta.get_high_permlevel_fields()]:
			if df.get('permlevel') not in meta.get_permlevel_access(parenttype=data.doctype):
				data.fields.remove(field)

def validate_filters(data, filters):
	if isinstance(filters, list):
		# filters as list
		for condition in filters:
			if len(condition)==3:
				# [fieldname, condition, value]
				fieldname = condition[0]
				if is_standard(fieldname):
					continue
				meta, df = get_meta_and_docfield(fieldname, data)
				if not df:
					raise_invalid_field(condition[0])
			else:
				# [doctype, fieldname, condition, value]
				fieldname = condition[1]
				if is_standard(fieldname):
					continue
				meta = frappe.get_meta(condition[0])
				if not meta.get_field(fieldname):
					raise_invalid_field(fieldname)

	else:
		for fieldname in filters:
			if is_standard(fieldname):
				continue
			meta, df = get_meta_and_docfield(fieldname, data)
			if not df:
				raise_invalid_field(fieldname)

def setup_group_by(data):
	'''Add columns for aggregated values e.g. count(name)'''
	if data.group_by and data.aggregate_function:
		if data.aggregate_function.lower() not in ('count', 'sum', 'avg'):
			frappe.throw(_('Invalid aggregate function'))

		if frappe.db.has_column(data.aggregate_on_doctype, data.aggregate_on_field):
			data.fields.append('{aggregate_function}(`tab{aggregate_on_doctype}`.`{aggregate_on_field}`) AS _aggregate_column'.format(**data))
			if data.aggregate_on_field:
				data.fields.append(f"`tab{data.aggregate_on_doctype}`.`{data.aggregate_on_field}`")
		else:
			raise_invalid_field(data.aggregate_on_field)

		data.pop('aggregate_on_doctype')
		data.pop('aggregate_on_field')
		data.pop('aggregate_function')

def raise_invalid_field(fieldname):
	frappe.throw(_('Field not permitted in query') + ': {0}'.format(fieldname), frappe.DataError)

def is_standard(fieldname):
	if '.' in fieldname:
		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
	return fieldname in default_fields or fieldname in optional_fields

def extract_fieldname(field):
	for text in (',', '/*', '#'):
		if text in field:
			raise_invalid_field(field)

	fieldname = field
	for sep in (' as ', ' AS '):
		if sep in fieldname:
			fieldname = fieldname.split(sep)[0]

	# certain functions allowed, extract the fieldname from the function
	if (fieldname.startswith('count(')
		or fieldname.startswith('sum(')
		or fieldname.startswith('avg(')):
		if not fieldname.strip().endswith(')'):
			raise_invalid_field(field)
		fieldname = fieldname.split('(', 1)[1][:-1]

	return fieldname

def get_meta_and_docfield(fieldname, data):
	parenttype, fieldname = get_parenttype_and_fieldname(fieldname, data)
	meta = frappe.get_meta(parenttype)
	df = meta.get_field(fieldname)
	return meta, df

def update_wildcard_field_param(data):
	if ((isinstance(data.fields, str) and data.fields == "*")
		or (isinstance(data.fields, (list, tuple)) and len(data.fields) == 1 and data.fields[0] == "*")):
		data.fields = frappe.db.get_table_columns(data.doctype)
		return True

	return False


def clean_params(data):
	for param in (
		"cmd",
		"data",
		"ignore_permissions",
		"view",
		"user",
		"csrf_token",
		"join"
	):
		data.pop(param, None)

def parse_json(data):
	if isinstance(data.get("filters"), str):
		data["filters"] = json.loads(data["filters"])
	if isinstance(data.get("or_filters"), str):
		data["or_filters"] = json.loads(data["or_filters"])
	if isinstance(data.get("fields"), str):
		data["fields"] = json.loads(data["fields"])
	if isinstance(data.get("docstatus"), str):
		data["docstatus"] = json.loads(data["docstatus"])
	if isinstance(data.get("save_user_settings"), str):
		data["save_user_settings"] = json.loads(data["save_user_settings"])
{'fraction_non_alphanumeric': 0.12874009380559598, 'fraction_numerical': 0.002749474365194889, 'mean_word_length': 5.4545929018789145, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1748232', 'n_tokens_mistral': 4566, 'n_tokens_neox': 4211, 'n_words': 935}