<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
import datetime
from frappe.core.page.dashboard.dashboard import cache_source, get_from_date_from_timespan
from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate
from frappe.model.document import Document

@frappe.whitelist()
@cache_source
def get(chart_name = None, chart = None, no_cache = None, from_date=None, to_date=None, refresh = None):
	if chart_name:
		chart = frappe.get_doc('Dashboard Chart', chart_name)
	else:
		chart = frappe._dict(frappe.parse_json(chart))

	timespan = chart.timespan
	timegrain = chart.time_interval
	filters = frappe.parse_json(chart.filters_json)

	# don't include cancelled documents
	filters['docstatus'] = ('<', 2)

	if not from_date:
		from_date = get_from_date_from_timespan(to_date, timespan)
	if not to_date:
		to_date = datetime.datetime.now()

	# get conditions from filters
	conditions, values = frappe.db.build_conditions(filters)
	if chart.chart_type == 'Group By':
		chart_config = get_group_by_chart_config(chart, filters)
	else:
		print('conditions', conditions)
		# query will return year, unit and aggregate value
		data = frappe.db.sql('''
			select
				extract(year from {datefield}) as _year,
				{unit_function} as _unit,
				{aggregate_function}({value_field})
			from `tab{doctype}`
			where
				{conditions}
				and {datefield} >= '{from_date}'
				and {datefield} <= '{to_date}'
			group by _year, _unit
			order by _year asc, _unit asc
		'''.format(
			unit_function = get_unit_function(chart.based_on, timegrain),
			datefield = chart.based_on,
			aggregate_function = get_aggregate_function(chart.chart_type),
			value_field = chart.value_based_on or '1',
			doctype = chart.document_type,
			conditions = conditions,
			from_date = from_date.strftime('%Y-%m-%d'),
			to_date = to_date
		), values)

		# result given as year, unit -> convert it to end of period of that unit
		result = convert_to_dates(data, timegrain)

		# add missing data points for periods where there was no result
		result = add_missing_values(result, timegrain, from_date, to_date)

		chart_config = {
			"labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
			"datasets": [{
				"name": chart.name,
				"values": [r[1] for r in result]
			}]
		}

	return chart_config


def get_group_by_chart_config(chart, filters):
	conditions, values = frappe.db.build_conditions(filters)
	data = frappe.db.sql('''
		select
			{aggregate_function}({value_field}) as count,
			{group_by_field} as name
		from `tab{doctype}`
		where {conditions}
		group by {group_by_field}
		order by count desc
	'''.format(
		aggregate_function = get_aggregate_function(chart.group_by_type),
		value_field = chart.aggregate_function_based_on or '1',
		field = chart.aggregate_function_based_on or chart.group_by_based_on,
		group_by_field = chart.group_by_based_on,
		doctype = chart.document_type,
		conditions = conditions,
	), values, as_dict = True)

	print('data', data)
	if data:
		if chart.number_of_groups and chart.number_of_groups < len(data):
			other_count = 0
			for i in range(chart.number_of_groups - 1, len(data)):
				other_count += data[i]['count']
			data = data[0: chart.number_of_groups - 1]
			data.append({'name': 'Other', 'count': other_count})

		chart_config = {
			"labels": [item['name'] if item['name'] else 'Not Specified' for item in data],
			"datasets": [{
				"name": chart.name,
				"values": [item['count'] for item in data]
			}]
		}
		return chart_config
	else:
		return None


def get_aggregate_function(chart_type):
	return {
		"Sum": "SUM",
		"Count": "COUNT",
		"Average": "AVG",
	}[chart_type]<commit_msg>fix: make separate get_group_by_chart_config function<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe import _
import datetime
from frappe.core.page.dashboard.dashboard import cache_source, get_from_date_from_timespan
from frappe.utils import nowdate, add_to_date, getdate, get_last_day, formatdate
from frappe.model.document import Document

@frappe.whitelist()
@cache_source
def get(chart_name = None, chart = None, no_cache = None, from_date=None, to_date=None, refresh = None):
	if chart_name:
		chart = frappe.get_doc('Dashboard Chart', chart_name)
	else:
		chart = frappe._dict(frappe.parse_json(chart))

	timespan = chart.timespan
	timegrain = chart.time_interval
	filters = frappe.parse_json(chart.filters_json)

	# don't include cancelled documents
	filters['docstatus'] = ('<', 2)

	if chart.chart_type == 'Group By':
		chart_config = get_group_by_chart_config(chart, filters)
	else:
		chart_config =  get_chart_config(chart, filters, timespan, timegrain, from_date, to_date)

	return chart_config


def get_chart_config(chart, filters, timespan, timegrain, from_date, to_date):
	if not from_date:
		from_date = get_from_date_from_timespan(to_date, timespan)
	if not to_date:
		to_date = datetime.datetime.now()

	# get conditions from filters
	conditions, values = frappe.db.build_conditions(filters)
	# query will return year, unit and aggregate value
	data = frappe.db.sql('''
		select
			extract(year from {datefield}) as _year,
			{unit_function} as _unit,
			{aggregate_function}({value_field})
		from `tab{doctype}`
		where
			{conditions}
			and {datefield} >= '{from_date}'
			and {datefield} <= '{to_date}'
		group by _year, _unit
		order by _year asc, _unit asc
	'''.format(
		unit_function = get_unit_function(chart.based_on, timegrain),
		datefield = chart.based_on,
		aggregate_function = get_aggregate_function(chart.chart_type),
		value_field = chart.value_based_on or '1',
		doctype = chart.document_type,
		conditions = conditions,
		from_date = from_date.strftime('%Y-%m-%d'),
		to_date = to_date
	), values)

	# result given as year, unit -> convert it to end of period of that unit
	result = convert_to_dates(data, timegrain)

	# add missing data points for periods where there was no result
	result = add_missing_values(result, timegrain, from_date, to_date)

	chart_config = {
		"labels": [formatdate(r[0].strftime('%Y-%m-%d')) for r in result],
		"datasets": [{
			"name": chart.name,
			"values": [r[1] for r in result]
		}]
	}

	return chart_config


def get_group_by_chart_config(chart, filters):
	conditions, values = frappe.db.build_conditions(filters)
	data = frappe.db.sql('''
		select
			{aggregate_function}({value_field}) as count,
			{group_by_field} as name
		from `tab{doctype}`
		where {conditions}
		group by {group_by_field}
		order by count desc
	'''.format(
		aggregate_function = get_aggregate_function(chart.group_by_type),
		value_field = chart.aggregate_function_based_on or '1',
		field = chart.aggregate_function_based_on or chart.group_by_based_on,
		group_by_field = chart.group_by_based_on,
		doctype = chart.document_type,
		conditions = conditions,
	), values, as_dict = True)

	if data:
		if chart.number_of_groups and chart.number_of_groups < len(data):
			other_count = 0
			for i in range(chart.number_of_groups - 1, len(data)):
				other_count += data[i]['count']
			data = data[0: chart.number_of_groups - 1]
			data.append({'name': 'Other', 'count': other_count})

		chart_config = {
			"labels": [item['name'] if item['name'] else 'Not Specified' for item in data],
			"datasets": [{
				"name": chart.name,
				"values": [item['count'] for item in data]
			}]
		}
		return chart_config
	else:
		return None


def get_aggregate_function(chart_type):
	return {
		"Sum": "SUM",
		"Count": "COUNT",
		"Average": "AVG",
	}[chart_type]
{'fraction_non_alphanumeric': 0.11700488055484203, 'fraction_numerical': 0.0035961983046493705, 'mean_word_length': 4.751107828655835, 'pattern_counts': {'":': 22, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2887298', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2670, 'n_words': 669}