<commit_before>'use strict';

var oForEach   = require('es5-ext/object/for-each')
  , Set        = require('es6-set')
  , isSet      = require('es6-set/is-set')
  , memoize    = require('memoizee')
  , getData    = require('../business-process-query/get-data')
  , getDateMap = require('../business-process-query/get-status-history-date-map');

/*
Result map of calculateStatusEventsSums:

[serviceName] = {
	// 1
	businessProcess[status]: sum,
	// 2
	certificate[certificateName][status]: sum,
	processingStep[stepName]: {
		// 3.1
		pending: {
			// 3.1.1
			businessProcess: sum,
			// 3.1.2
			certificate[certificateName]: sum
		},
		// 3.2
		byProcessor[processorId][status]: {
			// 3.2.1
			businessProcess: sum,
			// 3.2.2
			certificate[certificateName]: sum
		}
	}
};
 */

var dbDateToISO = function (date) {
	return date.toISOString().substring(0, 10);
};

var calculate = function (dateMap, data, dateFrom, dateTo) {
	var resultMap = {};

	dateFrom = dbDateToISO(dateFrom);
	dateTo = dbDateToISO(dateTo);

	var storeStatusData = function (resultSet, field, statusData) {
		if (Array.isArray(statusData)) {
			if (!resultSet[field]) resultSet[field] = new Set();

			statusData.forEach(Set.prototype.add.bind(resultSet[field]));
		} else {
			if (resultSet[field] == null) resultSet[field] = 0;

			resultSet[field] += statusData;
		}
	};

	var storePerStatusResult = function (data, resultSet) {
		oForEach(data, function (statusData, status) {
			storeStatusData(resultSet, status, statusData);
		});
	};

	oForEach(dateMap, function (dateData, date) {
		// TODO: This uses string comparison between ISO date string representation. Correct?
		if (date < dateFrom || date > dateTo) return;

		oForEach(dateData, function (serviceData, serviceName) {
			var serviceResult;

			if (!resultMap[serviceName]) {
				resultMap[serviceName] = {
					businessProcess: {},
					certificate: {},
					processingStep: {}
				};
			}

			serviceResult = resultMap[serviceName];

			// TODO: Is this even possible to have a day that had status events for processing
			// steps and not for business process?
			// if (serviceData.businessProcess) {

			// 1 [serviceName].businessProcess[status]
			storePerStatusResult(serviceData.businessProcess, serviceResult.businessProcess);

			// }

			if (serviceData.certificate) {
				oForEach(serviceData.certificate, function (certificateData, certificateName) {
					if (!serviceResult.certificate[certificateName]) {
						serviceResult.certificate[certificateName] = {};
					}

					// 2 [serviceName].certificate[certificateName][status]
					storePerStatusResult(certificateData, serviceResult.certificate[certificateName]);
				});
			}

			if (serviceData.processingStep) {
				oForEach(serviceData.processingStep, function (stepData, stepName) {
					var stepResult;

					if (!serviceResult.processingStep[stepName]) {
						serviceResult.processingStep[stepName] = {
							pending: { businessProcess: 0, certificate: {} },
							byProcessor: {}
						};
					}

					stepResult = serviceResult.processingStep[stepName];

					// 3.1.1 [serviceName].processingStep[stepName].pending.businessProcess
					storeStatusData(stepResult.pending, 'businessProcess', stepData.pending);

					// 3.1.2 [serviceName].processingStep[stepName].pending.certificate[certificateName]
					// TODO

					oForEach(stepData.byProcessor, function (byProcessorData, processorId) {
						var processorResult;

						if (!stepResult.byProcessor[processorId]) stepResult.byProcessor[processorId] = {};

						processorResult = stepResult.byProcessor[processorId];

						// storePerStatusResult(byProcessorData, processorResult);

						oForEach(byProcessorData, function (statusData, status) {
							if (!processorResult[status]) processorResult[status] = { certificate: {} };

							// 3.2.1 [serviceName].processingStep[stepName]
							//           .byProcessor[processorId][status].businessProcess
							storeStatusData(processorResult[status], 'businessProcess', statusData);
						});

						// 3.2.2 [serviceName].processingStep[stepName]
						//           .byProcessor[processorId][status].certificate[certificateName]
						// TODO
					});
				});
			}
		});
	});

	var eachStatusSetToSum = function (dataSet) {
		Object.keys(dataSet).forEach(function (status) {
			if (isSet(dataSet[status])) {
				dataSet[status] = dataSet[status].size;
			}
		});
	};

	oForEach(resultMap, function (serviceResult) {
		// 1 [serviceName].businessProcess[status]
		eachStatusSetToSum(serviceResult.businessProcess);

		// 2 [serviceName].certificate[certificateName][status]
		oForEach(serviceResult.certificate, eachStatusSetToSum);

		oForEach(serviceResult.processingStep, function (stepResult) {
			// 3.1.1 [serviceName].processingStep[stepName].pending.businessProcess
			stepResult.pending.businessProcess = stepResult.pending.businessProcess.size;
			// 3.1.2 [serviceName].processingStep[stepName].pending.certificate[certificateName]
			// TODO

			// 3.2.1 [serviceName].processingStep[stepName]
			//           .byProcessor[processorId][status].businessProcess
			// TODO: Should end at '.businessProcess'
			oForEach(stepResult.byProcessor, function (processorResult) {
				oForEach(processorResult, eachStatusSetToSum);
			});

			// 3.2.2 [serviceName].processingStep[stepName]
			//           .byProcessor[processorId][status].certificate[certificateName]
			// TODO
		});
	});

	return resultMap;
};

module.exports = memoize(function (dateFrom, dateTo) {
	var driver = require('mano').dbDriver;

	return getData(driver)(function (data) {
		return calculate(getDateMap(data), data, dateFrom, dateTo);
	});
}, { max: 1000, promise: true });<commit_msg>Issue #1758. Add step certificates calculation to calculateStatusEventsSums<commit_after>'use strict';

var oForEach   = require('es5-ext/object/for-each')
  , Set        = require('es6-set')
  , isSet      = require('es6-set/is-set')
  , memoize    = require('memoizee')
  , getData    = require('../business-process-query/get-data')
  , getDateMap = require('../business-process-query/get-status-history-date-map');

/*
Result map of calculateStatusEventsSums:

[serviceName] = {
	// 1
	businessProcess[status]: sum,
	// 2
	certificate[certificateName][status]: sum,
	processingStep[stepName]: {
		// 3.1
		pending: {
			// 3.1.1
			businessProcess: sum,
			// 3.1.2
			certificate[certificateName]: sum
		},
		// 3.2
		byProcessor[processorId][status]: {
			// 3.2.1
			businessProcess: sum,
			// 3.2.2
			certificate[certificateName]: sum
		}
	}
};
 */

var dbDateToISO = function (date) {
	return date.toISOString().substring(0, 10);
};

var calculate = function (dateMap, data, dateFrom, dateTo) {
	var resultMap = {};

	dateFrom = dbDateToISO(dateFrom);
	dateTo = dbDateToISO(dateTo);

	var storeStatusData = function (resultSet, field, statusData) {
		if (Array.isArray(statusData)) {
			if (!resultSet[field]) resultSet[field] = new Set();

			statusData.forEach(Set.prototype.add.bind(resultSet[field]));
		} else {
			if (resultSet[field] == null) resultSet[field] = 0;

			resultSet[field] += statusData;
		}
	};

	var storePerStatusResult = function (data, resultSet) {
		oForEach(data, function (statusData, status) {
			storeStatusData(resultSet, status, statusData);
		});
	};

	oForEach(dateMap, function (dateData, date) {
		// TODO: This uses string comparison between ISO date string representation. Correct?
		if (date < dateFrom || date > dateTo) return;

		oForEach(dateData, function (serviceData, serviceName) {
			var serviceResult;

			if (!resultMap[serviceName]) {
				resultMap[serviceName] = {
					businessProcess: {},
					certificate: {},
					processingStep: {}
				};
			}

			serviceResult = resultMap[serviceName];

			// TODO: Is this even possible to have a day that had status events for processing
			// steps and not for business process?
			// if (serviceData.businessProcess) {

			// 1 [serviceName].businessProcess[status]
			storePerStatusResult(serviceData.businessProcess, serviceResult.businessProcess);

			// }

			if (serviceData.certificate) {
				oForEach(serviceData.certificate, function (certificateData, certificateName) {
					if (!serviceResult.certificate[certificateName]) {
						serviceResult.certificate[certificateName] = {};
					}

					// 2 [serviceName].certificate[certificateName][status]
					storePerStatusResult(certificateData, serviceResult.certificate[certificateName]);
				});
			}

			if (serviceData.processingStep) {
				oForEach(serviceData.processingStep, function (stepData, stepName) {
					var stepResult;

					if (!serviceResult.processingStep[stepName]) {
						serviceResult.processingStep[stepName] = {
							pending: { businessProcess: 0, certificate: {} },
							byProcessor: {}
						};
					}

					stepResult = serviceResult.processingStep[stepName];

					// 3.1.1 [serviceName].processingStep[stepName].pending.businessProcess
					storeStatusData(stepResult.pending, 'businessProcess', stepData.pending);

					stepData.pending.forEach(function (bpId) {
						var certificates = data.businessProcesses.get(bpId).certificates;

						certificates.forEach(function (certificateName) {
							// 3.1.2 [serviceName].processingStep[stepName].pending.certificate[certificateName]
							storeStatusData(stepResult.pending.certificate, certificateName, stepData.pending);
						});
					});

					oForEach(stepData.byProcessor, function (byProcessorData, processorId) {
						var processorResult;

						if (!stepResult.byProcessor[processorId]) stepResult.byProcessor[processorId] = {};

						processorResult = stepResult.byProcessor[processorId];

						oForEach(byProcessorData, function (statusData, status) {
							var statusResult;

							if (!processorResult[status]) processorResult[status] = { certificate: {} };

							statusResult = processorResult[status];

							// 3.2.1 [serviceName].processingStep[stepName]
							//           .byProcessor[processorId][status].businessProcess
							storeStatusData(statusResult, 'businessProcess', statusData);

							statusData.forEach(function (bpId) {
								var certificates = data.businessProcesses.get(bpId).certificates;

								certificates.forEach(function (certificateName) {
									// 3.2.2 [serviceName].processingStep[stepName]
									//           .byProcessor[processorId][status].certificate[certificateName]
									storeStatusData(statusResult.certificate, certificateName, statusData);
								});
							});
						});
					});
				});
			}
		});
	});

	var eachStatusSetToSum = function (dataSet) {
		Object.keys(dataSet).forEach(function (status) {
			if (isSet(dataSet[status])) {
				dataSet[status] = dataSet[status].size;
			}
		});
	};

	oForEach(resultMap, function (serviceResult) {
		// 1 [serviceName].businessProcess[status]
		eachStatusSetToSum(serviceResult.businessProcess);

		// 2 [serviceName].certificate[certificateName][status]
		oForEach(serviceResult.certificate, eachStatusSetToSum);

		oForEach(serviceResult.processingStep, function (stepResult) {
			// 3.1.1 [serviceName].processingStep[stepName].pending.businessProcess
			stepResult.pending.businessProcess = stepResult.pending.businessProcess.size;
			// 3.1.2 [serviceName].processingStep[stepName].pending.certificate[certificateName]
			oForEach(stepResult.pending, eachStatusSetToSum);

			oForEach(stepResult.byProcessor, function (processorResult) {
				// 3.2.1 [serviceName].processingStep[stepName]
				//           .byProcessor[processorId][status].businessProcess
				oForEach(processorResult, eachStatusSetToSum);

				oForEach(processorResult, function (statusResult) {
					// 3.2.2 [serviceName].processingStep[stepName]
					//           .byProcessor[processorId][status].certificate[certificateName]
					oForEach(statusResult, eachStatusSetToSum);
				});
			});
		});
	});

	return resultMap;
};

module.exports = memoize(function (dateFrom, dateTo) {
	var driver = require('mano').dbDriver;

	return getData(driver)(function (data) {
		return calculate(getDateMap(data), data, dateFrom, dateTo);
	});
}, { max: 1000, promise: true });
{'fraction_non_alphanumeric': 0.13321785036707084, 'fraction_numerical': 0.00989853996535511, 'mean_word_length': 3.8341307814992027, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4931753', 'n_tokens_mistral': 4409, 'n_tokens_neox': 3614, 'n_words': 621}