<commit_before>/**
 * Data API.
 *
 * Site Kit by Google, Copyright 2019 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * External dependencies
 */
import DashboardAuthAlert from 'GoogleComponents/notifications/dashboard-auth-alert';
import DashboardPermissionAlert from 'GoogleComponents/notifications/dashboard-permission-alert';
import md5 from 'md5';
import {
	getStorage,
	getCurrentDateRangeSlug,
	fillFilterWithComponent,
	getQueryParameter,
	sortObjectProperties,
} from 'SiteKitCore/util';
import { cloneDeep, each, intersection, isEqual, sortBy } from 'lodash';

/**
 * WordPress dependencies
 */
import apiFetch from '@wordpress/api-fetch';
import { addQueryArgs } from '@wordpress/url';
import { addAction, applyFilters, doAction, addFilter, removeFilter } from '@wordpress/hooks';

export const TYPE_CORE = 'core';
export const TYPE_MODULES = 'modules';

/**
 * Ensures that the local datacache object is properly set up.
 */
const lazilySetupLocalCache = () => {
	global.googlesitekit.admin = global.googlesitekit.admin || {};

	if ( 'string' === typeof global.googlesitekit.admin.datacache ) {
		global.googlesitekit.admin.datacache = JSON.parse( global.googlesitekit.admin.datacache );
	}

	if ( 'object' !== typeof global.googlesitekit.admin.datacache ) {
		global.googlesitekit.admin.datacache = {};
	}
};

/**
 * Gets a copy of the given data request object with the data.dateRange populated via filter, if not set.
 * Respects the current dateRange value, if set.
 *
 * @param {Object} originalRequest Data request object.
 * @param {string} dateRange Default date range slug to use if not specified in the request.
 * @return {Object} New data request object.
 */
const requestWithDateRange = ( originalRequest, dateRange ) => {
	// Make copies for reference safety, ensuring data exists.
	const request = { data: {}, ...originalRequest };
	// Use the dateRange in request.data if passed, fallback to provided default value.
	request.data = { dateRange, ...request.data };

	return request;
};

const dataAPI = {

	maxRequests: 10,

	init() {
		if ( global.googlesitekit.initialized ) {
			return;
		}
		global.googlesitekit.initialized = true;
		this.collectModuleData = this.collectModuleData.bind( this );
		global.googlesitekit.cache = [];

		addAction(
			'googlesitekit.moduleLoaded',
			'googlesitekit.collectModuleListingData',
			this.collectModuleData
		);
	},

	/**
	 * Gets data for multiple requests from the cache in a single batch process.
	 *
	 * This is a replica of combinedGet but only fetching data from cache. No requests are done.
	 * Solves issue for publisher wins to retrieve data without performing additional requests.
	 * Likely this will be removed after refactoring.
	 *
 	 * @param {Array.<{ maxAge: timestamp, type: string, identifier: string, datapoint: string, callback: function }>} combinedRequest An array of data requests to resolve.
	 *
	 * @return {Promise} A promise for the cache lookup.
	 */
	combinedGetFromCache( combinedRequest ) {
		return new Promise( ( resolve, reject ) => {
			try {
				const responseData = [];
				const dateRange = getCurrentDateRangeSlug();
				each( combinedRequest, ( originalRequest ) => {
					const request = requestWithDateRange( originalRequest, dateRange );
					request.key = this.getCacheKey( request.type, request.identifier, request.datapoint, request.data );
					const cache = this.getCache( request.key, request.maxAge );

					if ( 'undefined' !== typeof cache ) {
						responseData[ request.key ] = cache;

						this.resolve( request, cache );
					}
				} );

				resolve( responseData );
			} catch ( err ) {
				reject();
			}
		} );
	},

	/**
	 * Gets data for multiple requests from the REST API using a single batch process.
	 *
 	 * @param {Array.<{ maxAge: timestamp, type: string, identifier: string, datapoint: string, callback: function }>} combinedRequest An array of data requests to resolve.
	 * @param {boolean} secondaryRequest Is this the second (or more) request?
	 */
	combinedGet( combinedRequest, secondaryRequest = false ) {
		// First, resolve any cache matches immediately, queue resolution of the rest.
		let dataRequest = [];
		let cacheDelay = 25;
		const dateRange = getCurrentDateRangeSlug();
		each( combinedRequest, ( originalRequest ) => {
			const request = requestWithDateRange( originalRequest, dateRange );
			request.key = this.getCacheKey( request.type, request.identifier, request.datapoint, request.data );
			const cache = this.getCache( request.key, request.maxAge );

			if ( 'undefined' !== typeof cache ) {
				setTimeout( () => {
					this.resolve( request, cache );
				}, cacheDelay );
				cacheDelay += 25;
			} else {
				// Add the request to the queue.
				dataRequest.push( request );
			}
		} );

		// Sort the modules by priority.
		dataRequest = sortBy( dataRequest, 'priority' );

		// Only request each key once.
		const toRequest = [];
		const deferredRequests = [];
		const keyIndexesMap = {};
		const noLowPriorityRequests = !! dataRequest.find( ( request ) => {
			return request.priority < 10;
		} );
		each( dataRequest, ( request, index ) => {
			// Defer any datapoints with a priority of 10 or greater into a second request.
			if ( ! secondaryRequest && 10 <= request.priority && noLowPriorityRequests ) {
				deferredRequests.push( request );
			} else if ( ! keyIndexesMap[ request.key ] ) {
				keyIndexesMap[ request.key ] = [ index ];
				toRequest.push( request );
			} else {
				keyIndexesMap[ request.key ].push( index );
			}
		} );

		const maxDatapointsPerRequest = 10;
		const currentRequest = toRequest.slice( 0, maxDatapointsPerRequest );
		let remainingDatapoints = toRequest.slice( maxDatapointsPerRequest );

		remainingDatapoints = remainingDatapoints.concat( deferredRequests );

		if ( 0 === currentRequest.length && 0 === remainingDatapoints.length ) {
			// Trigger an action indicating this data load completed from the cache.
			doAction( 'googlesitekit.dataLoaded', 'cache' );
			return;
		}

		// Fetch the remaining datapoints in another request.
		if ( 0 < remainingDatapoints.length && 0 < this.maxRequests-- ) {
			setTimeout( () => {
				this.combinedGet( remainingDatapoints, true );
			}, 50 );
		} else {
			this.maxRequests = 10;
		}

		const datacache = null !== getQueryParameter( 'datacache' );
		return apiFetch( {
			path: addQueryArgs( '/google-site-kit/v1/data/', { datacache: datacache || undefined } ),
			data: { request: currentRequest },
			method: 'POST',
		} ).then( ( results ) => {
			each( results, ( result, key ) => {
				if ( ! keyIndexesMap[ key ] ) {
					console.error( 'data_error', 'unknown response key ' + key ); // eslint-disable-line no-console
					return;
				}

				this.handleWPError( result );

				each( keyIndexesMap[ key ], ( index ) => {
					const request = dataRequest[ index ];

					this.setCache( request.key, result );
					this.resolve( request, result );
				} );

				// Trigger an action indicating this data load completed from the API.
				if ( 0 === remainingDatapoints.length ) {
					doAction( 'googlesitekit.dataLoaded', 'api' );
				}
			} );

			// Resolve any returned data requests, then re-request the remainder after a pause.
		} ).catch( ( err ) => {
			// Handle the error and give up trying.
			console.warn( 'Error caught during combinedGet', err ); // eslint-disable-line no-console
		} );
	},

	handleWPError( error ) {
		const wpErrorKeys = [ 'code', 'data', 'message' ];
		const commonKeys = intersection( wpErrorKeys, Object.keys( error ) );
		if ( ! isEqual( wpErrorKeys, commonKeys ) ) {
			return;
		}

		// eslint-disable-next-line no-console
		console.warn( 'WP Error in data response', error );
		const { data } = error;

		if ( ! data.reason ) {
			return;
		}

		let addedNoticeCount = 0;

		// Add insufficient scopes warning.
		if ( [ 'authError', 'insufficientPermissions' ].includes( data.reason ) ) {
			addFilter( 'googlesitekit.ErrorNotification',
				'googlesitekit.AuthNotification',
				fillFilterWithComponent( DashboardAuthAlert ), 1 );
			addedNoticeCount++;
		}

		// Insufficient access permissions.
		if ( 'forbidden' === data.reason ) {
			addFilter( 'googlesitekit.ErrorNotification',
				'googlesitekit.AuthNotification',
				fillFilterWithComponent( DashboardPermissionAlert ), 1 );
			addedNoticeCount++;<commit_msg>Prevent possible error if WP_Error has no reason.<commit_after>/**
 * Data API.
 *
 * Site Kit by Google, Copyright 2019 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
/**
 * External dependencies
 */
import DashboardAuthAlert from 'GoogleComponents/notifications/dashboard-auth-alert';
import DashboardPermissionAlert from 'GoogleComponents/notifications/dashboard-permission-alert';
import md5 from 'md5';
import {
	getStorage,
	getCurrentDateRangeSlug,
	fillFilterWithComponent,
	getQueryParameter,
	sortObjectProperties,
} from 'SiteKitCore/util';
import { cloneDeep, each, intersection, isEqual, sortBy } from 'lodash';

/**
 * WordPress dependencies
 */
import apiFetch from '@wordpress/api-fetch';
import { addQueryArgs } from '@wordpress/url';
import { addAction, applyFilters, doAction, addFilter, removeFilter } from '@wordpress/hooks';

export const TYPE_CORE = 'core';
export const TYPE_MODULES = 'modules';

/**
 * Ensures that the local datacache object is properly set up.
 */
const lazilySetupLocalCache = () => {
	global.googlesitekit.admin = global.googlesitekit.admin || {};

	if ( 'string' === typeof global.googlesitekit.admin.datacache ) {
		global.googlesitekit.admin.datacache = JSON.parse( global.googlesitekit.admin.datacache );
	}

	if ( 'object' !== typeof global.googlesitekit.admin.datacache ) {
		global.googlesitekit.admin.datacache = {};
	}
};

/**
 * Gets a copy of the given data request object with the data.dateRange populated via filter, if not set.
 * Respects the current dateRange value, if set.
 *
 * @param {Object} originalRequest Data request object.
 * @param {string} dateRange Default date range slug to use if not specified in the request.
 * @return {Object} New data request object.
 */
const requestWithDateRange = ( originalRequest, dateRange ) => {
	// Make copies for reference safety, ensuring data exists.
	const request = { data: {}, ...originalRequest };
	// Use the dateRange in request.data if passed, fallback to provided default value.
	request.data = { dateRange, ...request.data };

	return request;
};

const dataAPI = {

	maxRequests: 10,

	init() {
		if ( global.googlesitekit.initialized ) {
			return;
		}
		global.googlesitekit.initialized = true;
		this.collectModuleData = this.collectModuleData.bind( this );
		global.googlesitekit.cache = [];

		addAction(
			'googlesitekit.moduleLoaded',
			'googlesitekit.collectModuleListingData',
			this.collectModuleData
		);
	},

	/**
	 * Gets data for multiple requests from the cache in a single batch process.
	 *
	 * This is a replica of combinedGet but only fetching data from cache. No requests are done.
	 * Solves issue for publisher wins to retrieve data without performing additional requests.
	 * Likely this will be removed after refactoring.
	 *
 	 * @param {Array.<{ maxAge: timestamp, type: string, identifier: string, datapoint: string, callback: function }>} combinedRequest An array of data requests to resolve.
	 *
	 * @return {Promise} A promise for the cache lookup.
	 */
	combinedGetFromCache( combinedRequest ) {
		return new Promise( ( resolve, reject ) => {
			try {
				const responseData = [];
				const dateRange = getCurrentDateRangeSlug();
				each( combinedRequest, ( originalRequest ) => {
					const request = requestWithDateRange( originalRequest, dateRange );
					request.key = this.getCacheKey( request.type, request.identifier, request.datapoint, request.data );
					const cache = this.getCache( request.key, request.maxAge );

					if ( 'undefined' !== typeof cache ) {
						responseData[ request.key ] = cache;

						this.resolve( request, cache );
					}
				} );

				resolve( responseData );
			} catch ( err ) {
				reject();
			}
		} );
	},

	/**
	 * Gets data for multiple requests from the REST API using a single batch process.
	 *
 	 * @param {Array.<{ maxAge: timestamp, type: string, identifier: string, datapoint: string, callback: function }>} combinedRequest An array of data requests to resolve.
	 * @param {boolean} secondaryRequest Is this the second (or more) request?
	 */
	combinedGet( combinedRequest, secondaryRequest = false ) {
		// First, resolve any cache matches immediately, queue resolution of the rest.
		let dataRequest = [];
		let cacheDelay = 25;
		const dateRange = getCurrentDateRangeSlug();
		each( combinedRequest, ( originalRequest ) => {
			const request = requestWithDateRange( originalRequest, dateRange );
			request.key = this.getCacheKey( request.type, request.identifier, request.datapoint, request.data );
			const cache = this.getCache( request.key, request.maxAge );

			if ( 'undefined' !== typeof cache ) {
				setTimeout( () => {
					this.resolve( request, cache );
				}, cacheDelay );
				cacheDelay += 25;
			} else {
				// Add the request to the queue.
				dataRequest.push( request );
			}
		} );

		// Sort the modules by priority.
		dataRequest = sortBy( dataRequest, 'priority' );

		// Only request each key once.
		const toRequest = [];
		const deferredRequests = [];
		const keyIndexesMap = {};
		const noLowPriorityRequests = !! dataRequest.find( ( request ) => {
			return request.priority < 10;
		} );
		each( dataRequest, ( request, index ) => {
			// Defer any datapoints with a priority of 10 or greater into a second request.
			if ( ! secondaryRequest && 10 <= request.priority && noLowPriorityRequests ) {
				deferredRequests.push( request );
			} else if ( ! keyIndexesMap[ request.key ] ) {
				keyIndexesMap[ request.key ] = [ index ];
				toRequest.push( request );
			} else {
				keyIndexesMap[ request.key ].push( index );
			}
		} );

		const maxDatapointsPerRequest = 10;
		const currentRequest = toRequest.slice( 0, maxDatapointsPerRequest );
		let remainingDatapoints = toRequest.slice( maxDatapointsPerRequest );

		remainingDatapoints = remainingDatapoints.concat( deferredRequests );

		if ( 0 === currentRequest.length && 0 === remainingDatapoints.length ) {
			// Trigger an action indicating this data load completed from the cache.
			doAction( 'googlesitekit.dataLoaded', 'cache' );
			return;
		}

		// Fetch the remaining datapoints in another request.
		if ( 0 < remainingDatapoints.length && 0 < this.maxRequests-- ) {
			setTimeout( () => {
				this.combinedGet( remainingDatapoints, true );
			}, 50 );
		} else {
			this.maxRequests = 10;
		}

		const datacache = null !== getQueryParameter( 'datacache' );
		return apiFetch( {
			path: addQueryArgs( '/google-site-kit/v1/data/', { datacache: datacache || undefined } ),
			data: { request: currentRequest },
			method: 'POST',
		} ).then( ( results ) => {
			each( results, ( result, key ) => {
				if ( ! keyIndexesMap[ key ] ) {
					console.error( 'data_error', 'unknown response key ' + key ); // eslint-disable-line no-console
					return;
				}

				this.handleWPError( result );

				each( keyIndexesMap[ key ], ( index ) => {
					const request = dataRequest[ index ];

					this.setCache( request.key, result );
					this.resolve( request, result );
				} );

				// Trigger an action indicating this data load completed from the API.
				if ( 0 === remainingDatapoints.length ) {
					doAction( 'googlesitekit.dataLoaded', 'api' );
				}
			} );

			// Resolve any returned data requests, then re-request the remainder after a pause.
		} ).catch( ( err ) => {
			// Handle the error and give up trying.
			console.warn( 'Error caught during combinedGet', err ); // eslint-disable-line no-console
		} );
	},

	handleWPError( error ) {
		const wpErrorKeys = [ 'code', 'data', 'message' ];
		const commonKeys = intersection( wpErrorKeys, Object.keys( error ) );
		if ( ! isEqual( wpErrorKeys, commonKeys ) ) {
			return;
		}

		// eslint-disable-next-line no-console
		console.warn( 'WP Error in data response', error );
		const { data } = error;

		if ( ! data || ! data.reason ) {
			return;
		}

		let addedNoticeCount = 0;

		// Add insufficient scopes warning.
		if ( [ 'authError', 'insufficientPermissions' ].includes( data.reason ) ) {
			addFilter( 'googlesitekit.ErrorNotification',
				'googlesitekit.AuthNotification',
				fillFilterWithComponent( DashboardAuthAlert ), 1 );
			addedNoticeCount++;
		}

		// Insufficient access permissions.
		if ( 'forbidden' === data.reason ) {
			addFilter( 'googlesitekit.ErrorNotification',
				'googlesitekit.AuthNotification',
				fillFilterWithComponent( DashboardPermissionAlert ), 1 );
			addedNoticeCount++;
{'fraction_non_alphanumeric': 0.11661413777577667, 'fraction_numerical': 0.004277352543899144, 'mean_word_length': 3.5986024844720497, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3498645', 'n_tokens_mistral': 5854, 'n_tokens_neox': 5111, 'n_words': 1581}