<commit_before>DROP FUNCTION backfill_matviews ( date, text, date, boolean);

\set ON_ERROR_STOP 1

-- function

CREATE OR REPLACE FUNCTION backfill_matviews (
	firstday date,
	lastday date default NULL,
	reportsclean boolean default true )
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $f$
DECLARE thisday DATE := firstday;
BEGIN
-- this producedure is meant to be called manually
-- by administrators in order to clear and backfill
-- the various matviews in order to recalculate old
-- data which was erroneous.
-- it requires a start date, and optionally an end date
-- no longer takes a product parameter
-- optionally disable reports_clean backfill
-- since that takes a long time

-- VERSION 0.3

-- set optional end date
lastday := coalesce(lastday, current_date );

-- check parameters
IF firstday > current_date OR lastday > current_date or firstday > lastday THEN
	RAISE EXCEPTION 'date parameter error: cannot backfill into the future';
END IF;

-- fill in products
PERFORM update_product_versions();

IF forproduct <> '' THEN
	PERFORM 1 FROM products WHERE product_name = forproduct;
	IF NOT FOUND THEN
		RAISE EXCEPTION 'product % does not exist or is misspelled',forproduct;
	END IF;
END IF;

-- backfill reports_clean.  this takes a while,  and isn't limited
-- by product so if it's not needed
-- we provide a switch to disable it
IF reportsclean THEN
	RAISE INFO 'backfilling reports_clean';
	PERFORM backfill_reports_clean_by_date( firstday, lastday );
END IF;

-- loop through the days, backfilling one at a time
WHILE thisday <= lastday LOOP
	RAISE INFO 'backfilling other matviews for %',thisday;
	RAISE INFO 'adu';
	PERFORM backfill_adu(thisday);<commit_msg>More bug fixes per testing.<commit_after>DROP FUNCTION backfill_matviews ( date, text, date, boolean);

\set ON_ERROR_STOP 1

-- function

CREATE OR REPLACE FUNCTION backfill_matviews (
	firstday date,
	lastday date default NULL,
	reportsclean boolean default true )
RETURNS BOOLEAN
LANGUAGE plpgsql
AS $f$
DECLARE thisday DATE := firstday;
BEGIN
-- this producedure is meant to be called manually
-- by administrators in order to clear and backfill
-- the various matviews in order to recalculate old
-- data which was erroneous.
-- it requires a start date, and optionally an end date
-- no longer takes a product parameter
-- optionally disable reports_clean backfill
-- since that takes a long time

-- VERSION 0.3

-- set optional end date
lastday := coalesce(lastday, current_date );

-- check parameters
IF firstday > current_date OR lastday > current_date or firstday > lastday THEN
	RAISE EXCEPTION 'date parameter error: cannot backfill into the future';
END IF;

-- fill in products
PERFORM update_product_versions();

-- backfill reports_clean.  this takes a while,  and isn't limited
-- by product so if it's not needed
-- we provide a switch to disable it
IF reportsclean THEN
	RAISE INFO 'backfilling reports_clean';
	PERFORM backfill_reports_clean_by_date( firstday, lastday );
END IF;

-- loop through the days, backfilling one at a time
WHILE thisday <= lastday LOOP
	RAISE INFO 'backfilling other matviews for %',thisday;
	RAISE INFO 'adu';
	PERFORM backfill_adu(thisday);
{'fraction_non_alphanumeric': 0.06704617330803289, 'fraction_numerical': 0.002213788741302973, 'mean_word_length': 4.33389544688027, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2258517', 'n_tokens_mistral': 971, 'n_tokens_neox': 927, 'n_words': 412}