<commit_before>#!/usr/bin/python

import sys
import os
import psycopg2
import psycopg2.extensions
import psycopg2.extras
from optparse import OptionParser

# backfills architecture and core information into reports_clean
# upgrade script for 2.4.X
# note that the columns should be created in a separate SQL statement

parser = OptionParser()
parser.add_option("-s", "--start", dest="startdate",
                  help="oldest date to backfill to", metavar="DATE",
                  default="2011-12-23")
parser.add_option("-d", "--database", dest="database_name",
                  help="database to be extracted", metavar="DBNAME",
                  default="breakpad")
parser.add_option("-m", "--memory", dest="dbmemory",
                  help="amount of memory to set for operations", metavar="MEM",
                  default="1GB")
(options, args) = parser.parse_args()

print "Backfilling reports_clean, one week at a time."

#connect to postgresql
conn = psycopg2.connect("dbname=%s user=postgres" % options.database_name)

conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)

cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

cur.execute("""SET maintenance_work_mem = %s""", ( options.dbmemory, ) )

#check if we've already been run
cur.execute("""SELECT count(*) FROM (
	SELECT architecture, cores 
	FROM reports_clean 
	WHERE date_processed > %s 
	LIMIT 100) 
	WHERE architecture IS NOT NULL""")
	
if cur.fetchone()[0] > 50:
	print "architecture and cores are already backfilled, exiting"
	sys.exit(0)

cur.execute("""SELECT rcparts.relname as rcname, repparts.relname as repname
FROM pg_stat_user_tables AS rcparts
	JOIn pg_stat_user_tables AS repparts
		ON week_begins_partition(rcparts.relname) =
			week_begins_partition(repparts.relname)
WHERE rcparts.relname LIKE 'reports_clean_20%%'
	and repparts.relname LIKE 'reports_20%%'
	and week_ends_partition(rcparts.relname) >= %s
ORDER BY rcparts.relname;""", ( options.startdate, ))

partitions = cur.fetchall()
<commit_msg>Committing fix to backfill script per deployment testing.<commit_after>#!/usr/bin/python

import sys
import os
import psycopg2
import psycopg2.extensions
import psycopg2.extras
from optparse import OptionParser

# backfills architecture and core information into reports_clean
# upgrade script for 2.4.X
# note that the columns should be created in a separate SQL statement

parser = OptionParser()
parser.add_option("-s", "--start", dest="startdate",
                  help="oldest date to backfill to", metavar="DATE",
                  default="2011-12-23")
parser.add_option("-d", "--database", dest="database_name",
                  help="database to be extracted", metavar="DBNAME",
                  default="breakpad")
parser.add_option("-m", "--memory", dest="dbmemory",
                  help="amount of memory to set for operations", metavar="MEM",
                  default="1GB")
(options, args) = parser.parse_args()

print "Backfilling reports_clean, one week at a time."

#connect to postgresql
conn = psycopg2.connect("dbname=%s user=postgres" % options.database_name)

conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)

cur = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

cur.execute("""SET maintenance_work_mem = %s""", ( options.dbmemory, ) )

#check if we've already been run
cur.execute("""SELECT count(*) FROM (
	SELECT architecture, cores 
	FROM reports_clean 
	WHERE date_processed > %s 
	LIMIT 100) as sample
	WHERE architecture IS NOT NULL""", (options.startdate,))
	
if cur.fetchone()[0] > 50:
	print "architecture and cores are already backfilled, exiting"
	sys.exit(0)

cur.execute("""SELECT rcparts.relname as rcname, repparts.relname as repname
FROM pg_stat_user_tables AS rcparts
	JOIn pg_stat_user_tables AS repparts
		ON week_begins_partition(rcparts.relname) =
			week_begins_partition(repparts.relname)
WHERE rcparts.relname LIKE 'reports_clean_20%%'
	and repparts.relname LIKE 'reports_20%%'
	and week_ends_partition(rcparts.relname) >= %s
ORDER BY rcparts.relname;""", ( options.startdate, ))

partitions = cur.fetchall()

{'fraction_non_alphanumeric': 0.11324422843256379, 'fraction_numerical': 0.013608748481166465, 'mean_word_length': 4.41578947368421, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5513309', 'n_tokens_mistral': 1352, 'n_tokens_neox': 1272, 'n_words': 344}