<commit_before>#!/usr/bin/python

import requests
import ipdb
import mc_config
import psycopg2
import psycopg2.extras
import time

#assert pkg_resources.get_distribution("requests").version >= '1.2.3'

def get_db_info():
    config_file = mc_config.read_config()
    return config_file['database'][0]

def connect_to_database():
    db_info = get_db_info()
    conn = psycopg2.connect( database=db_info['db'], user=db_info['user'], password=db_info['pass'], host=db_info['host'], port=db_info['port'] )
    return conn
    

conn = connect_to_database()

cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

pg_last_import_id_var = 'LAST_REIMPORTED_STORY_SENTENCES_ID';

cursor.execute("SELECT * from database_variables where name = %s ", (pg_last_import_id_var,) )
#ipdb.set_trace()

result = cursor.fetchone()
if result == None:
    cursor.execute( "SELECT min(story_sentences_id)  from story_sentences")
    min_story_sentences_id = cursor.fetchone()['min']
    print min_story_sentences_id 
    cursor.execute("INSERT INTO database_variables(name, value) VALUES( %(name)s, %(value)s )", 
                   { 'name': pg_last_import_id_var, 'value': min_story_sentences_id - 1 } )
    conn.commit()
else:
    min_story_sentences_id = int(result['value'])

while True:
    r = requests.get( 'http://localhost:8983/solr/collection1/dataimport?command=status&wt=json', headers = { 'Accept': 'application/json'})  
    data = r.json()

    if data['status'] != 'busy':
        print data
        print data['status']

        cursor.execute("UPDATE database_variables set value = %(value)s where name = %(name)s ", 
                   { 'name': pg_last_import_id_var, 'value': min_story_sentences_id - 1 } )
        

        cursor.execute(
            """UPDATE story_sentences set db_row_last_updated = now() where """ 
            """ story_sentences_id > (select value::integer from database_variables where name = %s ) """ 
            """and story_sentences_id <= (select value::integer from database_variables where name = %s ) + 1000000 """,
            (  pg_last_import_id_var,  pg_last_import_id_var) )

        conn.commit

        print "importing with min_story_sentences_id {} ...".format( min_story_sentences_id )

        min_story_sentences_id += 1000000

        requests.get( 'http://localhost:8983/solr/collection1/dataimport?command=full-import&commit=true&clean=false' )
    else:
        print "import busy"

    print "sleeping"

    time.sleep( 20 )<commit_msg>Add commits. Make the import batch size a variable instead of the hard coded param.<commit_after>#!/usr/bin/python

import requests
import ipdb
import mc_config
import psycopg2
import psycopg2.extras
import time

#assert pkg_resources.get_distribution("requests").version >= '1.2.3'

def get_db_info():
    config_file = mc_config.read_config()
    return config_file['database'][0]

def connect_to_database():
    db_info = get_db_info()
    conn = psycopg2.connect( database=db_info['db'], user=db_info['user'], password=db_info['pass'], host=db_info['host'], port=db_info['port'] )
    return conn
    

conn = connect_to_database()

cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

pg_last_import_id_var = 'LAST_REIMPORTED_STORY_SENTENCES_ID';

cursor.execute("SELECT * from database_variables where name = %s ", (pg_last_import_id_var,) )
#ipdb.set_trace()

result = cursor.fetchone()
if result == None:
    cursor.execute( "SELECT min(story_sentences_id)  from story_sentences")
    min_story_sentences_id = cursor.fetchone()['min']
    print min_story_sentences_id 
    cursor.execute("INSERT INTO database_variables(name, value) VALUES( %(name)s, %(value)s )", 
                   { 'name': pg_last_import_id_var, 'value': min_story_sentences_id - 1 } )
    conn.commit()
else:
    min_story_sentences_id = int(result['value'])

print "min_story_sentences_id: {} ".format( min_story_sentences_id)

import_batch_size = 10000

while True:
    r = requests.get( 'http://localhost:8983/solr/collection1/dataimport?command=status&wt=json', headers = { 'Accept': 'application/json'})  
    data = r.json()

    if data['status'] != 'busy':
        print data
        print data['status']

        cursor.execute("UPDATE database_variables set value = %(value)s where name = %(name)s ", 
                   { 'name': pg_last_import_id_var, 'value': min_story_sentences_id - 1 } )
        

        conn.commit()

        print "Updating db_row_last_updated time for story sentences {}".format(min_story_sentences_id)

        cursor.execute(
            """UPDATE story_sentences set db_row_last_updated = now() where """ 
            """ story_sentences_id > (select value::integer from database_variables where name = %s ) """ 
            """and story_sentences_id <= (select value::integer from database_variables where name = %s ) + %s """,
            (  pg_last_import_id_var,  pg_last_import_id_var, import_batch_size) )

        conn.commit()

        print "importing with min_story_sentences_id {} ...".format( min_story_sentences_id )

        min_story_sentences_id += import_batch_size

        requests.get( 'http://localhost:8983/solr/collection1/dataimport?command=full-import&commit=true&clean=false' )
    else:
        print "import busy"

    print "sleeping"

    time.sleep( 20 )
{'fraction_non_alphanumeric': 0.12429484768710042, 'fraction_numerical': 0.01184655885671305, 'mean_word_length': 3.8442622950819674, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6622951', 'n_tokens_mistral': 1788, 'n_tokens_neox': 1708, 'n_words': 340}