# Path: common/db.py
# def user_db_prefix():
# def database_name(domain, family):
# def use_db(conn, domain, family, cursor_class = None):
# def create_conn(**kwargs):
#     def __init__(self, db_name):
#     def open(self):
#     def close(self):
# def connection(db_obj):
# class UserDb(object):
# 
# Path: jobs/sge_jobs.py
# class DbJob(db.UserDb):
#     def __init__(self):
#     def get_job_table(self, state_filter, limit = 50, offset = 0,
#                       cmd_filter = None):
#     def get_accounting_table(self, limit = 50, offset = 0, job_ids = None):
#     def pending_request(self, limit = 16, offset = 0):
#     def _add_request(self, jobname, run_cmd, args, max_vmem, cpu_bound, force):
#     def add_request(self, jobname, run_cmd, args,  max_vmem,
#                     cpu_bound = True, force = False):
#     def exec_request(self, r):
#     def run_batch(self, nr_running, limit = 16):
#     def _exec_check(self, request):
#     def check_running(self):
#     def search_accounting(self, jobs, last_time_day = 2):
#     def update_accounting(self):
# def quote_arg(arg):
# def job_cmdline_arg(request, cmd):
# def sge_cmdline_arg(request):


import sys
import os
import hashlib
import MySQLdb
from common import db
from ws_namespaces import index as index, namespaces as namespaces
from jobs import sge_jobs

    return result

def prepare_request(db_hocr, lang):
    ns_nr = index_ns_nr(lang)

    conn, cursor = open_db(lang, 'wikisource')
    q = 'SELECT page_title FROM page WHERE page_namespace=%s and page_is_redirect=0'
    cursor.execute(q, ns_nr)

    titles = [ x[0] for x in cursor.fetchall() if x[0].endswith('.djvu') or x[0].endswith('.pdf') ]
    close_db(conn, cursor)

    file_to_sha1 = fetch_file_sha1(lang, titles)

    q = 'SELECT sha1, title FROM hocr WHERE lang=%s'
    db_hocr.cursor.execute(q, (lang, ))
    temp = db_hocr.cursor.fetchall()
    hocr_sha1 = set()
    for f in temp:
        hocr_sha1.add(f['sha1'])

    # attempt to rename or tidy can't work as we lost the old title.
    # and sha1 is not unique in the db. Tidying unused hocr must be done
    # separately.
    for title in file_to_sha1:
        if file_to_sha1[title] not in hocr_sha1:
            print lang, title, file_to_sha1[title]
            add_hocr_request(lang, title)


# Path: common/db.py
# def user_db_prefix():
# def database_name(domain, family):
# def use_db(conn, domain, family, cursor_class = None):
# def create_conn(**kwargs):
#     def __init__(self, db_name):
#     def open(self):
#     def close(self):
# def connection(db_obj):
# class UserDb(object):
# 
# Path: jobs/sge_jobs.py
# class DbJob(db.UserDb):
#     def __init__(self):
#     def get_job_table(self, state_filter, limit = 50, offset = 0,
#                       cmd_filter = None):
#     def get_accounting_table(self, limit = 50, offset = 0, job_ids = None):
#     def pending_request(self, limit = 16, offset = 0):
#     def _add_request(self, jobname, run_cmd, args, max_vmem, cpu_bound, force):
#     def add_request(self, jobname, run_cmd, args,  max_vmem,
#                     cpu_bound = True, force = False):
#     def exec_request(self, r):
#     def run_batch(self, nr_running, limit = 16):
#     def _exec_check(self, request):
#     def check_running(self):
#     def search_accounting(self, jobs, last_time_day = 2):
#     def update_accounting(self):
# def quote_arg(arg):
# def job_cmdline_arg(request, cmd):
# def sge_cmdline_arg(request):


# Path: hocr/hocr_request.py
import sys
import os
import hashlib
import MySQLdb
from common import db
from ws_namespaces import index as index, namespaces as namespaces
from jobs import sge_jobs


    return result

def prepare_request(db_hocr, lang):
    ns_nr = index_ns_nr(lang)

    conn, cursor = open_db(lang, 'wikisource')
    q = 'SELECT page_title FROM page WHERE page_namespace=%s and page_is_redirect=0'
    cursor.execute(q, ns_nr)

    titles = [ x[0] for x in cursor.fetchall() if x[0].endswith('.djvu') or x[0].endswith('.pdf') ]
    close_db(conn, cursor)

    file_to_sha1 = fetch_file_sha1(lang, titles)

    q = 'SELECT sha1, title FROM hocr WHERE lang=%s'
    db_hocr.cursor.execute(q, (lang, ))
    temp = db_hocr.cursor.fetchall()
    hocr_sha1 = set()
    for f in temp:
        hocr_sha1.add(f['sha1'])

    # attempt to rename or tidy can't work as we lost the old title.
    # and sha1 is not unique in the db. Tidying unused hocr must be done
    # separately.
    for title in file_to_sha1:
        if file_to_sha1[title] not in hocr_sha1:
            print lang, title, file_to_sha1[title]
            add_hocr_request(lang, title)


class DbHocr(db.UserDb):