# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):
# 
# Path: common/qstat.py
# def running_jobs(job_base_name):
# 
# Path: common/db.py
# def user_db_prefix():
# def database_name(domain, family):
# def use_db(conn, domain, family, cursor_class = None):
# def create_conn(**kwargs):
#     def __init__(self, db_name):
#     def open(self):
#     def close(self):
# def connection(db_obj):
# class UserDb(object):


import sys
import os
import json
import hashlib
import time
import subprocess
import re
import collections
from common import utils
from common import qstat
from common import db
    def _exec_check(self, request):
        q = 'UPDATE job SET job_state="accounting" WHERE job_id=%s'
        self.cursor.execute(q, [ request['job_id'] ])

        q = 'INSERT into accounting (job_id, sge_jobnumber) VALUE (%s, %s)'
        self.cursor.execute(q, [ request['job_id'], request['sge_jobnumber'] ])

        self.conn.commit()

    def check_running(self):
        sge_running = qstat.running_jobs('')
        if sge_running:
            with db.connection(self):
                q = 'SELECT job_id, sge_jobnumber, job_args FROM job WHERE job_state="running"'
                self.cursor.execute(q)
                for r in self.cursor.fetchall():
                    if not r['sge_jobnumber'] in sge_running:
                        self._exec_check(r)
            return len(sge_running)
        return None

    # Limiting is necessary because a job can be finished but not yet in the
    # accouting file (cache effect) so we can easily scan the whole file. To
    # avoid that we limit the backward search to two days by default.
    # float is allowed so last_time_day = 1.0/24 is an hour.
    def search_accounting(self, jobs, last_time_day = 2):
        last_time_day = max(1.0/24, last_time_day)
        now = int(time.time())
        count = 0
        nr_job = len(jobs)

# Path: common/utils.py
# def read_file(filename):
# def write_file(filename, text):
# def save_obj(filename, data):
# def load_obj(filename):
# def sha1(filename):
# def write_sha1(sha1, filename):
# def url_opener():
# def copy_file_from_url(url, out_file, expect_sha1 = None, max_retry = 4):
# def compress_file_data(out_filename, data, compress_type):
# def compress_file(out_filename, in_filename, compress_type):
# def uncompress_file(filename, compress_type):
# def _retry_on_eintr(func, *args):
# def safe_read(fd):
# def safe_write(fd, text):
# def print_traceback(*kwargs):
# def readline_backward(filename, buf_size=8192):
# 
# Path: common/qstat.py
# def running_jobs(job_base_name):
# 
# Path: common/db.py
# def user_db_prefix():
# def database_name(domain, family):
# def use_db(conn, domain, family, cursor_class = None):
# def create_conn(**kwargs):
#     def __init__(self, db_name):
#     def open(self):
#     def close(self):
# def connection(db_obj):
# class UserDb(object):


# Path: jobs/sge_jobs.py
import sys
import os
import json
import hashlib
import time
import subprocess
import re
import collections
from common import utils
from common import qstat
from common import db

    def _exec_check(self, request):
        q = 'UPDATE job SET job_state="accounting" WHERE job_id=%s'
        self.cursor.execute(q, [ request['job_id'] ])

        q = 'INSERT into accounting (job_id, sge_jobnumber) VALUE (%s, %s)'
        self.cursor.execute(q, [ request['job_id'], request['sge_jobnumber'] ])

        self.conn.commit()

    def check_running(self):
        sge_running = qstat.running_jobs('')
        if sge_running:
            with db.connection(self):
                q = 'SELECT job_id, sge_jobnumber, job_args FROM job WHERE job_state="running"'
                self.cursor.execute(q)
                for r in self.cursor.fetchall():
                    if not r['sge_jobnumber'] in sge_running:
                        self._exec_check(r)
            return len(sge_running)
        return None

    # Limiting is necessary because a job can be finished but not yet in the
    # accouting file (cache effect) so we can easily scan the whole file. To
    # avoid that we limit the backward search to two days by default.
    # float is allowed so last_time_day = 1.0/24 is an hour.
    def search_accounting(self, jobs, last_time_day = 2):
        last_time_day = max(1.0/24, last_time_day)
        now = int(time.time())
        count = 0
        nr_job = len(jobs)

        for line in utils.readline_backward(accounting_path):