<commit_before># -*- coding: utf-8 -*-

"""From proprietary import call:
    - param=init_param() and init_import(param, cnt_total) before the import
    - import functions:
        -- added, answer_id = update_or_insert_answer()
        -- owned_book_id = update_or_insert_owned_book()
        -- update_or_insert_impressions()
    - counter_and_commit_if_100(param, added) with each imported book;
            added is (1st tuple item of the) result from update_or_insert_answer()
    - finished(param) after the import

Dependent on pydal, but independent on the (rest of) Web2py framework.
    The exception is use of current., which makes possible calls from Web2py framework without additional parameters;
    implicit parameter set via current. is always handled at the beginning of the func
"""

import datetime
import hashlib

from gluon import current

from c_db import PublLengths
from c_utils import ean_to_fbi, suplemental_fbi
from dal_idx import idx_row
from dal_utils import answer_by_ean, answer_by_hash, get_library

if False:  # for IDE only, need web2py/__init__.py
    from web2py.gluon import current


# temporary setting - TODO: use better setting after we will allow more sources in single application
extsrc = current.db().select(current.db.extsrc.id, orderby=current.db.extsrc.id).first()
extsrc_id = extsrc.id if extsrc else None


def init_param():
    """
        initialize record counts and table dictionaries for places/locations and (md5publ)redirects
    """
    param = {}
    param['cnt_done'] = param['cnt_new'] = 0
    param['redirects'] = load_redirects()  # dict: md5publ -> md5publ(main) if book was joined with other (was/will be implemented later)
    param['places'] = load_places()        # dict: place -> (place_id)
    return param


def init_import(param, cnt_total=None, db=None):
    """
        initialize the import in database
    """
    if db is None:
        db = current.db

    library_id = param['_library_id']
    if cnt_total is None:
        st_imp_rik = db(db.library.id == library_id).select().first().st_imp_rik
        cnt_total = int(0.5 * 10 ** st_imp_rik)
    param['cnt_total'] = cnt_total
    db.library[library_id] = dict(imp_total=cnt_total, imp_done=0, imp_new=0, imp_proc=0.0)
    db.commit()


def set_proc(library_id, proc, db=None):
    if db is None:
        db = current.db

    db.library[library_id] = dict(imp_proc=proc)
    db.commit()


def counter_and_commit_if_100(param, added):
    """
        increment the counter and commit at the chunk (of length 100 rows) end
        added - (1st tuple item of the) result from update_or_insert_answer()
    """
    param['cnt_new'] += added
    param['cnt_done'] += 1
    if not param['cnt_done'] % 100:
        do_commit(param['_library_id'], param)


def finished(param, db=None, session=None):
    """
        commit yet uncommitted books and set imp_proc=100.0
    """
    if db is None:
        db = current.db
    if session is None:
        session = current.session

    if session.import_run_id:
        db.import_run[session.import_run_id] = dict(finished=datetime.datetime.utcnow(), cnt_total=param['cnt_done'], cnt_new=param['cnt_new'])
            # we do not prefer use cnt_total, because it can be just an estimate based on rik size inside; so cnt_done could be better
        del session.import_run_id
    do_commit(param['_library_id'], param, finished=True)


def cancel_import(library_id=None, db=None, auth=None, scheduler=None):
    """
        used by link in views/upload/running.html
    """
    if db is None:
        db = current.db
    if auth is None:
        auth = current.auth
    if scheduler is None:
        scheduler = current.scheduler

    if library_id is None:
        library_id = auth.library_id

    running = db((db.import_run.finished == None) & (db.import_run.library_id == library_id)).select(
            orderby=~db.import_run.started)
    for imp in running:
        imp.update_record(failed=True, finished=datetime.datetime.utcnow())
        if imp.scheduler_task_id:
            scheduler.stop_task(imp.scheduler_task_id)
        # we do not prefer use cnt_total, because it can be just an estimate based on rik size inside; so cnt_done could be better
    do_commit(auth.library_id, None, finished=True)


def do_commit(library_id, param, finished=False, db=None):
    if db is None:
        db = current.db

    if param:   # proc before 2.0% can be used before main processing, example: db.library[auth.library_id] = dict(imp_proc=0.3)
        upd = dict(imp_done=param['cnt_done'], imp_new=param['cnt_new'], imp_proc=100.0 if finished else max(2.0, min(99.9, 100.0 * param['cnt_done'] / param['cnt_total'])))
        if finished:
            upd['last_import'] = datetime.datetime.utcnow()
    else:  # ie. if cancel!
        upd = dict(imp_proc=100.0)
    db.library[library_id] = upd
    db.commit()


def clear_before_import(incremental=False, db=None, auth=None, session=None):
    if db is None:
        db = current.db
    if auth is None:
        auth = current.auth
    if session is None:
        session = current.session<commit_msg>ladeni scheduleru<commit_after># -*- coding: utf-8 -*-

"""From proprietary import call:
    - param=init_param() and init_import(param, cnt_total) before the import
    - import functions:
        -- added, answer_id = update_or_insert_answer()
        -- owned_book_id = update_or_insert_owned_book()
        -- update_or_insert_impressions()
    - counter_and_commit_if_100(param, added) with each imported book;
            added is (1st tuple item of the) result from update_or_insert_answer()
    - finished(param) after the import

Dependent on pydal, but independent on the (rest of) Web2py framework.
    The exception is use of current., which makes possible calls from Web2py framework without additional parameters;
    implicit parameter set via current. is always handled at the beginning of the func
"""

import datetime
import hashlib

from gluon import current

from c_db import PublLengths
from c_utils import ean_to_fbi, suplemental_fbi
from dal_idx import idx_row
from dal_utils import answer_by_ean, answer_by_hash, get_library

if False:  # for IDE only, need web2py/__init__.py
    from web2py.gluon import current


# temporary setting - TODO: use better setting after we will allow more sources in single application
extsrc = current.db().select(current.db.extsrc.id, orderby=current.db.extsrc.id).first()
extsrc_id = extsrc.id if extsrc else None


def init_param():
    """
        initialize record counts and table dictionaries for places/locations and (md5publ)redirects
    """
    param = {}
    param['cnt_done'] = param['cnt_new'] = 0
    param['redirects'] = load_redirects()  # dict: md5publ -> md5publ(main) if book was joined with other (was/will be implemented later)
    param['places'] = load_places()        # dict: place -> (place_id)
    return param


def init_import(param, cnt_total=None, db=None):
    """
        initialize the import in database
    """
    if db is None:
        db = current.db

    library_id = param['_library_id']
    if cnt_total is None:
        st_imp_rik = db(db.library.id == library_id).select().first().st_imp_rik
        cnt_total = int(0.5 * 10 ** st_imp_rik)
    param['cnt_total'] = cnt_total
    db.library[library_id] = dict(imp_total=cnt_total, imp_done=0, imp_new=0, imp_proc=0.0)
    db.commit()


def set_proc(library_id, proc, db=None):
    if db is None:
        db = current.db

    db.library[library_id] = dict(imp_proc=proc)
    db.commit()


def counter_and_commit_if_100(param, added):
    """
        increment the counter and commit at the chunk (of length 100 rows) end
        added - (1st tuple item of the) result from update_or_insert_answer()
    """
    param['cnt_new'] += added
    param['cnt_done'] += 1
    if not param['cnt_done'] % 100:
        print(param['cnt_done'])
        do_commit(param['_library_id'], param)


def finished(param, db=None, session=None):
    """
        commit yet uncommitted books and set imp_proc=100.0
    """
    if db is None:
        db = current.db
    if session is None:
        session = current.session

    if session.import_run_id:
        db.import_run[session.import_run_id] = dict(finished=datetime.datetime.utcnow(), cnt_total=param['cnt_done'], cnt_new=param['cnt_new'])
            # we do not prefer use cnt_total, because it can be just an estimate based on rik size inside; so cnt_done could be better
        del session.import_run_id
    do_commit(param['_library_id'], param, finished=True)


def cancel_import(library_id=None, db=None, auth=None, scheduler=None):
    """
        used by link in views/upload/running.html
    """
    if db is None:
        db = current.db
    if auth is None:
        auth = current.auth
    if scheduler is None:
        scheduler = current.scheduler

    if library_id is None:
        library_id = auth.library_id

    running = db((db.import_run.finished == None) & (db.import_run.library_id == library_id)).select(
            orderby=~db.import_run.started)
    for imp in running:
        imp.update_record(failed=True, finished=datetime.datetime.utcnow())
        if imp.scheduler_task_id:
            scheduler.stop_task(imp.scheduler_task_id)
        # we do not prefer use cnt_total, because it can be just an estimate based on rik size inside; so cnt_done could be better
    do_commit(auth.library_id, None, finished=True)


def do_commit(library_id, param, finished=False, db=None):
    if db is None:
        db = current.db

    if param:   # proc before 2.0% can be used before main processing, example: db.library[auth.library_id] = dict(imp_proc=0.3)
        upd = dict(imp_done=param['cnt_done'], imp_new=param['cnt_new'], imp_proc=100.0 if finished else max(2.0, min(99.9, 100.0 * param['cnt_done'] / param['cnt_total'])))
        if finished:
            upd['last_import'] = datetime.datetime.utcnow()
    else:  # ie. if cancel!
        upd = dict(imp_proc=100.0)
    db.library[library_id] = upd
    print('commit')
    db.commit()


def clear_before_import(incremental=False, db=None, auth=None, session=None):
    if db is None:
        db = current.db
    if auth is None:
        auth = current.auth
    if session is None:
        session = current.session
{'fraction_non_alphanumeric': 0.10222092910483949, 'fraction_numerical': 0.011056153622345069, 'mean_word_length': 3.205546492659054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '771746', 'n_tokens_mistral': 3326, 'n_tokens_neox': 3231, 'n_words': 946}