<commit_before>#!/usr/bin/python3.5
import grp
import os
import pwd
import shutil
import stat
import sys
import tarfile
import traceback
import copy
from _sha256 import sha256

from common.serializers.serialization import ledger_txn_serializer, serialize_msg_for_signing
from indy_common.config_helper import NodeConfigHelper
from indy_common.config_util import getConfig
from indy_common.constants import CONFIG_LEDGER_ID
from indy_node.server.config_req_handler import ConfigReqHandler
from indy_node.server.domain_req_handler import DomainReqHandler
from indy_node.server.pool_req_handler import PoolRequestHandler
from plenum.common.constants import TXN_TYPE, TXN_PAYLOAD, \
    TXN_PAYLOAD_METADATA, TXN_PAYLOAD_METADATA_DIGEST, POOL_LEDGER_ID, DOMAIN_LEDGER_ID
from plenum.common.txn_util import transform_to_new_format, get_from, get_req_id, get_payload_data, \
    get_protocol_version, get_seq_no, get_type
from plenum.common.types import f, OPERATION
from plenum.persistence.req_id_to_txn import ReqIdrToTxn
from storage.helper import initKeyValueStorage
from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys
from stp_core.common.log import getlogger

logger = getlogger()

ENV_FILE_PATH = "/etc/indy/indy.env"
ledger_types = ['pool', 'domain', 'config']
config = getConfig()


def set_own_perm(usr, dir):
    uid = pwd.getpwnam(usr).pw_uid
    gid = grp.getgrnam(usr).gr_gid
    perm_mask_rw = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP
    perm_mask_rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP

    os.chown(dir, uid, gid)
    os.chmod(dir, perm_mask_rwx)
    for croot, sub_dirs, cfiles in os.walk(dir):
        for fs_name in sub_dirs:
            os.chown(os.path.join(croot, fs_name), uid, gid)
            os.chmod(os.path.join(croot, fs_name), perm_mask_rwx)
        for fs_name in cfiles:
            os.chown(os.path.join(croot, fs_name), uid, gid)
            os.chmod(os.path.join(croot, fs_name), perm_mask_rw)


def get_node_name():
    node_name = None
    node_name_key = 'NODE_NAME'

    if os.path.exists(ENV_FILE_PATH):
        with open(ENV_FILE_PATH, "r") as fenv:
            for line in fenv.readlines():
                if line.find(node_name_key) != -1:
                    node_name = line.split('=')[1].strip()
                    break
    else:
        logger.error("Path to env file does not exist")

    return node_name


def get_ledger_id_by_txn_type(txn_type):

    def get_types_for_req_handler(req_handler):
        return list(req_handler.write_types) + list(req_handler.query_types)

    if txn_type in get_types_for_req_handler(PoolRequestHandler):
        return POOL_LEDGER_ID
    elif txn_type in get_types_for_req_handler(DomainReqHandler):
        return DOMAIN_LEDGER_ID
    elif txn_type in get_types_for_req_handler(ConfigReqHandler):
        return CONFIG_LEDGER_ID
    else:
        logger.error("Unknown txn_type: {}".format(txn_type))
        logger.error("Cannot write txn into SeqNoDB, because cannot define ledger_id")
        sys.exit(1)


def migrate_txn_log(db_dir, db_name):

    def put_into_seq_no_db(txn):
        # If there is no reqId, then it's genesis txn
        if get_req_id(txn) is None:
            return
        txn_new = copy.deepcopy(txn)
        operation = get_payload_data(txn_new)
        operation[TXN_TYPE] = get_type(txn_new)
        dct = {
            f.IDENTIFIER.nm: get_from(txn_new),
            f.REQ_ID.nm: get_req_id(txn_new),
            OPERATION: operation,
        }
        if get_protocol_version(txn_new) is not None:
            dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new)
        digest = sha256(serialize_msg_for_signing(dct)).hexdigest().encode()
        seq_no = get_seq_no(txn_new).encode()
        ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE])<commit_msg>[FIX migration] remove encode for seq_no<commit_after>#!/usr/bin/python3.5
import grp
import os
import pwd
import shutil
import stat
import sys
import tarfile
import traceback
import copy
from _sha256 import sha256

from common.serializers.serialization import ledger_txn_serializer, serialize_msg_for_signing
from indy_common.config_helper import NodeConfigHelper
from indy_common.config_util import getConfig
from indy_common.constants import CONFIG_LEDGER_ID
from indy_node.server.config_req_handler import ConfigReqHandler
from indy_node.server.domain_req_handler import DomainReqHandler
from indy_node.server.pool_req_handler import PoolRequestHandler
from plenum.common.constants import TXN_TYPE, TXN_PAYLOAD, \
    TXN_PAYLOAD_METADATA, TXN_PAYLOAD_METADATA_DIGEST, POOL_LEDGER_ID, DOMAIN_LEDGER_ID
from plenum.common.txn_util import transform_to_new_format, get_from, get_req_id, get_payload_data, \
    get_protocol_version, get_seq_no, get_type
from plenum.common.types import f, OPERATION
from plenum.persistence.req_id_to_txn import ReqIdrToTxn
from storage.helper import initKeyValueStorage
from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys
from stp_core.common.log import getlogger

logger = getlogger()

ENV_FILE_PATH = "/etc/indy/indy.env"
ledger_types = ['pool', 'domain', 'config']
config = getConfig()


def set_own_perm(usr, dir):
    uid = pwd.getpwnam(usr).pw_uid
    gid = grp.getgrnam(usr).gr_gid
    perm_mask_rw = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP
    perm_mask_rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP

    os.chown(dir, uid, gid)
    os.chmod(dir, perm_mask_rwx)
    for croot, sub_dirs, cfiles in os.walk(dir):
        for fs_name in sub_dirs:
            os.chown(os.path.join(croot, fs_name), uid, gid)
            os.chmod(os.path.join(croot, fs_name), perm_mask_rwx)
        for fs_name in cfiles:
            os.chown(os.path.join(croot, fs_name), uid, gid)
            os.chmod(os.path.join(croot, fs_name), perm_mask_rw)


def get_node_name():
    node_name = None
    node_name_key = 'NODE_NAME'

    if os.path.exists(ENV_FILE_PATH):
        with open(ENV_FILE_PATH, "r") as fenv:
            for line in fenv.readlines():
                if line.find(node_name_key) != -1:
                    node_name = line.split('=')[1].strip()
                    break
    else:
        logger.error("Path to env file does not exist")

    return node_name


def get_ledger_id_by_txn_type(txn_type):

    def get_types_for_req_handler(req_handler):
        return list(req_handler.write_types) + list(req_handler.query_types)

    if txn_type in get_types_for_req_handler(PoolRequestHandler):
        return POOL_LEDGER_ID
    elif txn_type in get_types_for_req_handler(DomainReqHandler):
        return DOMAIN_LEDGER_ID
    elif txn_type in get_types_for_req_handler(ConfigReqHandler):
        return CONFIG_LEDGER_ID
    else:
        logger.error("Unknown txn_type: {}".format(txn_type))
        logger.error("Cannot write txn into SeqNoDB, because cannot define ledger_id")
        sys.exit(1)


def migrate_txn_log(db_dir, db_name):

    def put_into_seq_no_db(txn):
        # If there is no reqId, then it's genesis txn
        if get_req_id(txn) is None:
            return
        txn_new = copy.deepcopy(txn)
        operation = get_payload_data(txn_new)
        operation[TXN_TYPE] = get_type(txn_new)
        dct = {
            f.IDENTIFIER.nm: get_from(txn_new),
            f.REQ_ID.nm: get_req_id(txn_new),
            OPERATION: operation,
        }
        if get_protocol_version(txn_new) is not None:
            dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new)
        digest = sha256(serialize_msg_for_signing(dct)).hexdigest().encode()
        seq_no = get_seq_no(txn_new)
        ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE])
{'fraction_non_alphanumeric': 0.08309786796814796, 'fraction_numerical': 0.0035961983046493705, 'mean_word_length': 3.6268568033273914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2354041', 'n_tokens_mistral': 2848, 'n_tokens_neox': 2780, 'n_words': 548}