text:
import pandas as pd
import io
from sklearn import preprocessing
from collections import deque
import numpy as np
import random
import time
import ipfshttpclient
import csv
import json
import web3
import tensorflow as tf
import config as cfg
# %load_ext tensorboard

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, LSTM, BatchNormalization
from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard
from web3 import Web3
from coinpaprika import client as Coinpaprika

cfg.DEBUG='TRUE'

### FETCHING DATA FROM A REAL TIME API

client = Coinpaprika.Client()
coins = client.coins()
coinList = []
listHashes = []
symbols = ["BTC", "ETH", "LTC", "BCH"]
names = ["Bitcoin", "Ethereum", "Litecoin","Bitcoin Cash"]

def check_if_exists(x, ls):
    if x in ls:
      return True
    else:
      return False

for coin in coins:
  if(check_if_exists(coin['symbol'], symbols) and check_if_exists(coin['name'],names)):
       coinList.append({"id": coin['id'], "symbol": coin['symbol']})

print(coinList)

def createFiles (id,symbol):
  response = client.historical(f"{id}", start="2022-05-08T00:00:00Z")  #YYY-MM-DD
  ourdata = []
  #csvheader = ['TimeStamp','Price','Volume','Market_Cap']

  for x in response:
    listing = [x["timestamp"], x["price"], x["volume_24h"], x["market_cap"]]
    ourdata.append(listing)

  with open(f"{symbol}-USD.csv",'w', encoding='UTF8',newline='') as f:
    writer = csv.writer(f)
    #writer.writerow(csvheader)
    writer.writerows(ourdata)

  return("done")

### CONNECTING TO IPFS TO STORE FILES

ipfsClient = ipfshttpclient.connect('/ip4/127.0.0.1/tcp/5001/http')
print(ipfsClient)

for coin in coinList:
  print(f"{createFiles(coin['id'], coin['symbol'])}")
  res = ipfsClient.add(f"{coin['symbol']}-USD.csv")
  listHashes.append(f"{res['Hash']}_{coin['symbol']}-USD")

### CONNECTING TO BLOCKCHAIN

web3 = Web3(Web3.HTTPProvider("http://127.0.0.1:9545",request_kwargs={'timeout':60}))
web3.isConnected()

# READING THE CONTRACT
with open('contracts/jsons/MainContract.json') as json_file:
    data = json.load(json_file)

print("Are we Connected to the local blockchain node?: ",web3.isConnected())

# PREPARE THE CONTRACT FOR DEPLOYMENT
web3.eth.defaultAccount=web3.eth.accounts[0]
myContract_instance = web3.eth.contract(abi=data['abi'], bytecode=data['bytecode'])
tx_hash=myContract_instance.constructor().transact()

# CONFIRMATION OF CONTRACT DEPLOYMENT
tx_receipt = web3.eth.waitForTransactionReceipt(tx_hash)

# CREATE A HANDLE TO THE CONTRACT INSTANCE WITH THE NEWLY-DEPLOYED ADDRESS
myContractaddress = web3.eth.contract(
    address=tx_receipt.contractAddress,
    abi=data['abi'],
)
print('Contract deployed at address: ' , myContractaddress.address)

# DEFINE ACTORS
offer_account = web3.eth.accounts[0]
solver_1 = web3.eth.accounts[1]
solver_2 = web3.eth.accounts[2]
solver_3 = web3.eth.accounts[3]

# CHECK INITAL BALANCES
print('Initial balance of the Primary Challenge Offerer account is: ', web3.fromWei(web3.eth.getBalance(offer_account),'ether'))
print('Initial balance of the Solver_1 account is: ', web3.fromWei(web3.eth.getBalance(solver_1),'ether'))
print('Initial balance of the Solver_2 account is: ', web3.fromWei(web3.eth.getBalance(solver_2),'ether'))
print('Initial balance of the Solver_3 account is: ', web3.fromWei(web3.eth.getBalance(solver_3),'ether'))
print('Initial balance of the COntract  account is: ', web3.fromWei(web3.eth.getBalance(myContractaddress.address),'ether'))

# FUNDING CONTRACT
fund_tx = web3.eth.sendTransaction({
    'from': offer_account,
    'to': myContractaddress.address,
    'value': web3.toWei(10, "ether")
})
print('After funding, balance of the COntract  account is: ', web3.fromWei(web3.eth.getBalance(myContractaddress.address), 'ether'))
print('After funding, balance of the Primary Challenge Offerer account is: ', web3.fromWei(web3.eth.getBalance(offer_account),'ether'))

#Storing the list of ipfs hash to blockchain by the owner of the contract
# 50% => 0.5 * 10000 => 5000
# this function is called by offer account to perform the initialization
init_tx_hash = myContractaddress.functions.initialization(50000,listHashes,1, 3).transact()
init_tx_receipt = web3.eth.wait_for_transaction_receipt(init_tx_hash)

# this function is called by any other solver
web3.eth.defaultAccount = solver_1
getList = myContractaddress.functions.get_train_data().call()
print(getList)

# Let's suppose the user wants to train the model for LTC-USD
# use the list to get the hash value for LTC-USD
RATIO_TO_PREDICT = 'LTC-USD'
FUTURE_PERIOD_PREDICT = myContractaddress.functions.future_predict_peroid().call()
SEQ_LEN = myContractaddress.functions.sequence_len().call()
EPOCHS = 50
BATCH_SIZE = 64
# getfile = ipfsClient.cat(f"{myHash}")
# getfile = getfile.decode('utf-8')
# print(getfile)

def classify(current, future):
  if float(future) > float(current):
    return 1
  else:
    return 0

def preprocess_df(df):
  df = df.drop('future',1)

  for col in df.columns:
    if col != 'target':
      df[col] = df[col].pct_change() # normalizing values
      df.dropna(inplace=True)
      df[col] = preprocessing.scale(df[col].values)
  df.dropna(inplace=True)

  sequential_data = []
  prev_days = deque(maxlen=SEQ_LEN)

  for i in df.values:
    prev_days.append([n for n in i[:-1]])
    if len(prev_days) == SEQ_LEN:
      sequential_data.append([np.array(prev_days), i[-1]])

  random.shuffle(sequential_data)

  # buys = []
  # sells = []

  # for seq, target in sequential_data:
  #   if target == 0:
  #       sells.append([seq,target])
  #   elif target == 1:
  #       buys.append([seq,target])

  # random.shuffle(buys)
  # random.shuffle(sells)

  # lower = min(len(buys), len(sells))

  # buys = buys[:lower]
  # sells = sells[:lower]

  #sequential_data = buys+sells

  random.shuffle(sequential_data)

  X = []
  y = []

  for seq,target in sequential_data:
    X.append(seq)
    y.append(target)

  return np.array(X), np.array(y)

main_df = pd.DataFrame()

for data in getList:
  x = data.split('_')
  myHash = x[0]
  dataset = f"{x[1]}.csv"
  df = pd.read_csv(dataset,names=["time",f"{x[1]}_close",f"{x[1]}_volume","market_cap"])
  df.set_index("time", inplace=True)
  df = df[[f"{x[1]}_close",f"{x[1]}_volume"]]

  if len(main_df) == 0:
      main_df = df
  else:
      main_df = main_df.join(df)

main_df['future'] = main_df[f"{RATIO_TO_PREDICT}_close"].shift(-FUTURE_PERIOD_PREDICT)
main_df['target'] = list(map(classify, main_df[f"{RATIO_TO_PREDICT}_close"], main_df['future']))

# Considering last 20 percent data for testing
times = sorted(main_df.index.values)
last_5pct = times[-int(0.2*len(times))] #unix time stamp value of last 5%
print(last_5pct)

validation_main_df = main_df[(main_df.index >= last_5pct)] # testing data set
main_df = main_df[(main_df.index < last_5pct)]

train_x, train_y = preprocess_df(main_df)
validation_x,validation_y = preprocess_df(validation_main_df)

# print(f"traindata: {len(train_x)}  validation: {len(validation_x)}")
# unique, counts = np.unique(train_y, return_counts=True)
# print(unique, counts)
# for i in unique:
#   index = np.where(train_y == i)[0][0]
#   if i == 0:
#     print('tain_y dont buys:', counts[index])
#   elif i == 1:
#     print('train_y buys:', counts[index])

# unique, counts = np.unique(validation_y, return_counts=True)
# print(unique, counts)
# for i in unique:
#   index = np.where(validation_y == i)[0][0]
#   if i == 0:
#     print('validation_y dont buys:', counts[index])
#   elif i == 1:
#     print('validation_y buys:', counts[index])

# print(f"dont buys: {train_y.count(0)} buys: {train_y.count(1)}")
# print(f"Validation dont buys: {validation_y.count(0)} Validation buys: {validation_y.count(1)}")
accuracies = []

def model_setup(dense_layer, layer_size, lstm_layer, activation_fun, loss_fun):

  NAME = "{}-lstm-{}-nodes-{}-dense-{}".format(lstm_layer, layer_size, dense_layer, int(time.time()))
  tensorboard = tf.keras.callbacks.TensorBoard(log_dir='logs/{}'.format(NAME))
  model = Sequential()

  for i in range(lstm_layer-1):
    model.add(LSTM(layer_size, input_shape=(train_x.shape[1:]), return_sequences=True))
    model.add(Dropout(0.2))
    model.add(BatchNormalization())

  model.add(LSTM(128, input_shape=(train_x.shape[1:])))
  model.add(Dropout(0.2))
  model.add(BatchNormalization())

  for l in range(dense_layer):
    model.add(Dense(32))
    model.add(tf.keras.layers.Activation(activation_fun))

  model.add(Dense(2, activation="softmax"))

  opt = tf.keras.optimizers.Adam(lr=0.001, decay=1e-6)

  model.compile(loss=loss_fun,
                optimizer=opt,
                metrics='accuracy')

  tensorboard = tf.keras.callbacks.TensorBoard(log_dir=f"logs/{NAME}")

  filepath = "RNN_Final-{epoch:02d}-{val_accuracy:.3f}" #unique file name that will include the epoch and the validation acc for that epoch
  checkpoint = ModelCheckpoint("models/{}.model".format(filepath, monitor='val_accuracy',verbose=1, save_best_only=True, mode='max')) #saves only the best ones
  history = model.fit(train_x,
                    train_y,
                    batch_size=BATCH_SIZE,
                    epochs=EPOCHS,
                    validation_data=(validation_x, validation_y),
                    callbacks=[tensorboard, checkpoint])
  x = history.history['accuracy']
  val = '%.5f'%(max(x))
  x = float(val) * 100000.0
  accuracies.append(int(x))

# SUBMITTING SOLUTION TO THE BLOCKCHAIN BY SOLVER 1
web3.eth.defaultAccount = solver_1
sol_1_hash = myContractaddress.functions.submit_model(solver_1,1,128,3,'relu', 'sparse_categorical_crossentropy').transact()
sol_1_receipt = web3.eth.wait_for_transaction_receipt(sol_1_hash)

# SUBMITTING SOLUTION TO THE BLOCKCHAIN BY SOLVER 2
web3.eth.defaultAccount = solver_2
sol_2_hash = myContractaddress.functions.submit_model(solver_2,2,64,4,'sigmoid', 'sparse_categorical_crossentropy').transact()
sol_1_receipt = web3.eth.wait_for_transaction_receipt(sol_1_hash)

# GET ALL SUBMISSIONS
web3.eth.defaultAccount = offer_account
submissions = myContractaddress.functions.get_all_submissions().call()
print(submissions)

# OWNER WILL EVALUATE THE MODEL ON HIS OWN
for sol in submissions:
  model_setup(sol[1],sol[2],sol[3],sol[4],sol[5])

# NOW OWNER WILL EVALUATE THE RESULTS
web3.eth.defaultAccount = offer_account
for i in accuracies:
   set_model_hash = myContractaddress.functions.set_evaluation_metrics(i, accuracies.index(i)).transact()
   set_model_receipt = web3.eth.wait_for_transaction_receipt(set_model_hash)

# FOR CLOSING THE PROCESS

final_hash = myContractaddress.functions.finalize_contract().transact()
final_receipt = web3.eth.wait_for_transaction_receipt(final_hash)

max_stars_repo_name: tagupta/Using-ML-in-blockchain-

max_issues_repo_name: tagupta/Using-ML-in-blockchain-

max_forks_repo_name: tagupta/Using-ML-in-blockchain-


text:
# Copyright 2020 Yuhao Zhang and Arun Kumar. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from collections import namedtuple
import struct
# import bitstruct as bitstruct
import pandas as pd
from cerebro_gpdb.utils import logs
import socket
import numpy as np
import ctypes
import glob
import sys
IS_PYTHON3 = sys.version_info.major >= 3
HEAP_HASNULL = 0x0001
HEAP_NATTS_MASK = 0x07FF
TYPALIGN_INT = 'i'
VARHDRSZ_EXTERNAL = 4
SIZE_OF_VARATT_EXTERNAL = 16
INDEPENDENT_VAR_OFF = 4
FULL_PD_UPPER = 64
# AD_HOC_BYTES = b'0\x08\x00\x00\x00\x00\x00\x00h'
BLOCK_SIZE = 32768
BLCKSZ = BLOCK_SIZE
PAGE_HEADER_LEN = 24
ITEM_IDENTIFIER_LEN = 4
VARHDRSZ = 4
SIZE_OF_ITEMIDDATA = ITEM_IDENTIFIER_LEN
ITEM_HEADER_LEN = 23
MAXIMUM_ALIGNOF = 8
SIZE_OF_PGLZ_HEADER = 8
CHUNK_ID_AND_CHUNK_SEQ_LEN = 8
TOAST_MAX_CHUNK_SIZE = 8140
SIZE_OF_PAGEHEADERDATA = PAGE_HEADER_LEN + ITEM_IDENTIFIER_LEN
PGLZ_IMPLEMENTATION = 'py'  # C implementation not working as of now
# lib = ctypes.CDLL(os.path.abspath("./pg_lzcompress.so"))
# pglz_decompress_c = wrap_function(lib, 'pglz_decompress', None,
# [ctypes.c_char_p, ctypes.c_char_p])

def wrap_function(lib, funcname, restype, argtypes):
    """Simplify wrapping ctypes functions"""
    func = lib.__getattr__(funcname)
    func.restype = restype
    func.argtypes = argtypes
    return func

def SET_VARSIZE(bytea, header):
    bytea[:4] = header

def att_align_nominal(off):
    cur_offset = off
    LEN = cur_offset
    ALIGNOF_INT = 4
    ALIGNVAL = ALIGNOF_INT
    return (LEN + ALIGNVAL - 1) & ~(ALIGNVAL - 1)

def TYPEALIGN(ALIGNVAL, LEN):
    return (LEN + ALIGNVAL - 1) & ~(ALIGNVAL - 1)

def MAX_ALIGN(LEN):
    return TYPEALIGN(MAXIMUM_ALIGNOF, (LEN))

def VARSIZE_1B_E():
    return SIZE_OF_VARATT_EXTERNAL + VARHDRSZ_EXTERNAL

def VARSIZE_1B(bytea):
    header = get_1b_header(bytea)
    return (header) & 0x7F

# def VARATT_IS_EXTERNAL(byte):
#     return byte == 0x80

def VARDATA(bytea):
    return bytea[4:]

def get_1b_header(bytea):
    if isinstance(bytea, int):
        return bytea
    else:
        return struct.unpack('@B', bytea[0:1])[0]

def VARATT_IS_1B(bytea):
    header = get_1b_header(bytea)
    return (header & 0x80) == 0x80

def VARATT_IS_4B_U(bytea):
    header = get_1b_header(bytea)
    return (header & 0xC0) == 0x00

def VARATT_IS_EXTENDED(bytea):
    return not VARATT_IS_4B_U(bytea)

def VARATT_IS_1B_E(bytea):
    header = get_1b_header(bytea)
    return header == 0x80

def VARATT_IS_4B_C(bytea):
    header = get_1b_header(bytea)
    return (header & 0xC0) == 0x40
def VARATT_IS_COMPRESSED(bytea):
    return VARATT_IS_4B_C(bytea)

def VARATT_IS_EXTERNAL(bytea):
    return VARATT_IS_1B_E(bytea)

def VARSIZE(bytea):
    return socket.ntohl(struct.unpack(str('@I'), bytea[:4])[0]) & 0x3FFFFFFF

def GET_VARSIZE_4B_C(total_length):
    length_tmp = (total_length & 0x3FFFFFFF) | 0x40000000
    length_byte = struct.pack('>I', length_tmp)
    return length_byte

def GET_VARSIZE_4B(total_length):
    length_tmp = (total_length & 0x3FFFFFFF)
    length_byte = struct.pack('>I', length_tmp)
    return length_byte

# def item_identifiers_debug(item_identifiers_bytes):
#     for i in range(0, len(item_identifiers_bytes), ITEM_IDENTIFIER_LEN):
#         packed_identifier = item_identifiers_bytes[i:i + ITEM_IDENTIFIER_LEN]
#         print(packed_identifier)
#         lp_off, lp_flags, lp_len = bitstruct.unpack('u15u2u15<',
#                                                     packed_identifier)
#         print(lp_off, lp_flags, lp_len)

def get_num_chunks(ressize):
    return ((ressize - 1) // TOAST_MAX_CHUNK_SIZE) + 1

def expand_input_dims(input_data):
    input_data = np.array(input_data, dtype=np.float32)
    input_data = np.expand_dims(input_data, axis=0)
    return input_data

def np_array_float32(var, var_shape):
    arr = np.frombuffer(var, dtype=np.float32)
    arr.shape = var_shape
    return arr

def np_array_int16(var, var_shape):
    arr = np.frombuffer(var, dtype=np.int16)
    arr.shape = var_shape
    return arr

def deserialize_bytea(attname, dest_data, var_shape):
    if attname == 'dependent_var':
        np_data = np_array_int16(dest_data, var_shape)
    elif attname == 'independent_var':
        np_data = np_array_float32(dest_data, var_shape)
    return np_data

def GET_RAWSIZE_FROM_COMPRESSED(bytea):
    return struct.unpack('@I', bytea[4:8])[0]

def acc(bytea, i):
    return bytea[i:i+1]

def pglz_decompress_py(source, dest):
    varsize_source = VARSIZE(source)
    rawsize_source = GET_RAWSIZE_FROM_COMPRESSED(source)
    sp = 0 + SIZE_OF_PGLZ_HEADER
    srcend = 0 + varsize_source
    dp = 0
    destend = dp + rawsize_source
    while (sp < srcend and dp < destend):
        ctrl = source[sp]
        ctrl = get_1b_header(ctrl)
        sp += 1
        ctrlc = 0
        while(ctrlc < 8 and sp < srcend):
            if (ctrl & 1):
                length = (get_1b_header(source[sp]) & 0x0f) + 3
                off = ((
                    get_1b_header(source[sp]) & 0xf0) << 4) | get_1b_header(
                    source[sp+1])
                sp += 2
                if (length == 18):
                    length += get_1b_header(source[sp])
                    sp += 1
                if (dp + length > destend):
                    dp += length
                    break
                while (length):
                    length -= 1
                    dest[dp] = dest[dp - off]
                    dp += 1
            else:
                if (dp >= destend):
                    break
                dest[dp] = source[sp]
                dp += 1
                sp += 1

            ctrl >>= 1
            ctrlc += 1
    if (dp != destend or sp != srcend):
        raise Exception("compressed data is corrupt")
    return dest

def pglz_decompress(source, dest, implementation):
    if implementation == 'py':
        return pglz_decompress_py(source, dest)

def pre_alloc_dest(bytea, implementation):
    dest_data_size = struct.unpack('@I', bytea[4:8])[0]
#     dest_size = dest_data_size + VARHDRSZ
#     dest_header = GET_VARSIZE_4B(dest_size)
#     SET_VARSIZE(dest, dest_header)
#     dest_data = VARDATA(dest)
    if implementation == 'py':
        dest_data = bytearray(dest_data_size)
    elif implementation == 'c':
        dest_data = ctypes.create_string_buffer(dest_data_size)

    return dest_data, dest_data_size

def deserialize_page_header(page_bytes):
    deserialized_header = struct.unpack(
        str('@qHHHHHHI'), page_bytes[:PAGE_HEADER_LEN])
    deserialized_header = list(deserialized_header)
    deserialized_header[-2] &= 0x00FF
    header = PageHeader(*deserialized_header)
    item_identifiers_begin = PAGE_HEADER_LEN
    item_identifiers_end = getattr(header, 'pd_lower')
    item_identifiers_len = item_identifiers_end - item_identifiers_begin
    if (item_identifiers_len % ITEM_IDENTIFIER_LEN) != 0:
        raise Exception("""Item identifier length do not match.
                        Must be a multiply of {}.""".format(
            ITEM_IDENTIFIER_LEN))
    item_identifiers_bytes = page_bytes[
        item_identifiers_begin:item_identifiers_end]
    item_num = item_identifiers_len // ITEM_IDENTIFIER_LEN
    return header, item_identifiers_bytes, item_num

def deserialize_item_header(item):
    deserialized_item_header = struct.unpack(
        str('@IIIHHHHHB'), item[:ITEM_HEADER_LEN])
    item_header = HeapTupleHeader(*deserialized_item_header)
    return item_header

def extract_tuple(item, item_header, tuple_data_len):
    t_hoff = getattr(item_header, 't_hoff')
    tuple_data_bytea = item[t_hoff:t_hoff+tuple_data_len]
    return tuple_data_bytea

def deserialize_item(page_bytes, packed_identifier):
    assert len(packed_identifier) == ITEM_IDENTIFIER_LEN
    bitlengthes = [15, 2, 15]
#     LSB first
    names = ['lp_len', 'lp_flags', 'lp_off']
    values = []
    bitstring = '{:032b}'.format(struct.unpack('@I', packed_identifier)[0])
    off = 0
    for bitlength, name in zip(bitlengthes, names):
        bitfield = bitstring[off:off + bitlength]
        number = int(bitfield, base=2)
        values.append(number)
        off += bitlength
    values = values[::-1]
    item_identifier = ItemIdentifier(*values)
    lp_off, lp_flags, lp_len = item_identifier
    item = page_bytes[lp_off:lp_off + lp_len]
    item_header = deserialize_item_header(item)
    t_hoff = getattr(item_header, 't_hoff')
    tupdata = item[t_hoff:]
    return tupdata, item_header, item_identifier

# def desirialize_tupdata(tupdata, toast=False):
#     if not toast:
#         length = VARSIZE_1B_E()
#         c = 0
#         n = 4
#         dist_key = struct.unpack('@I', tupdata[c:n])[0]
#         c = n
#         n += length
#         independent_var = struct.unpack('@BBBBiiII', tupdata[c:n])
#         c = n
#         n += length
#         dependent_var = struct.unpack('@BBBBiiII', tupdata[c:n])
#         buffer_id = struct.unpack('@I', tupdata[-4:])[0]
#         independent_var_tuple = Tuple(*independent_var)
#         dependent_var_tuple = Tuple(*dependent_var)
#         return dist_key, independent_var_tuple, dependent_var_tuple, buffer_id
#     else:
#         chunk_id, chunk_seq = struct.unpack(str('@II'), tupdata[:8])
#         chunk_data = tupdata[8:]
#         return chunk_id, chunk_seq, chunk_data
def desirialize_tupdata(tupdata, toast=False):
    if not toast:
        buffer_id = struct.unpack('@I', tupdata[-4:])[0]
        length = VARSIZE_1B_E()
        c = 0
        n = 4
        dist_key = struct.unpack('@I', tupdata[c:n])[0]
        c = n
        n += length
        independent_var = struct.unpack('@BBBBiiII', tupdata[c:n])
        c = n
        n += length
        dep_header = tupdata[c:n]
        buffer_id = struct.unpack('@I', tupdata[-4:])[0]
        dependent_var = struct.unpack('@BBBBiiII', tupdata[c:n])
        if VARATT_IS_EXTENDED(dep_header) and not VARATT_IS_EXTERNAL(dep_header) and VARATT_IS_COMPRESSED(dep_header):
            bytea = tupdata[c:]
            dependent_var = [x for x in dependent_var] + [False, bytea]
        else:
            dependent_var = [x for x in dependent_var] + [True, None]
        independent_var = [x for x in independent_var] + [True, None]
        independent_var_tuple = Tuple(*independent_var)
        dependent_var_tuple = Tuple(*dependent_var)
        return dist_key, independent_var_tuple, dependent_var_tuple, buffer_id
    else:
        chunk_id, chunk_seq = struct.unpack(str('@II'), tupdata[:8])
        chunk_data = tupdata[8:]
        return chunk_id, chunk_seq, chunk_data

def read_page_at(f, pos):
    f.seek(pos)
    page_bytes = f.read(BLOCK_SIZE)
    return page_bytes

def generator_page(filename, toast=False, debug=False):
    seg_files = glob.glob(filename + '.*')
    seg_files.sort()
    seg_files.sort(key=len)
    seg_files = [filename] + seg_files
    for filename in seg_files:
        with open(filename, 'rb') as f:
            curr_pos = 0
            while True:

                page_bytes = read_page_at(f, curr_pos)
                if len(page_bytes) != BLOCK_SIZE:
                    if len(page_bytes) != 0:
                        raise Exception("FATAL")
                    else:
                        break
                if debug:
                    print("PAGE: {}".format(curr_pos))
                curr_pos = f.tell()
                page_header, item_identifiers_bytes, item_num = \
                    deserialize_page_header(page_bytes)
                if toast:
                    pd_upper = getattr(page_header, 'pd_upper')
                    pd_special = getattr(page_header, 'pd_special')
                    assert pd_special == BLOCK_SIZE, \
                        "THERE SHALL NOT BE INDICES"
                    lp_flags = 1
                    lp_off = pd_upper

                    for idx in range(item_num):
                        lp_off = MAX_ALIGN(lp_off)
                        item_header = page_bytes[lp_off:lp_off +
                                                 ITEM_HEADER_LEN]
                        item_header = deserialize_item_header(item_header)
                        t_hoff = getattr(item_header, 't_hoff')
                        tupdata_off = lp_off + t_hoff
                        chunkdata_off = tupdata_off + \
                            CHUNK_ID_AND_CHUNK_SEQ_LEN
                        chunkdata_header = page_bytes[
                            chunkdata_off:chunkdata_off + VARHDRSZ]
                        chunksize = VARSIZE(chunkdata_header)
                        lp_len = t_hoff + CHUNK_ID_AND_CHUNK_SEQ_LEN + \
                            chunksize
                        item_identifier = ItemIdentifier(
                            *(lp_off, lp_flags, lp_len))
                        item = page_bytes[lp_off:lp_off + lp_len]

                        tupdata = item[t_hoff:]
                        chunk_id, chunk_seq, chunk_data = desirialize_tupdata(
                            tupdata, toast=True)
                        lp_off += lp_len
                        if debug:
                            yield chunk_id, chunk_seq, chunk_data, (
                                page_bytes, page_header,
                                item_identifiers_bytes, lp_off,
                                item_num, item_identifier, item_header,
                                tupdata)
                        else:
                            yield chunk_id, chunk_seq, chunk_data, None
                else:
                    for i in range(0, len(item_identifiers_bytes),
                                   ITEM_IDENTIFIER_LEN):
                        packed_identifier = item_identifiers_bytes[
                            i:i + ITEM_IDENTIFIER_LEN]
                        tupdata, item_header, item_identifier = \
                            deserialize_item(
                                page_bytes, packed_identifier)
                        dist_key, independent_var_tuple, dependent_var_tuple, \
                            buffer_id = desirialize_tupdata(tupdata)
                        if debug:
                            yield dist_key, \
                                independent_var_tuple, \
                                dependent_var_tuple, \
                                buffer_id, \
                                (page_bytes,
                                 page_header,
                                 item_identifiers_bytes,
                                 None,
                                 item_num,
                                 item_identifier,
                                 item_header,
                                 tupdata)
                        else:
                            yield dist_key, independent_var_tuple, \
                                dependent_var_tuple, buffer_id, None

def table_page_read(page_dir):
    logs("START READING TABLE: {}".format(page_dir))
    page_gen = generator_page(page_dir, debug=False)
    all_data = []
    all_toast = []
    while True:
        try:
            res = next(page_gen)
            dist_key, \
                independent_var_tuple, \
                dependent_var_tuple, \
                buffer_id, \
                _ = res

            all_data.append(res[:-1])
            all_toast.append(
                list(independent_var_tuple[4:]) + [dist_key,
                                                   buffer_id,
                                                   'independent_var'])
            all_toast.append(
                list(dependent_var_tuple[4:]) + [
                    dist_key,
                    buffer_id,
                    'dependent_var'])
        except StopIteration:
            break
    df_data = pd.DataFrame(all_data, columns=[
        'dist_key',
        'independent_var_tuple',
        'dependent_var_tuple',
        'buffer_id'])
    df_toast = pd.DataFrame(all_toast, columns=[
        'va_rawsize',
        'va_extsize',
        'va_valueid',
        'va_toastrelid',
        'external',
        'bytea',
        'dist_key',
        'buffer_id',
        'attname']
    )
    logs("END READING TABLE: {}".format(page_dir))
    return df_data, df_toast

def read_toast_all_bytes(toast_page_path, df_toast, debug=False):
    toast_gen = generator_page(toast_page_path, toast=True, debug=debug)
    grand_bytes = {}
    i = 0
    while True:
        try:
            i += 1
            chunk_id, chunk_seq, chunk_data, debug_data = next(toast_gen)
            if debug:
                (
                    page_bytes,
                    page_header,
                    item_identifiers_bytes,
                    lp_off,
                    item_num,
                    item_identifier,
                    item_header,
                    tupdata
                ) = debug_data
            if int(chunk_id) in df_toast['va_valueid'].values:
                if debug:
                    print(i, chunk_id, chunk_seq)
                if chunk_id not in grand_bytes:
                    grand_bytes[chunk_id] = []
                grand_bytes[chunk_id].append([chunk_seq, chunk_data])
        except StopIteration:
            break
    return grand_bytes

def detoast(grand_bytes, df_toast, df_shape, table_name):
    grand_bytes_con = {}
    df_toast_external = df_toast.loc[df_toast['external'] == True]
    df_toast_internal = df_toast.loc[df_toast['external'] == False]
    # decompress unexternal data:
    for i, row in df_toast_internal.iterrows():
        buffer_id = row['buffer_id']
        bytea = row['bytea']
        attname = row['attname']
        va_rawsize = row['va_rawsize']
        pglz_header = bytea[:8]
        vl_len_, rawsize = struct.unpack('@ii', pglz_header)
        assert rawsize == va_rawsize, 'raw size does not match!'
        dest_data, dest_data_size = pre_alloc_dest(
                bytea, implementation=PGLZ_IMPLEMENTATION)
        dest_data = pglz_decompress(
                    bytea, dest_data, implementation=PGLZ_IMPLEMENTATION)
        shape_info = df_shape.loc[
                    (df_shape['table_name'] == table_name)
                    & (df_shape['buffer_id'] == buffer_id)].iloc[0]
        var_shape = shape_info[attname + '_shape']
        np_data = deserialize_bytea(attname, dest_data, var_shape)
        if buffer_id not in grand_bytes_con:
            grand_bytes_con[buffer_id] = {
                "dependent_var": None, "independent_var": None}
        grand_bytes_con[buffer_id][attname] = np_data
    for k, v in grand_bytes.items():
        logs("ID:{}".format(k))
        va_rawsize, \
            va_extsize, \
            va_valueid, \
            va_toastrelid, \
            external, \
            bytea, \
            dist_key, \
            buffer_id, \
            attname = df_toast_external.loc[
                df_toast_external['va_valueid'] == k].iloc[0]
        shape_info = df_shape.loc[
            (df_shape['table_name'] == table_name)
            & (df_shape['buffer_id'] == buffer_id)].iloc[0]

        var_shape = shape_info[attname + '_shape']
        numchunks = get_num_chunks(va_extsize)
        numchunks_actual = len(v)
        assert numchunks == numchunks_actual

        ressize_include_header = VARHDRSZ + va_extsize
        header_byte = GET_VARSIZE_4B_C(ressize_include_header)
        ressize = va_extsize
        v.sort(key=lambda x: x[0])
        v_parts = [header_byte]
        for idx, chunk in v:
            assert not VARATT_IS_EXTENDED(chunk)
            chunksize = VARSIZE(chunk) - VARHDRSZ
            chunkdata = VARDATA(chunk)
            v_parts.append(chunkdata[:chunksize])
            if ((idx == 0) and (chunksize < ressize)
                    and (chunksize != TOAST_MAX_CHUNK_SIZE)):
                raise Exception("max chunk size error")
            if (idx < numchunks - 1) and (chunksize != TOAST_MAX_CHUNK_SIZE):
                raise Exception("unexpected chunk size")
            if (idx == numchunks - 1) and (
                    (idx * TOAST_MAX_CHUNK_SIZE + chunksize) != ressize):
                raise Exception("unexpected chunk size")

        v = []
        bytea = b''.join(v_parts)
        assert len(bytea) == ressize_include_header, \
            "final size does not match"
        v_parts = []
        dest_data, dest_data_size = pre_alloc_dest(
            bytea, implementation=PGLZ_IMPLEMENTATION)
        logs("START DECOMPRESSING")
        dest_data = pglz_decompress(
            bytea, dest_data, implementation=PGLZ_IMPLEMENTATION)
        logs("END DECOMPRESSING")
        bytea = None
        np_data = deserialize_bytea(attname, dest_data, var_shape)
        dest_data = None
        if buffer_id not in grand_bytes_con:
            grand_bytes_con[buffer_id] = {
                "dependent_var": None, "independent_var": None}

        grand_bytes_con[buffer_id][attname] = np_data
    return grand_bytes_con

def toast_page_read(page_dir, df_toast, df_shape, table_name):
    logs("START READING TOAST: {}".format(page_dir))
    grand_bytes = read_toast_all_bytes(page_dir, df_toast)
    logs("END READING TOAST: {}".format(page_dir))
    logs("START DETOASTING")
    grand_bytes_con = detoast(grand_bytes, df_toast, df_shape, table_name)
    logs("END DETOASTING")
    logs("ITEM NUM:{}, BUFFER_IDs:{}".format(
        len(grand_bytes_con.keys()), grand_bytes_con.keys()))
    return grand_bytes_con

LP_LENGTH_GLOBAL = TOAST_MAX_CHUNK_SIZE + VARHDRSZ + MAX_ALIGN(
    ITEM_HEADER_LEN) + CHUNK_ID_AND_CHUNK_SEQ_LEN
PageHeader = namedtuple('PageHeader', [
    'pd_lsn',
    'pd_tli',
    'pd_flags',
    'pd_lower',
    'pd_upper',
    'pd_special',
    'pd_pagesize_version',
    'pd_prune_xid',
])
HeapTupleHeader = namedtuple('HeapTupleHeader', [
    't_xmin', 't_xmax', 't_cid', 'bi_hi', 'bi_lo', 'ip_posid', 't_infomask2',
    't_infomask', 't_hoff'
])
ItemIdentifier = namedtuple('ItemIdentifier', ['lp_off', 'lp_flags', 'lp_len'])
TupleDesc = namedtuple(
    'TupleDesc',
    ['natts', 'tdtypeid', 'tdtypmod', 'tdrefcount', 'constr', 'attrs'])
FormData_pg_attribute = namedtuple(
    'FormData_pg_attribute',
    ['attrelid', 'attname', 'atttypid', 'attlen', 'attnum', 'attalign'])
Tuple = namedtuple('Tuple', [
    'header',
    'length',
    'dummy1',
    'dummy2',
    'va_rawsize',
    'va_extsize',
    'va_valueid',
    'va_toastrelid',
    'external',
    'bytea'
])

ext: py

max_stars_repo_name: makemebitter/cerebro-ds

max_issues_repo_name: makemebitter/cerebro-ds

max_forks_repo_name: makemebitter/cerebro-ds

reason: import numpy


text:
import torch
from torch import nn
import torch.nn.functional as F
from torch.autograd import Variable
import scipy as sp
import os
import pdb

def normalize_rows(x):
    diagonal = torch.einsum("ij,ij->i", [x, x])[:, None]
    return x / torch.sqrt(diagonal)

class Vmodel(nn.Module):
    def __init__(self, P, Q, p, q):
        super(Vmodel, self).__init__()
        self.x0 = nn.Parameter(torch.randn(P, p))
        self.v0 = nn.Parameter(torch.randn(Q, q))
        self._init_params()

    def x(self):
        return normalize_rows(self.x0)

    def v(self):
        return normalize_rows(self.v0)

    def forward(self, d, w):
        # embed
        X = F.embedding(d, self.x())
        W = F.embedding(w, self.v())
        # multiply
        V = torch.einsum("ij,ik->ijk", [X, W])
        V = V.reshape([V.shape[0], -1])
        return V

    def _init_params(self):
        self.x0.data[:, 0] = 1.0
        self.x0.data[:, 1:] = 1e-3 * torch.randn(*self.x0[:, 1:].shape)
        self.v0.data[:] = torch.eye(*self.v0.shape) + 1e-3 * torch.randn(*self.v0.shape)

if __name__ == "__main__":

    P = 4
    Q = 4
    p = 2
    q = 2

    pdb.set_trace()

    vm = Vmodel(P, Q, p, q).cuda()

    # _d and _w
    _d = sp.kron(sp.arange(P), sp.ones(2))
    _w = sp.kron(sp.ones(2), sp.arange(Q))

    # d and w
    d = Variable(torch.Tensor(_d).long(), requires_grad=False).cuda()
    w = Variable(torch.Tensor(_w).long(), requires_grad=False).cuda()

    V = vm(d, w)
    pdb.set_trace()

max_stars_repo_name: ahmerb/GPPVAE

max_issues_repo_name: ahmerb/GPPVAE

max_forks_repo_name: ahmerb/GPPVAE

reason: import scipy


text:
import tensorflow as tf
from galaxy_CapsNet import *
import cv2
import numpy as np
from galaxy_data import *

batch_size = 32
num_show = 5
is_multi_galaxy = True
is_shift_ag = True
irun = 0
lr = 1e-3
steps = 100000
save_frequence = 2500
decay_frequence = 5000
is_show_multi_rec = False
is_show_sample = False
key = -1
min_lr = 5e-6

if is_multi_galaxy:
    train_iter = multigalaxy_train_iter(iters=steps,batch_size=batch_size,is_shift_ag = True)
    test_iter = multigalaxy_test_iter(iters=steps, batch_size=batch_size,is_shift_ag = True)
else:
    train_iter = galaxy_train_iter(iters=steps,batch_size=batch_size,is_shift_ag = True)
    test_iter = galaxy_test_iter(iters=steps, batch_size=batch_size,is_shift_ag = True)
multi_iter = multigalaxy_test_iter(iters=steps,batch_size=num_show,is_shift_ag = True)

tf.reset_default_graph()
tf.set_random_seed(1234)
net = CapsNet(is_multi_galaxy=is_multi_galaxy)
tf.summary.scalar('error_rate_on_test_set', (1.0 - net.accuracy) * 100.0)
tf.summary.scalar('loss_reconstruction_on_test_set', net.loss_rec)
merged = tf.summary.merge_all()
init = tf.global_variables_initializer()

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
#config.gpu_options.per_process_gpu_memory_fraction = 0.33
sess = tf.Session(config=config)
writer = tf.summary.FileWriter("./sum",sess.graph)
var_to_save = [var for var in tf.global_variables() if ('Adam' not in var.name) and ('Momentum' not in var.name)]
saver = tf.train.Saver(var_list=var_to_save, max_to_keep=10)

sess.run(init)

RESTORE = False
if RESTORE:
    try:
        ckpt = tf.train.get_checkpoint_state('./cpt')
        saver.restore(sess, ckpt.model_checkpoint_path)
        print('Loading model succeeded...')
    except:
        print('Loading model failed or model doest not exist')

for X,Y in train_iter:

    X_TEST, Y_TEST = test_iter.__next__()

    # LS, LS_REC, ACC, _ = sess.run([net.loss, net.loss_rec, net.accuracy, net.train], feed_dict={net.x: X, net.y: Y, net.lr: lr, net.is_training:True})
    LS, ACC, _ = sess.run([net.loss, net.accuracy, net.train], feed_dict={net.x: X, net.y: Y, net.lr: lr, net.is_training:True})
    ACC_TEST, result = sess.run([net.accuracy,merged], feed_dict={net.x: X_TEST, net.y: Y_TEST, net.is_training:False})

    writer.add_summary(result, irun)

    # print('{},{},{},{},{}'.format(irun, LS, LS_REC, ACC, ACC_TEST))
    print('{}, LS: {:.4f}, Train ACC:{:.3f}, Test ACC: {:.3f}'.format(irun, LS, ACC, ACC_TEST))

    """
    if is_show_sample:
        H_SAM = np.random.randn(num_show*2, 2, 32)
        H_SAM = H_SAM / (0.001 + np.sum(H_SAM ** 2.0, axis=-1, keepdims=True) ** 0.5)
        Y_SAM = np.eye(2)[np.array(list(range(2)) * num_show)].astype(float)
        X_SAM = sess.run(net.x_sample, feed_dict={net.h_sample: H_SAM, net.y_sample: Y_SAM, net.is_training:False})
        images_sample = X_SAM.reshape([num_show, 2, 212, 212, 1])
        images_sample = np.concatenate(images_sample, axis=1)
        images_sample = cv2.resize(np.concatenate(images_sample, axis=1), dsize=(0, 0), fx=3, fy=3)
        cv2.imshow('SampleFromH', images_sample)

    if is_show_multi_rec:
        X_MULTI,Y_MULTI = multi_iter.__next__()
        X_REC1,X_REC2 = sess.run(net.x_recs, feed_dict={net.x: X_MULTI, net.y: Y_MULTI, net.is_training:False})
        # turn the composed image to be 3 channel gray image
        images_org = np.stack([X_MULTI[:num_show,:,:,0]]*3,axis=-1)
        black = np.zeros([num_show, 212, 212, 1])
        images_recs = np.concatenate([black, X_REC1, X_REC2], axis=-1)
        images_rec1 = np.concatenate([black, black, X_REC2], axis=-1)
        images_rec2 = np.concatenate([black, X_REC1, black], axis=-1)
        image_show = np.concatenate([images_org, images_recs, images_rec1, images_rec2], axis=2)
        image_show = cv2.resize(np.concatenate(image_show, axis=0), dsize=(0, 0), fx=3, fy=3)
        cv2.imshow('MultigalaxyReconstruction', image_show)

    if is_show_multi_rec or is_multi_galaxy:
        key = cv2.waitKey(1)

    if key == ord('s'):
        cv2.imwrite('MultigalaxyReconstruction%d.png' % irunloss_reconstruction_on_test_set, image_show * 255.0)
        cv2.imwrite('SampleFromH%d.png' % irun, images_sample * 255.0)
    """

    if (irun+1) % save_frequence == 0:
        saver.save(sess, './cpt/model2-{}.ckpt'.format(irun+1))

    if (irun+1) % decay_frequence == 0 and lr > min_lr:
        lr *= 0.5

    irun += 1


text:
import os
import time
from collections import namedtuple
from logging import DEBUG, getLogger

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.distributions import Categorical
from torch.utils.data import DataLoader

from accel.explorers.epsilon_greedy import LinearDecayEpsilonGreedy
from accel.replay_buffers.rollout_buffer import RolloutBuffer, Transition

logger = getLogger(__name__)

class PPO:
    def __init__(self, envs, eval_env, steps, actor, critic,
                 device, lmd=0.95, gamma=0.99, batch_size=128,
                 lr=2.5e-4, horizon=128, clip_eps=0.2, epoch_per_update=4, entropy_coef=0.01,
                 load="", eval_interval=50000, epoch_per_eval=3, comet_exp=None, value_loss_coef=0.5,
                 value_clipping=True, atari=False):
        self.envs = envs
        self.eval_env = eval_env
        self.lmd = lmd
        self.gamma = gamma
        self.device = device
        self.batch_size = batch_size
        self.clip_eps = clip_eps
        self.epoch_per_update = epoch_per_update
        self.entropy_coef = entropy_coef
        self.eval_interval = eval_interval
        self.epoch_per_eval = epoch_per_eval
        self.lr = lr
        self.value_loss_coef = value_loss_coef
        self.value_clipping = value_clipping
        self.atari = atari

        self.steps = steps
        self.horizon = horizon
        self.elapsed_step = 0
        self.best_score = -1e10
        self.max_grad_norm = 0.5

        self.actor = actor.to(device)
        self.critic = critic.to(device)
        self.optimizer = optim.Adam(
            list(self.actor.parameters()) + list(self.critic.parameters()), lr=lr, eps=1e-5)

        # TODO detach scheduler from epsilon greedy
        self.lr_scheduler = LinearDecayEpsilonGreedy(
            start_eps=1, end_eps=0.01, decay_steps=self.steps)

        if load:
            self.actor.load_state_dict(torch.load(
                f'{load}/actor.model', map_location=device))
            self.critic.load_state_dict(torch.load(
                f'{load}/critic.model', map_location=device))

        self.comet_exp = comet_exp

    def act(self, obs, greedy=False):
        pass

    def update(self, obs, action, next_obs, reward, valid):
        pass

    def train(self):
        pass

    def run(self):
        self.train_start_time = time.time()
        self.log_file_name = 'scores.txt'

        obs = self.envs.reset()

        buffer = RolloutBuffer(self.gamma, self.lmd)
        self.next_eval_cnt = 1

        while self.elapsed_step < self.steps:
            # collect trajectories
            buffer.clear()
            self.critic.eval()
            self.actor.eval()
            for i in range(self.horizon):
                obs_tensor = torch.tensor(
                    obs, dtype=torch.float32).to(self.device)

                with torch.no_grad():
                    action_logits = self.actor(obs_tensor)
                dist = Categorical(logits=action_logits)
                actions = dist.sample()
                log_prob = dist.log_prob(actions).cpu().numpy()
                actions = actions.cpu().numpy()
                next_obs, reward, done, info = self.envs.step(actions)

                transition = Transition(
                    obs, actions, next_obs, reward, ~done, log_prob)

                with torch.no_grad():
                    values = self.critic(
                        obs_tensor).flatten().detach().cpu().numpy()
                buffer.push(transition, values)

                obs = next_obs

            self.elapsed_step += self.horizon * self.envs.num_envs

            obs_tensor = torch.tensor(obs, dtype=torch.float32).to(self.device)
            with torch.no_grad():
                values = self.critic(
                    obs_tensor).flatten().detach().cpu().numpy()

            # compute advantage estimates A_t using GAE
            buffer.final_state_value(values)

            next_lr = self.lr_scheduler.calc_eps(self.elapsed_step) * self.lr
            for pg in self.optimizer.param_groups:
                pg['lr'] = next_lr

            dataset = buffer.create_dataset()
            dataloader = DataLoader(
                dataset, batch_size=self.batch_size, shuffle=True)

            value_loss_epoch = 0.
            actor_loss_epoch = 0.
            entropy_bonus_epoch = 0.
            added_cnt = 0

            self.critic.train()
            self.actor.train()
            for _ in range(self.epoch_per_update):
                for (ob_, ret_, ac_, log_prob_old_,
                     gae_, value_) in dataloader:
                    added_cnt += 1
                    ob_, ac_ = ob_.to(self.device), ac_.to(self.device)
                    log_prob_old_ = log_prob_old_.to(self.device)
                    gae_ = gae_.to(self.device)

                    # update the policy by maximizing PPO-Clip objective
                    action_logits = self.actor(ob_)
                    dist = Categorical(logits=action_logits)
                    log_prob = dist.log_prob(ac_)
                    entropy_bonus = dist.entropy().mean()
                    ratio = torch.exp(log_prob - log_prob_old_)

                    surr1 = ratio * gae_
                    surr2 = torch.clip(ratio, 1 - self.clip_eps,
                                       1 + self.clip_eps) * gae_

                    # minus means "ascent"
                    actor_loss = -torch.min(surr1, surr2).mean()

                    # value function learning
                    ret_, value_ = ret_.to(self.device), value_.to(self.device)
                    pred = self.critic(ob_).flatten()

                    if self.value_clipping:
                        clipped_pred = value_ + \
                            (pred - value_).clamp(-self.clip_eps, self.clip_eps)

                        value_loss = (pred - ret_).pow(2)
                        clipped_value_loss = (clipped_pred - ret_).pow(2)
                        value_loss = torch.max(
                            value_loss, clipped_value_loss).mean() / 2

                    else:
                        value_loss = F.mse_loss(pred, ret_) / 2

                    loss = value_loss * self.value_loss_coef + \
                        actor_loss - entropy_bonus * self.entropy_coef

                    self.optimizer.zero_grad()
                    loss.backward()
                    nn.utils.clip_grad_norm_(
                        list(self.actor.parameters()) + list(self.critic.parameters()), self.max_grad_norm)
                    self.optimizer.step()

                    value_loss_epoch += value_loss.item()
                    actor_loss_epoch += actor_loss.item()
                    entropy_bonus_epoch += entropy_bonus.item()

            value_loss_epoch /= added_cnt
            actor_loss_epoch /= added_cnt
            entropy_bonus_epoch /= added_cnt

            self.critic.eval()
            self.actor.eval()

            if self.elapsed_step >= self.next_eval_cnt * self.eval_interval:
                if self.comet_exp is not None:
                    dic = {
                        'value_loss': value_loss_epoch,
                        'actor_loss': actor_loss_epoch,
                        'entropy_bonus': entropy_bonus_epoch
                    }
                    self.comet_exp.log_metrics(dic, step=self.elapsed_step)

                self.evaluate()

        if self.comet_exp is not None:
            now = time.time()
            elapsed = now - self.train_start_time
            self.comet_exp.log_metric(
                'duration', np.round(
                    elapsed / 60 / 60, 2))
        print('Complete')

    def evaluate(self):
        self.next_eval_cnt += 1

        total_reward = 0.

        for _ in range(self.epoch_per_eval):
            while True:
                done = False
                obs = self.eval_env.reset()

                while not done:
                    obs_tensor = torch.tensor(
                        obs, dtype=torch.float32).unsqueeze(0).to(self.device)
                    with torch.no_grad():
                        action_logits = self.actor(obs_tensor)
                    dist = Categorical(logits=action_logits)
                    actions = dist.sample().cpu().numpy()

                    obs, reward, done, _ = self.eval_env.step(actions[0])

                    total_reward += reward

                if self.atari:
                    if self.eval_env.was_real_done:
                        break
                else:
                    if done:
                        break

        total_reward /= self.epoch_per_eval
        if self.comet_exp is not None:
            self.comet_exp.log_metric(
                'reward', total_reward, step=self.elapsed_step)

        if total_reward > self.best_score:
            self.best_score = total_reward
            dirname = f'{self.elapsed_step}'
            if not os.path.exists(dirname):
                os.mkdir(dirname)

            model_name = f'{dirname}/actor.model'
            torch.save(self.actor.state_dict(), model_name)
            if self.comet_exp is not None:
                self.comet_exp.log_model(
                    'best_actor', model_name, overwrite=True)

            model_name = f'{dirname}/critic.model'
            torch.save(self.critic.state_dict(), model_name)
            if self.comet_exp is not None:
                self.comet_exp.log_model(
                    'best_critic', model_name, overwrite=True)

        now = time.time()
        elapsed = now - self.train_start_time
        log = f'{self.elapsed_step} {total_reward:.1f} {elapsed:.1f}\n'

        logger.info(
            f'Eval result | total_step: {self.elapsed_step} '
            f'score: {total_reward:.1f} elapsed: {elapsed:.1f}')

        with open(self.log_file_name, 'a') as f:
            f.write(log)

max_stars_repo_name: waffoo/accel

max_issues_repo_name: waffoo/accel

max_forks_repo_name: waffoo/accel


text:
import os
import tensorflow as tf
import math
import numpy as np
import itertools
import cv2
import pickle
import copy
import torch
from pathlib import Path
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
'''
0 = all messages are logged (default behavior)
1 = INFO messages are not printed
2 = INFO and WARNING messages are not printed
3 = INFO, WARNING, and ERROR messages are not printed
'''

from waymo_open_dataset.utils import frame_utils, transform_utils, range_image_utils
from waymo_open_dataset import dataset_pb2
from waymo_open_dataset import label_pb2
from waymo_open_dataset.protos import metrics_pb2
'''
0 = all messages are logged (default behavior)
1 = INFO messages are not printed
2 = INFO and WARNING messages are not printed
3 = INFO, WARNING, and ERROR messages are not printed
'''
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

enum_LabelType = label_pb2.Label.Type

class FrameParser:
    def __init__(self,):
        pass

    @staticmethod
    def parse_tfrecord(tfrecord_file):
        tfrecords = tf.data.TFRecordDataset(tfrecord_file, compression_type='')
        return tfrecords

    @staticmethod
    def parse_tfrecord_to_frame(tfrecord_file):
        tfrecord = FrameParser.parse_tfrecord(tfrecord_file)
        for data in tfrecord:
            frame = dataset_pb2.Frame()
            frame.ParseFromString(bytearray(data.numpy()))
            frame.images.sort(key=lambda image: image.name)
            frame.camera_labels.sort(key=lambda label: label.name)
            frame.lasers.sort(key=lambda laser: laser.name)
            yield frame

    # @staticmethod
    # def objects2label_format(input_file, output_dir):
    #     '''
    #     transfer metrics_pb2.Objects to dataset_pb2.Label format
    #     :return:
    #     '''
    #     streams = open(input_file, 'rb').read()
    #     objects = metrics_pb2.Objects()
    #     objects.ParseFromString(bytearray(streams))
    #
    #     frame_labels_map = {}
    #     for object in objects.objects:
    #         context_name = object.context_name
    #         frame_timestamp_micros = object.frame_timestamp_micros
    #         label = object.object
    #         score = object.score
    #         camera_name = object.camera_name
    #
    #         if context_name not in frame_labels_map:
    #             frame_labels_map[context_name] = {}
    #         frame_labels_map[context_name][frame_timestamp_micros] = label
    #
    #     for frame, labels in frame_labels_map.items():
    #         _keys = frame_labels_map[frame].sort()
    #         for index, key in enumerate(_keys):
    #             frame_id = frame + '_%s' % index
    #             with open(Path(output_dir) / frame_id + '.bin', 'wb') as f:
    #                 pickle.dump(frame_labels_map[frame][key],f)

    @staticmethod
    def check_numpy_to_torch(x):
        if isinstance(x, np.ndarray):
            return torch.from_numpy(x).float(), True
        return x, False

    @staticmethod
    def rotate_points_along_z(points, angle):
        """
        Args:
            points: (B, N, 3 + C)
            angle: (B), angle along z-axis, angle increases x ==> y
        Returns:

        """
        points, is_numpy = FrameParser.check_numpy_to_torch(points)
        angle, _ = FrameParser.check_numpy_to_torch(angle)

        cosa = torch.cos(angle)
        sina = torch.sin(angle)
        zeros = angle.new_zeros(points.shape[0])
        ones = angle.new_ones(points.shape[0])
        rot_matrix = torch.stack((
            cosa, sina, zeros,
            -sina, cosa, zeros,
            zeros, zeros, ones
        ), dim=1).view(-1, 3, 3).float()
        points_rot = torch.matmul(points[:, :, 0:3], rot_matrix)
        points_rot = torch.cat((points_rot, points[:, :, 3:]), dim=-1)
        return points_rot.numpy() if is_numpy else points_rot

    @staticmethod
    def boxes_to_corners_3d(boxes3d):
        """
            7 -------- 4
           /|         /|
          6 -------- 5 .
          | |        | |
          . 3 -------- 0
          |/         |/
          2 -------- 1
        Args:
            boxes3d:  (N, 7) [x, y, z, dx, dy, dz, heading], (x, y, z) is the box center

        Returns:
        """
        boxes3d, is_numpy = FrameParser.check_numpy_to_torch(boxes3d)
        template = boxes3d.new_tensor((
            [1, 1, -1], [1, -1, -1], [-1, -1, -1], [-1, 1, -1],
            [1, 1, 1], [1, -1, 1], [-1, -1, 1], [-1, 1, 1],
        )) / 2

        corners3d = boxes3d[:, None, 3:6].repeat(1, 8, 1) * template[None, :, :]
        corners3d = FrameParser.rotate_points_along_z(corners3d.view(-1, 8, 3), boxes3d[:, 6]).view(-1, 8, 3)
        corners3d += boxes3d[:, None, 0:3]

        return corners3d.numpy() if is_numpy else corners3d

    @staticmethod
    def show_image_with_box(image, boxes, categorys, output_dir, name=None, thickness=1, font_scale=0.3):
        image = copy.deepcopy(image)
        image = np.ascontiguousarray(image[:, :, ::-1])  # RGB -> BGR
        boxes = copy.deepcopy(boxes)
        color = (0, 255, 255)
        for i in range(len(boxes)):
            # extract the bounding box coordinates
            cx, cy = boxes[i][0], boxes[i][1]
            w, h = boxes[i][2], boxes[i][3]
            xmin, ymin = cx - w / 2, cy - h / 2
            xmin, ymin, w, h = map(int, [xmin, ymin, w, h])
            category_index = categorys[i]
            category = enum_LabelType.Name(category_index)

            # draw a bounding box rectangle and label on the image
            cv2.rectangle(image, (xmin, ymin), (xmin + w, ymin + h), color=color, thickness=thickness)
            confidence = 1.0
            text = f"{category}: {confidence:.2f}"

            # calculate text width & height to draw the transparent boxes as background of the text
            (text_width, text_height) = \
                cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, fontScale=font_scale, thickness=thickness)[0]
            text_offset_x = xmin
            text_offset_y = ymin - 5
            box_coords = ((text_offset_x, text_offset_y), (text_offset_x + text_width + 2, text_offset_y - text_height))

            overlay = image.copy()
            cv2.rectangle(overlay, box_coords[0], box_coords[1], color=color, thickness=cv2.FILLED)
            # add opacity (transparency to the box)
            image = cv2.addWeighted(overlay, 0.6, image, 0.4, 0)
            # now put the text (label: confidence %)
            cv2.putText(image, text, (xmin, ymin - 5), cv2.FONT_HERSHEY_SIMPLEX,
                        fontScale=font_scale, color=(0, 0, 0), thickness=thickness)

        cv2.imwrite(os.path.join(output_dir, "image_detection_result_{}.jpg".format(name)), image)

    @staticmethod
    def show_laser_with_box(
            pcds, pcds_color=None,
            box3ds=None, box3d_color=[255, 0, 0],
            categorys=None,
            pred_box3ds=None, pred_box3d_color=[0, 0, 255],
            pred_categorys=None,
            x_range=(-40, 62.4), y_range=(-40, 40), z_range=(-3, 8),
            name='laser'
    ):
        '''

        :param pcds: pcds numpy float32 array (n, c)
        :param pcds_color: pcds_color uint8 array (n, 3) denotes the color of each point 0~255
        :param box3ds: box3d_corners float32 numpy array; eight corners of a 3D bounding box, [x,y,z,l,w,h,heading]
        :param box3d_color:
        :param categorys:
        :param pred_box3ds:
        :param pred_box3d_color:
        :param pred_categorys:
        :param x_range:
        :param y_range:
        :param z_range:
        :param name:
        :return:
        '''
        import open3d as o3d
        #
        #
        #
        pcd_l = o3d.geometry.PointCloud()
        pcd_l.points = o3d.utility.Vector3dVector(pcds[:, :3])
        if pcds_color is not None:
            pcd_l.colors = o3d.utility.Vector3dVector(pcds_color)

        lines = [[0, 1], [1, 5], [5, 4], [4, 0],
                 [3, 2], [2, 6], [6, 7], [7, 3],
                 [0, 3], [4, 7], [1, 2], [5, 6]]

        line_sets = []
        if box3ds is not None:
            box3d_corners = FrameParser.boxes_to_corners_3d(box3ds)
            for box3d_corner in box3d_corners:
                line_set = o3d.geometry.LineSet()
                line_set.points = o3d.utility.Vector3dVector(box3d_corner)

                line_set.lines = o3d.utility.Vector2iVector(lines)
                line_set.colors = o3d.utility.Vector3dVector([box3d_color for i in range(len(lines))])

                line_sets.append(line_set)

        if pred_box3ds is not None:
            pred_box3d_corners = FrameParser.boxes_to_corners_3d(pred_box3ds)
            for box3d_corner in pred_box3d_corners:
                line_set = o3d.geometry.LineSet()
                line_set.points = o3d.utility.Vector3dVector(box3d_corner)

                line_set.lines = o3d.utility.Vector2iVector(lines)
                line_set.colors = o3d.utility.Vector3dVector([pred_box3d_color for i in range(len(lines))])

                line_sets.append(line_set)

        line_set = o3d.geometry.LineSet()
        corners = []
        for z in z_range:
            corners.append([x_range[0], y_range[0], z])
            corners.append([x_range[0], y_range[1], z])
            corners.append([x_range[1], y_range[1], z])
            corners.append([x_range[1], y_range[0], z])

        line_set.points = o3d.utility.Vector3dVector(corners)
        colors = [[0, 0, 0] for i in range(len(lines))]
        line_set.lines = o3d.utility.Vector2iVector(lines)
        line_set.colors = o3d.utility.Vector3dVector(colors)
        line_sets.append(line_set)
        o3d.visualization.draw_geometries([*line_sets, pcd_l], window_name=name)

    @staticmethod
    def convert_range_image_to_point_cloud(frame,
                                           range_images,
                                           camera_projections,
                                           range_image_top_pose,
                                           ri_index=0):
        """Convert range images to point cloud.
        clone from waymo_open_dataset(https://github.com/waymo-research/waymo-open-dataset)
        Args:
          frame: open dataset frame
           range_images: A dict of {laser_name, [range_image_first_return,
             range_image_second_return]}.
           camera_projections: A dict of {laser_name,
             [camera_projection_from_first_return,
             camera_projection_from_second_return]}.
          range_image_top_pose: range image pixel pose for top lidar.
          ri_index: 0 for the first return, 1 for the second return.

        Returns:
          points: {[N, 3]} list of 3d lidar points of length 5 (number of lidars). [x, y, z]
          cp_points: {[N, 6]} list of camera projections of length 5
            (number of lidars).
        """
        calibrations = sorted(frame.context.laser_calibrations, key=lambda c: c.name)
        points = []
        cp_points = []

        frame_pose = tf.convert_to_tensor(
            value=np.reshape(np.array(frame.pose.transform), [4, 4]))
        # [H, W, 6]
        range_image_top_pose_tensor = tf.reshape(
            tf.convert_to_tensor(value=range_image_top_pose.data),
            range_image_top_pose.shape.dims)
        # [H, W, 3, 3]
        range_image_top_pose_tensor_rotation = transform_utils.get_rotation_matrix(
            range_image_top_pose_tensor[..., 0], range_image_top_pose_tensor[..., 1],
            range_image_top_pose_tensor[..., 2])
        range_image_top_pose_tensor_translation = range_image_top_pose_tensor[..., 3:]
        range_image_top_pose_tensor = transform_utils.get_transform(
            range_image_top_pose_tensor_rotation,
            range_image_top_pose_tensor_translation)
        for c in calibrations:
            range_image = range_images[c.name][ri_index]
            if len(c.beam_inclinations) == 0:  # pylint: disable=g-explicit-length-test
                beam_inclinations = range_image_utils.compute_inclination(
                    tf.constant([c.beam_inclination_min, c.beam_inclination_max]),
                    height=range_image.shape.dims[0])
            else:
                beam_inclinations = tf.constant(c.beam_inclinations)

            beam_inclinations = tf.reverse(beam_inclinations, axis=[-1])
            extrinsic = np.reshape(np.array(c.extrinsic.transform), [4, 4])

            range_image_tensor = tf.reshape(
                tf.convert_to_tensor(value=range_image.data), range_image.shape.dims)
            pixel_pose_local = None
            frame_pose_local = None
            if c.name == dataset_pb2.LaserName.TOP:
                pixel_pose_local = range_image_top_pose_tensor
                pixel_pose_local = tf.expand_dims(pixel_pose_local, axis=0)
                frame_pose_local = tf.expand_dims(frame_pose, axis=0)
            range_image_mask = range_image_tensor[..., 0] > 0
            range_image_cartesian = range_image_utils.extract_point_cloud_from_range_image(
                tf.expand_dims(range_image_tensor[..., 0], axis=0),
                tf.expand_dims(extrinsic, axis=0),
                tf.expand_dims(tf.convert_to_tensor(value=beam_inclinations), axis=0),
                pixel_pose=pixel_pose_local,
                frame_pose=frame_pose_local)

            range_image_cartesian = tf.squeeze(range_image_cartesian, axis=0)
            points_tensor = tf.gather_nd(range_image_cartesian,
                                         tf.compat.v1.where(range_image_mask))

            cp = camera_projections[c.name][ri_index]
            cp_tensor = tf.reshape(tf.convert_to_tensor(value=cp.data), cp.shape.dims)
            cp_points_tensor = tf.gather_nd(cp_tensor,
                                            tf.compat.v1.where(range_image_mask))
            points.append(points_tensor.numpy())
            cp_points.append(cp_points_tensor.numpy())

        return points, cp_points

    @staticmethod
    def convert_range_image_to_point_cloud_V2(frame,
                                           range_images,
                                           camera_projections,
                                           range_image_top_pose,
                                           ri_index=0):
        """Convert range images to point cloud.

        Args:
          frame: open dataset frame
           range_images: A dict of {laser_name, [range_image_first_return,
             range_image_second_return]}.
           camera_projections: A dict of {laser_name,
             [camera_projection_from_first_return,
             camera_projection_from_second_return]}.
          range_image_top_pose: range image pixel pose for top lidar.
          ri_index: 0 for the first return, 1 for the second return.

        Returns:
          points: {[N, 4]} list of 3d lidar points of length 5 (number of lidars). [x, y, z, intensity, elongation]
          cp_points: {[N, 6]} list of camera projections of length 5
            (number of lidars).
        """
        calibrations = sorted(frame.context.laser_calibrations, key=lambda c: c.name)
        points = []
        cp_points = []

        frame_pose = tf.convert_to_tensor(
            value=np.reshape(np.array(frame.pose.transform), [4, 4]))
        # [H, W, 6]
        range_image_top_pose_tensor = tf.reshape(
            tf.convert_to_tensor(value=range_image_top_pose.data),
            range_image_top_pose.shape.dims)
        # [H, W, 3, 3]
        range_image_top_pose_tensor_rotation = transform_utils.get_rotation_matrix(
            range_image_top_pose_tensor[..., 0], range_image_top_pose_tensor[..., 1],
            range_image_top_pose_tensor[..., 2])
        range_image_top_pose_tensor_translation = range_image_top_pose_tensor[..., 3:]
        range_image_top_pose_tensor = transform_utils.get_transform(
            range_image_top_pose_tensor_rotation,
            range_image_top_pose_tensor_translation)
        for c in calibrations:
            range_image = range_images[c.name][ri_index]
            if len(c.beam_inclinations) == 0:  # pylint: disable=g-explicit-length-test
                beam_inclinations = range_image_utils.compute_inclination(
                    tf.constant([c.beam_inclination_min, c.beam_inclination_max]),
                    height=range_image.shape.dims[0])
            else:
                beam_inclinations = tf.constant(c.beam_inclinations)

            beam_inclinations = tf.reverse(beam_inclinations, axis=[-1])
            extrinsic = np.reshape(np.array(c.extrinsic.transform), [4, 4])

            range_image_tensor = tf.reshape(
                tf.convert_to_tensor(value=range_image.data), range_image.shape.dims)
            pixel_pose_local = None
            frame_pose_local = None
            if c.name == dataset_pb2.LaserName.TOP:
                pixel_pose_local = range_image_top_pose_tensor
                pixel_pose_local = tf.expand_dims(pixel_pose_local, axis=0)
                frame_pose_local = tf.expand_dims(frame_pose, axis=0)
            range_image_mask = range_image_tensor[..., 0] > 0
            range_image_cartesian = range_image_utils.extract_point_cloud_from_range_image(
                tf.expand_dims(range_image_tensor[..., 0], axis=0),
                tf.expand_dims(extrinsic, axis=0),
                tf.expand_dims(tf.convert_to_tensor(value=beam_inclinations), axis=0),
                pixel_pose=pixel_pose_local,
                frame_pose=frame_pose_local)
            range_image_cartesian = tf.squeeze(range_image_cartesian, axis=0)
            # see waymo_open_dataset/dataset.proto RangeImage.range_image_compressed
            range_image_intensity = tf.expand_dims(range_image_tensor[..., 1], axis=-1)
            range_image_elongation = tf.expand_dims(range_image_tensor[..., 2], axis=-1)
            range_image_features = tf.concat((range_image_cartesian, range_image_intensity, range_image_elongation), axis=-1)
            points_tensor = tf.gather_nd(range_image_features,
                                         tf.compat.v1.where(range_image_mask))

            cp = camera_projections[c.name][ri_index]
            cp_tensor = tf.reshape(tf.convert_to_tensor(value=cp.data), cp.shape.dims)
            cp_points_tensor = tf.gather_nd(cp_tensor,
                                            tf.compat.v1.where(range_image_mask))
            points.append(points_tensor.numpy())
            cp_points.append(cp_points_tensor.numpy())

        return points, cp_points

    @staticmethod
    def parse_image(frame, camera_name='UNKNOWN', parse_label=True, output_dir='', vis=False):
        assert camera_name in dataset_pb2.CameraName.Name.keys()
        camera_name_index = dataset_pb2.CameraName.Name.Value(camera_name)
        image_array = None
        for image in frame.images:
            if image.name == camera_name_index:
                image_tensor = tf.image.decode_jpeg(image.image)
                image_array = image_tensor.numpy()
                break
        camera_labels = None
        if parse_label:
            for camera_labels in frame.camera_labels:
                if camera_labels.name == camera_name_index:
                    camera_labels = camera_labels.labels
                    if vis:
                        boxes, categorys = [], []
                        for label in camera_labels:
                            box, category = label.box, label.type
                            # see https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/label.proto
                            center_x, center_y, w, h = box.center_x, box.center_y, box.length, box.width
                            boxes.append([center_x, center_y, w, h])
                            categorys.append(category)
                        boxes = np.asarray(boxes)
                        categorys = np.asarray(categorys)
                        FrameParser.show_image_with_box(image_array, boxes, categorys, output_dir, camera_name)
                    break
        if image_array is None:
            success = False
        else:
            success = True
        return success, image_array, camera_labels

    @staticmethod
    def drop_laser_labels_with_name(laser_labels, keeped_category_types):
        '''
        only keep laser label which category in category_names
        :param laser_labels: iterative label_pb2.Label
        :param keeped_category_types: list string
        :return: iterative label_pb2.Label
        '''
        _laser_labels = []
        for laser_label in laser_labels:
            _type = label_pb2.Label.Type.Name(laser_label.type)
            if _type in keeped_category_types:
                _laser_labels.append(laser_label)
        return _laser_labels

    @staticmethod
    def parse_laser_labels(laser_labels):
        laser_box3d, laser_box3d_categorys, detection_difficulty_level, num_lidar_points_in_box = [], [], [], []
        for laser_label in laser_labels:
            box = laser_label.box
            center_x, center_y, center_z = box.center_x, box.center_y, box.center_z
            width, length, height = box.width, box.length, box.height
            heading = box.heading
            category = enum_LabelType.Name(laser_label.type)
            # see https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/label.proto
            laser_box3d.append([center_x, center_y, center_z, length, width, height, heading])
            laser_box3d_categorys.append(category)
            detection_difficulty_level.append(laser_label.detection_difficulty_level)
            num_lidar_points_in_box.append(laser_label.num_lidar_points_in_box)
        laser_box3d = np.asarray(laser_box3d)
        laser_box3d_categorys = np.asarray(laser_box3d_categorys)
        detection_difficulty_level = np.asarray(detection_difficulty_level)
        num_lidar_points_in_box = np.asarray(num_lidar_points_in_box)
        return laser_box3d, laser_box3d_categorys, detection_difficulty_level, num_lidar_points_in_box

    @staticmethod
    def parse_laser(frame, parse_label=True, output_dir="", vis=False):
        '''

        :param parse_label:
        :param vis:
        :return:
            points_all: np.ndarray
                [N, 4] list of 3d lidar points of length 3 or 5.
                [x, y, z, intensity(optional), elongation(optional)]
            laser labels: format https://github.com/waymo-research/waymo-open-dataset/blob/master/waymo_open_dataset/label.proto
                message Label {
                  // Upright box, zero pitch and roll.
                  message Box {
                    optional double center_x = 1;
                    optional double center_y = 2;
                    optional double center_z = 3;
                    optional double length = 5;
                    optional double width = 4;
                    optional double height = 6;
                    optional double heading = 7;
                  }
                  optional Box box = 1;

                  enum Type {
                    TYPE_UNKNOWN = 0;
                    TYPE_VEHICLE = 1;
                    TYPE_PEDESTRIAN = 2;
                    TYPE_SIGN = 3;
                    TYPE_CYCLIST = 4;
                  }
                  optional Type type = 3;

                  enum DifficultyLevel {
                    UNKNOWN = 0;
                    LEVEL_1 = 1;
                    LEVEL_2 = 2;
                  }
                  optional DifficultyLevel detection_difficulty_level = 5;
                  optional DifficultyLevel tracking_difficulty_level = 6;
                  optional int32 num_lidar_points_in_box = 7;
                }
        '''
        range_images, camera_projections, range_image_top_pose = frame_utils.parse_range_image_and_camera_projection(frame)
        points, cp_points = FrameParser.convert_range_image_to_point_cloud_V2(
            frame,
            range_images,
            camera_projections,
            range_image_top_pose,
            ri_index=0
        )

        # 3d points in vehicle frame.
        points_all = np.concatenate(points, axis=0)
        # camera projection corresponding to each point.
        cp_points_all = np.concatenate(cp_points, axis=0)
        laser_labels = frame.laser_labels
        if parse_label:
            if vis:
                laser_box3d, laser_box3d_categorys, detection_difficulty_level, num_lidar_points_in_box \
                    = FrameParser.parse_laser_labels(laser_labels)
                np.save(os.path.join(output_dir, "laser_points.npy"), points_all)
                np.save(os.path.join(output_dir, "laser_cp_points.npy"), cp_points_all)
                np.save(os.path.join(output_dir, "laser_box3d.npy"), laser_box3d)
                np.save(os.path.join(output_dir, "laser_box3d_categorys.npy"), laser_box3d_categorys)
                # FrameParser.show_laser_with_box(pcds=points_all, box3ds=boxes, categorys=categorys)
        return True, points_all, cp_points_all, laser_labels

    @staticmethod
    def extract_image_features_to_laser_points(camera_images, camera_labels, point2pixel, points=None, camera_focus_info=None):
        """

        :param camera_images: dict, key:camera_name value:pixel_array
        :param camera_labels: dict, key:camera_name value:box label in label_pb2.Label format
        :param point2pixel: np.array , point to pixel map in RangeImage.camera_projection_compressed format ,shape N*6,
        :return:
        """
        # Lidar point to camera image projections. A point can be projected to
        # multiple camera images. We pick the first two at the following order:
        # [FRONT, FRONT_LEFT, FRONT_RIGHT, SIDE_LEFT, SIDE_RIGHT].
        multi_camera, points_num, dims = point2pixel.shape
        point2pixel = point2pixel.reshape((points_num * multi_camera , dims))  # index, loc_x, loc_y
        if points is not None:
            assert points.shape[0] == points_num
            multi_points = points.reshape(1, points_num, -1).\
                repeat(multi_camera, axis=0).\
                reshape(multi_camera * points_num, -1)
        semantic_features = np.zeros((points_num * multi_camera, 1), dtype=np.float)  # category
        geometric_features = np.zeros((points_num * multi_camera, 5), dtype=np.float) # see ImVoteNet paper
        texture_features = np.zeros((points_num * multi_camera, 3), dtype=np.float)  # RGB

        # convert camera_labels to np.array format
        camera_labels_array = {}
        for camera_name, camera_label in camera_labels.items():
            camera_labels_array[camera_name] = np.asarray(
                [[_label.box.center_x, _label.box.center_y, _label.box.length, _label.box.width, _label.type] for _label in camera_label]
            )

        camera_names = camera_images.keys()
        for camera_name in camera_names:
            camera_label = camera_labels_array[camera_name]
            np.random.shuffle(camera_label)  # shuttle array in axis 0
            camera_index = dataset_pb2.CameraName.Name.Value(camera_name)
            camera_image = camera_images[camera_name]
            # if camera_index == 0:  # not projection to any image
            #     continue

            index = point2pixel[:, 0] == camera_index
            if index.any() is False:
                continue
            N = index.sum()
            pixel_loc_x = point2pixel[index, 1]
            pixel_loc_y = point2pixel[index, 2]

            # extract texture features
            texture_features[index] = camera_image[pixel_loc_y, pixel_loc_x]

            # extract semantic features
            # extract geometric features
            if camera_label.size == 0:
                semantic_features[index, 0] = -1  # background
                # geometric_features[index] = 0
            else:
                top_left_x = camera_label[:, 0] - camera_label[:, 2] / 2
                top_left_y = camera_label[:, 1] - camera_label[:, 3] / 2
                bottom_right_x = camera_label[:, 0] + camera_label[:, 2] / 2
                bottom_right_y = camera_label[:, 1] + camera_label[:, 3] / 2
                # N * M, N points num, M boxes num
                match = (pixel_loc_x[:, None] > top_left_x[None, :]) \
                          & (pixel_loc_x[:, None] < bottom_right_x[None, :]) \
                          & (pixel_loc_y[:, None] > top_left_y[None, :]) \
                          & (pixel_loc_y[:, None] < bottom_right_y[None, :])
                # assert match.sum(axis=1) <= 1  # point may matched to multi box
                label_feature = np.zeros(N, np.float)
                geometric_feature = np.zeros((N, 5), np.float)
                bg = match.any(axis=1) == False
                label_feature[bg] = -1  # background
                geometric_feature[bg] = 0

                fg = match.any(axis=1) == True
                match_fg = match[fg]
                box_argmax = match_fg.argmax(axis=1)  # find first matched box
                matched_camera_label = camera_label[box_argmax, -1]  # -1 is the label of box
                matched_camera_center_x = camera_label[box_argmax, 0]
                matched_camera_center_y = camera_label[box_argmax, 1]

                label_feature[fg] = matched_camera_label
                semantic_features[index, 0] = label_feature

                delta_u = pixel_loc_x[fg] - matched_camera_center_x
                delta_v = pixel_loc_y[fg] - matched_camera_center_y
                assert points is not None
                assert camera_focus_info is not None
                assert points.shape[0] == points_num
                f_u, f_v = camera_focus_info[camera_name]['f_u'], \
                           camera_focus_info[camera_name]['f_v']
                point_x = multi_points[index][fg, 0]
                point_y = multi_points[index][fg, 1]
                point_z = multi_points[index][fg, 2]
                # NOTE that u,v in Image correspond to y,z in Laser coordinate
                depth = point_x
                oc_ = np.stack(
                    (point_y + delta_u / f_u * depth, point_z + delta_v / f_v * depth, depth), axis=-1)
                oc_len = np.linalg.norm(oc_, axis=1, keepdims=True)
                oc_normalize = oc_ / oc_len
                geometric_feature[fg, 0] = delta_u / f_u * depth
                geometric_feature[fg, 1] = delta_v / f_v * depth
                geometric_feature[fg, 2:5] = oc_normalize
                geometric_features[index] = geometric_feature

        # if laser point matched to multi camera image, only keep one matched feature
        point2pixel = point2pixel.reshape(multi_camera, points_num, -1)
        semantic_features = semantic_features.reshape(multi_camera, points_num, -1)
        texture_features = texture_features.reshape(multi_camera, points_num, -1)
        geometric_features = geometric_features.reshape(multi_camera, points_num, -1)
        _semantic_features = np.zeros(shape=(points_num, 1), dtype=np.float)
        _geometric_features = np.zeros(shape=(points_num, 5), dtype=np.float)
        _texture_features = np.zeros(shape=(points_num, 3), dtype=np.float)  # RGB
        for i in range(multi_camera):
            _point2pixel = point2pixel[i]
            fg = _point2pixel[:, 0] != 0
            _semantic_features[fg] = semantic_features[i][fg]
            _texture_features[fg] = texture_features[i][fg]
            _geometric_features[fg] = geometric_features[i][fg]
        _texture_features = _texture_features / 256  # normalize to [0,1]
        return np.concatenate((_semantic_features, _texture_features, _geometric_features), axis=-1)

if __name__ == '__main__':
    _dir = '/data/'
    tfrecord_file = os.path.join(_dir, 'frames')
    for frame in FrameParser.parse_tfrecord_to_frame(tfrecord_file):
        FrameParser.parse_image(frame, 'FRONT', vis=True)
        FrameParser.parse_image(frame, 'FRONT_LEFT', vis=True)
        FrameParser.parse_image(frame, 'FRONT_RIGHT', vis=True)
        FrameParser.parse_image(frame, 'SIDE_LEFT', vis=True)
        FrameParser.parse_image(frame, 'SIDE_RIGHT', vis=True)
        FrameParser.parse_laser(frame, parse_label=True, vis=True)

        laser_points = np.load(os.path.join(_dir,'laser_points.npy'))
        laser_box3d = np.load(os.path.join(_dir,'laser_box3d.npy'))
        laser_box3d_categorys = np.load(os.path.join(_dir,'laser_box3d_categorys.npy'))
        FrameParser.show_laser_with_box(pcds=laser_points, box3ds=laser_box3d, categorys=laser_box3d_categorys)


text:
'''
Created on 05.04.2021

@author: michael
'''
from xml.dom.minidom import parseString, Element
import pytesseract
import numpy
from PIL import Image
import re

class ElementWrapper:

    def get_bounding_box(self):

        x1 = int(self.element.getAttribute("HPOS"))
        y1 = int(self.element.getAttribute("VPOS"))
        x2 = x1 + self.get_width()
        y2 = y1 + self.get_height()
        return x1, y1, x2, y2

    def get_width(self):

        return int(self.element.getAttribute("WIDTH"))

    def get_height(self):

        return int(self.element.getAttribute("HEIGHT"))

class LineObject(ElementWrapper):

    def __init__(self, lineElement: Element):

        self.element = lineElement

    def get_strings(self):

        elements = []
        for string_element in self.element.getElementsByTagName("String"):
            elements.append(StringObject(string_element))

        return elements

class StringObject(ElementWrapper):
    '''
    Helper class for string representations in the AltoPageLayout class.
    '''

    dotted_characters = {'i': 2, 'ä': 3, 'ö': 3, 'ü': 3, 'Ä': 3, 'Ö': 3, 'Ü': 3, '.': 1, '!': 2, ':': 2, ';': 2}

    def _get_confidence(self):

        return float(self.element.getAttribute("WC"))

    def _get_string_length(self):

        return len(self.get_text())

    def __init__(self, stringElement: Element):

        self.element = stringElement

    def is_string_with_dots(self):

        # Avoid strings with accented characters etc.
        empty_string = re.sub("[a-zA-ZäöüÄÖÜ.!:;]*", '', self.get_text())
        if len(empty_string) != 0:
            return False
        without_dotted = re.sub("[iäöüÄÖÜ.!:;]", '', self.get_text())
        if self.get_text() != without_dotted:
            return True
        return False

    def get_number_of_shapes(self):

        shapes = 0
        for character in self.get_text():
            if character in self.dotted_characters:
                shapes += self.dotted_characters[character]
            else:
                shapes += 1
        return shapes

    def get_text(self):

        return self.element.getAttribute("CONTENT")

    len = property(_get_string_length)
    confidentiality = property(_get_confidence)

class AltoPageLayout:
    '''
    This is a class to get information about an image using OCR.
    '''

    def __init__(self, img, language: str='deu'):

        self.img = img
        self.dom = parseString(pytesseract.image_to_alto_xml(img,lang=language).decode('utf-8'))

    def _get_confidence(self):

        characters = 0
        conf = 0.0
        for string in self.get_all_strings():
            characters += string.len
            conf += string.confidentiality * string.len
        return conf / characters

    def write_to_file(self, filename):

        file = open(filename, "w")
        self.dom.writexml(file, indent="   ")
        file.close()

    def get_all_strings(self) -> [StringObject]:

        strings = []
        for string in self.dom.getElementsByTagName("String"):
            strings.append(StringObject(string))
        return strings

    def get_all_lines(self) -> [LineObject]:

        lines = []
        for line in self.dom.getElementsByTagName("TextLine"):
            lines.append(LineObject(line))
        return lines

    def get_big_text_block_coordinates(self):

        block = self.get_big_text_block()
        x1 = int(block.getAttribute("HPOS"))
        y1 = int(block.getAttribute("VPOS"))
        x2 = x1 + int(block.getAttribute("WIDTH"))
        y2 = y1 + int(block.getAttribute("HEIGHT"))

        return x1, y1, x2, y2

    def get_median_text_height(self):

        heights = []
        for string in self.dom.getElementsByTagName("String"):
            heights.append(int(string.getAttribute("HEIGHT")))
        return numpy.median(heights)

    def get_layout(self):

        return self.dom.gmittelwertetElementsByTagName("Layout")[0]

    def get_first_page(self):

        return self.dom.getElementsByTagName("Page")[0]

    def get_big_text_block(self) -> Element:

        for i in range(0,10):
            for element in self.dom.getElementsByTagName("TextBlock"):
                width = int(element.getAttributeNode("WIDTH").value)
                height = int(element.getAttributeNode("HEIGHT").value)
                if width * height > 10000 * (10-i):
                    return element
        # Safety
        return self.dom.getElementsByTagName("TextBlock")[0]

    def get_text_mask(self):

        mask = numpy.ones((self.img.height, self.img.width), dtype=bool)
        for string in self.get_all_strings():
            x1, y1, x2, y2 = string.get_bounding_box()
            mask[y1:y2, x1:x2] = False
        return Image.fromarray(mask)

    confidence = property(_get_confidence)

max_stars_repo_name: archivsozialebewegungen/ScanConvert

max_issues_repo_name: archivsozialebewegungen/ScanConvert

max_forks_repo_name: archivsozialebewegungen/ScanConvert


text:
# plot loss curves for training and validation

# import data using pandas
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

font = {'family' : 'normal',
        'weight' : 'bold',
        'size'   : 18}

plt.rc('font', **font)

dirname = "/home/emma/gnn_dna/rollouts/llama-c3/"
trainfilename = "train_loss.csv"
df1 = pd.read_csv(dirname+trainfilename)
print(df1.head())
df1 = df1.rename(columns={"loss":"Train Loss"})

valfilename = "val_loss.csv"
df = pd.read_csv(dirname+valfilename)
df = df.rename(columns={"loss":"Validation Loss"})
print(df.head())

val_loss = df["Validation Loss"]
df1 = df1.join(val_loss)
print(df1.head())

# plot curves
df1.plot(x="global_step", y=["Train Loss", "Validation Loss"], linewidth=8)
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.title("Llama, r = 3.0")
plt.savefig("llama_c3_loss.png", bbox_inches='tight')
plt.show()


text:
import matplotlib
# matplotlib.use("Agg")
import matplotlib.pyplot as plt
import numpy as np
import gc
from datetime import datetime
from time import time

def Pre_Initialization(parameters):
    dX, T_max, dT = parameters['dX'], parameters['T_max'], parameters['dT']
    X_min, X_max= parameters['X_min'], parameters['X_max']

    nT = int(T_max/dT)            #number of times
    T = np.arange(0,T_max,dT)     #space of time

    nX = int((X_max-X_min)/dX)    #number of traits
    X = np.arange(X_min,X_max,dX) #space of traits

    f0 = np.exp(-np.power(np.absolute(X-parameters['x_mean0']),2)/parameters['sigma0']*parameters['eps'])# initial density
    rho0=(parameters['b_r'])/(parameters['C'])
    f0=f0/(np.sum(f0)*parameters['dX'])*rho0

    f = np.empty([nT, nX])        #densities for all times
    f[0]=f0

    # Computing constant death and mutation kernels
    Death = parameters['d_r']*np.power(np.absolute(X),parameters['d_e'])
    Mutation_kernel=np.exp(-(X/(parameters['sigma']*parameters['eps']))**2*1/2)*parameters['dX']
    Mutation_kernel=parameters['b_r']*Mutation_kernel/np.sum(Mutation_kernel)
    X0 = np.random.normal(parameters["x_mean0"], parameters['sigma0'], parameters['N0'])

    pre_init_values = dict(
        f = f,
        T = T,
        nX = nX,
        nT = nT,
        X = X, # vector of traits
        X0 = X0, # initial population
        Death = Death,
        Mutation_kernel = Mutation_kernel
        )
    return pre_init_values

def Next_Generation(f,parameters, pre_init_values):
    dX, T_max, dT = parameters['dX'], parameters['T_max'], parameters['dT']
    X_min, X_max= parameters['X_min'], parameters['X_max']
    Death, Mutation_kernel = pre_init_values['Death'], pre_init_values['Mutation_kernel']

    X_min_new=int(-X_min/dX)          # Bounds for the new indexes that must be kept after the convolution
    X_max_new=int((-2*X_min+X_max)/dX)
    I=range(pre_init_values['nX'])    # all the indexes

    rho = np.sum(f)*parameters['dX'] # I don't like the idea of comparing float with an integer, but okay

    death_term = Death + rho*parameters['C']
    birth_part = np.convolve(Mutation_kernel,f)[X_min_new:X_max_new]
    transfer_part = parameters['tau']/rho*parameters['dX']*np.fromiter((np.sum(f[:i])-np.sum(f[i:]) for i in I),float)
    #new_f = np.maximum(0,f + dT/parameters['eps']*(np.multiply(f,(-death_term+ transfer_part))+birth_part))
    new_f = f + dT/parameters['eps']*(np.multiply(f,(-death_term + transfer_part))+birth_part)
    return new_f

def build_and_save(f, parameters, pre_init_values, path):
    par_str = '' # create a string of parameters to pass into pots
    for k,v in parameters.items():
        if k == 'N0' or k=='b_r':
            smth =",\n"
        else:
            smth=", "
        par_str+=k+"="+str(v)+smth
    # Now we have to compute the mean trait and the population size at each time!
    X, nT = pre_init_values['X'], pre_init_values['nT']
    X_min, X_max, dX = parameters['X_min'], parameters['X_max'], parameters['dX']
    sum_f = np.sum(f, axis = 1)*dX
    computed_mean = np.divide(np.sum(X*f, axis = 1), sum_f)
    figure = plt.figure()
    plt.suptitle(par_str, y = 1.1)
    grid = plt.GridSpec(2,5, wspace = 0.9, hspace = 0.5)
    fig1 = plt.subplot(grid[:,:-2])
    fig1.imshow(f.transpose()[::-1],cmap=plt.cm.jet, aspect = 'auto', extent = (0,parameters['T_max'], X_min, X_max),
            vmin = 0, vmax = 4)
    fig1.set_xlabel('time')
    fig1.set_ylabel('trait')
    fig1.set_title('Population dynamics')
    fig2 = plt.subplot(grid[0,3:])
    plt.plot(np.arange(nT)*parameters['dT'], sum_f)
    fig2.set_title("Rho")
    fig3 = plt.subplot(grid[1,3:])
    plt.plot(np.arange(nT)*parameters['dT'], computed_mean)
    fig3.set_title("Mean trait")
    current_time = datetime.now().time()
    figure.savefig(str(path +"plot_"+str(current_time)[0:8]+".pdf"), bbox_inches ='tight')
    np.savetxt(str(path+"mean_trait_"+str(current_time)[0:8]+".txt"), computed_mean, delimiter = ',')
    np.savetxt(str(path+"population_size_"+str(current_time)[0:8]+".txt"), sum_f, delimiter = ',')
	# plt.show()