<commit_before>#!/usr/bin/env python2
import json

import psycopg2
from websocket import create_connection

import api
import config


ws = create_connection(config.WEBSOCKET_URL)

con = psycopg2.connect(**config.POSTGRES)
cur = con.cursor()

query = "TRUNCATE assets"
cur.execute(query)

query = "ALTER SEQUENCE assets_id_seq RESTART WITH 1;"
cur.execute(query)

# alter sequence of the ops once a day here
query = "DELETE FROM ops WHERE oid NOT IN (SELECT oid FROM ops ORDER BY oid DESC LIMIT 10);"
cur.execute(query)

for x in range(0, 10):
    query = "UPDATE ops set oid=%s WHERE oid IN (SELECT oid FROM ops ORDER BY oid LIMIT 1 OFFSET %s);"
    cur.execute(query, (x+1, x))

query = "ALTER SEQUENCE ops_oid_seq RESTART WITH 11;"
cur.execute(query)

con.commit()

all_assets = []

ws.send('{"id":1, "method":"call", "params":[0,"list_assets",["AAAAA", 100]]}')
result = ws.recv()
j = json.loads(result)

all_assets.append(j);

len_result = len(j["result"])

print len_result
#print all_assets

while len_result == 100:
    ws.send('{"id":1, "method":"call", "params":[0,"list_assets",["'+j["result"][99]["symbol"]+'", 100]]}')
    result = ws.recv()
    j = json.loads(result)
    len_result = len(j["result"])
    all_assets.append(j);

for x in range(0, len(all_assets)):
    size = len(all_assets[x]["result"])
    print size

    for i in range(0, size):
        symbol = all_assets[x]["result"][i]["symbol"]
        asset_id = all_assets[x]["result"][i]["id"]

        precision = 5
        try:
            data3 = api._get_asset(asset_id)
            current_supply = data3[0]["current_supply"]
            precision = data3[0]["precision"]
            # print current_supply
        except:
            price = 0
            continue

        try:
            holders = api._get_asset_holders_count(asset_id)
            # print holders
        except:
            holders = 0
            continue

        if symbol == config.CORE_ASSET_SYMBOL:
            type_ = "Core Token"
        elif all_assets[x]["result"][i]["issuer"] == "1.2.0":
            type_ = "SmartCoin"
        else:
            type_ = "User Issued"
        #print all_assets[x]["result"][i]

        try:
            data = api._get_volume(config.CORE_ASSET_SYMBOL, symbol)
        except:
            continue

        #print symbol
        #print data["quote_volume"]

        try:
            data2 = api._get_ticker(config.CORE_ASSET_SYMBOL, symbol)
            price = data2["latest"]
            #print price

            if str(price) == 'inf':
               continue
            #    exit

            #print price
        except:
            price = 0
            continue

        mcap = int(current_supply) * float(price)

        query = "INSERT INTO assets (aname, aid, price, volume, mcap, type, current_supply, holders, wallettype, precision) VALUES({})".format(', '.join(('%s',)*9))
        print(symbol)
        cur.execute(query, (symbol, asset_id, price, data['base_volume'], str(mcap), type_, str(current_supply), str(holders), str(precision)))
        con.commit()


# with updated volume, add stats
query = "select sum(volume) from assets WHERE aname!='BTS'"
cur.execute(query)
results = cur.fetchone()
volume = results[0]

query = "select sum(mcap) from assets"<commit_msg>Add back accidentally omitted column<commit_after>#!/usr/bin/env python2
import json

import psycopg2
from websocket import create_connection

import api
import config


ws = create_connection(config.WEBSOCKET_URL)

con = psycopg2.connect(**config.POSTGRES)
cur = con.cursor()

query = "TRUNCATE assets"
cur.execute(query)

query = "ALTER SEQUENCE assets_id_seq RESTART WITH 1;"
cur.execute(query)

# alter sequence of the ops once a day here
query = "DELETE FROM ops WHERE oid NOT IN (SELECT oid FROM ops ORDER BY oid DESC LIMIT 10);"
cur.execute(query)

for x in range(0, 10):
    query = "UPDATE ops set oid=%s WHERE oid IN (SELECT oid FROM ops ORDER BY oid LIMIT 1 OFFSET %s);"
    cur.execute(query, (x+1, x))

query = "ALTER SEQUENCE ops_oid_seq RESTART WITH 11;"
cur.execute(query)

con.commit()

all_assets = []

ws.send('{"id":1, "method":"call", "params":[0,"list_assets",["AAAAA", 100]]}')
result = ws.recv()
j = json.loads(result)

all_assets.append(j);

len_result = len(j["result"])

print len_result
#print all_assets

while len_result == 100:
    ws.send('{"id":1, "method":"call", "params":[0,"list_assets",["'+j["result"][99]["symbol"]+'", 100]]}')
    result = ws.recv()
    j = json.loads(result)
    len_result = len(j["result"])
    all_assets.append(j);

for x in range(0, len(all_assets)):
    size = len(all_assets[x]["result"])
    print size

    for i in range(0, size):
        symbol = all_assets[x]["result"][i]["symbol"]
        asset_id = all_assets[x]["result"][i]["id"]

        precision = 5
        try:
            data3 = api._get_asset(asset_id)
            current_supply = data3[0]["current_supply"]
            precision = data3[0]["precision"]
            # print current_supply
        except:
            price = 0
            continue

        try:
            holders = api._get_asset_holders_count(asset_id)
            # print holders
        except:
            holders = 0
            continue

        if symbol == config.CORE_ASSET_SYMBOL:
            type_ = "Core Token"
        elif all_assets[x]["result"][i]["issuer"] == "1.2.0":
            type_ = "SmartCoin"
        else:
            type_ = "User Issued"
        #print all_assets[x]["result"][i]

        try:
            data = api._get_volume(config.CORE_ASSET_SYMBOL, symbol)
        except:
            continue

        #print symbol
        #print data["quote_volume"]

        try:
            data2 = api._get_ticker(config.CORE_ASSET_SYMBOL, symbol)
            price = data2["latest"]
            #print price

            if str(price) == 'inf':
               continue
            #    exit

            #print price
        except:
            price = 0
            continue

        mcap = int(current_supply) * float(price)

        query = "INSERT INTO assets (aname, aid, price, volume, mcap, type, current_supply, holders, wallettype, precision) VALUES({})".format(', '.join(('%s',)*10))
        print(symbol)
        cur.execute(query, (symbol, asset_id, price, data['base_volume'], str(mcap), type_, str(current_supply), str(holders), '', str(precision)))
        con.commit()


# with updated volume, add stats
query = "select sum(volume) from assets WHERE aname!='BTS'"
cur.execute(query)
results = cur.fetchone()
volume = results[0]

query = "select sum(mcap) from assets"
{'fraction_non_alphanumeric': 0.1372876213592233, 'fraction_numerical': 0.014108009708737865, 'mean_word_length': 2.6791294642857144, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2818743', 'n_tokens_mistral': 2226, 'n_tokens_neox': 2117, 'n_words': 537}