<commit_before>#!/usr/bin/env ruby

require "httpclient"
require "json"

require_relative "config"
require_relative "myfunc"
require_relative "mylogger"

####################################
# blockchain
#

# TODO write a class/function to communicate with rpc server

def chain_post (data:{}, timeout:5)
  $LOG.debug (method(__method__).name) { {"parameters"=>method(__method__).parameters.map { |arg| "#{arg[1]} = #{eval arg[1].to_s}" }.join(', ') } }
  if data.nil? or data.empty?
    return
  end

  client = HTTPClient.new 
  client.connect_timeout=timeout
  client.receive_timeout=timeout
  
  myconfig = my_chain_config
  uri  = myconfig["uri"]
  user = myconfig["user"]
  pass = myconfig["pass"]
 
  client.set_auth uri, user, pass
 
  begin
    response = client.post uri, data.to_json, nil
    #$LOG.debug (method(__method__).name) { response }
    response_content = response.content
    $LOG.debug (method(__method__).name) { {"response_content" => response_content} }
    response_json = JSON.parse response_content
    if not response_json["error"].nil?
      #$LOG.debug (method(__method__).name) { response_json["error"] }
      #response_content = response.body
    end
    return response_json
  rescue Exception => e
    print "chain_post error: "
    puts e
  end
end

# params is an array
def chain_command (command:nil, params:nil, timeout:5)
  $LOG.debug (method(__method__).name) { {"parameters"=>method(__method__).parameters.map { |arg| "#{arg[1]} = #{eval arg[1].to_s}" }.join(', ') } }
  if command.nil? or params.nil? 
    return
  end
  #request_data= '{"jsonrpc": "2.0", "method": "blockchain_get_asset", "params":["'+quote.upcase+'"], "id":1}'
  data = {
    "jsonrpc" => "2.0",
    "method"  => command,
    "params"  => params,
    "id"      => 0
  }
  return chain_post data:data, timeout:timeout
end

def fetch_chain (quote="bts", base="cny", max_orders=5)
  chain_fetch quote:quote, base:base, max_orders:max_orders
end

def chain_fetch (quote:"bts", base:"cny", max_orders:5)

  response_json = chain_command command:"blockchain_get_asset", params:[quote.upcase]
  quote_precision = response_json["result"]["precision"]

  response_json = chain_command command:"blockchain_get_asset", params:[base.upcase]
  base_precision = response_json["result"]["precision"]

  response_json = chain_command command:"blockchain_median_feed_price", params:[base.upcase]
  feed_price = response_json["result"]

  response_json = chain_command command:"blockchain_market_order_book", params:[base.upcase, quote.upcase]
  ob = response_json["result"]
  #puts JSON.pretty_generate ob["result"]

  shorts = []
  if (not quote.nil?) and quote.downcase == "bts"
    response_json = chain_command command:"blockchain_market_list_shorts", params:[base.upcase]
    shorts = response_json["result"]
  end
  
  #ask orders and cover orders are in same array. filter invalid short orders here. TODO maybe wrong logic here
  asks = ob[1].delete_if {|e| e["type"] == "cover_order" and
                              Time.parse(e["expiration"]+' +0000') > Time.now and
                              e["market_index"]["order_price"]["ratio"].to_f < feed_price
                              #feed_price > e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision
                       }.sort_by {|e| (e["type"] == "ask_order" and e["market_index"]["order_price"]["ratio"].to_f) or
                                      (e["type"] == "cover_order" and e["market_index"]["order_price"]["ratio"] >= feed_price and 
                                                                      feed_price * 0.9) or
                                      (e["type"] == "cover_order" and feed_price) 
                       }.first(max_orders)
#puts ob[0].concat(shorts)
  bids = ob[0].concat(shorts).sort_by {|e| 
begin
                  (e["type"] == "bid_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                  (e["type"] == "short_order" and ( e["state"]["limit_price"].nil? ? feed_price :
                           [ e["state"]["limit_price"]["ratio"].to_f*quote_precision/base_precision, feed_price ].min ) )
rescue
puts e
end
                       }.reverse.first(max_orders)
  
  #asks_new=Hash[*asks.map["price","volume"]]
  asks_new=[]
  bids_new=[]
  bids.each do |e|
    item = {
      "price"=> ( (e["type"] == "bid_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                  (e["type"] == "short_order" and ( e["state"]["limit_price"].nil? ? feed_price :
                           [ e["state"]["limit_price"]["ratio"].to_f*quote_precision/base_precision, feed_price ].min ) )
                ),
      "volume"=>e["state"]["balance"].to_f/base_precision
    }
    if e["type"] == "bid_order" 
      item["volume"] /= item["price"]
    elsif e["type"] == "short_order"
      item["volume"] /= 2
    end
    bids_new.push item
  end
  asks.each do |e|
    item = {
      "price"=> ((e["type"] == "ask_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                 (e["type"] == "cover_order" and e["market_index"]["order_price"]["ratio"] >= feed_price and
                          ( (bids.empty? or feed_price * 0.9 < bids[0]["price"]) ? 
                                 feed_price * 0.9 : bids[0]["price"] + 0.000001 ) ) or # hack price to not match
                 (e["type"] == "cover_order" and feed_price) 
                ),
      "volume"=>e["state"]["balance"].to_f/quote_precision
    }
    #item["volume"] /= item["price"]
    asks_new.push item
  end

  # if there are orders can be matched, wait until they matched.
  if not asks_new[0].nil? and not bids_new[0].nil? and asks_new[0]["price"] <= bids_new[0]["price"]
    asks_new=[]
    bids_new=[]
  end
  
  #return<commit_msg>fix short volume bug in chain_fetch; tweak default timeout in chain_command and chain_post<commit_after>#!/usr/bin/env ruby

require "httpclient"
require "json"

require_relative "config"
require_relative "myfunc"
require_relative "mylogger"

####################################
# blockchain
#

# TODO write a class/function to communicate with rpc server

def chain_post (data:{}, timeout:55)
  $LOG.debug (method(__method__).name) { {"parameters"=>method(__method__).parameters.map { |arg| "#{arg[1]} = #{eval arg[1].to_s}" }.join(', ') } }
  if data.nil? or data.empty?
    return
  end

  client = HTTPClient.new 
  client.connect_timeout=timeout
  client.receive_timeout=timeout
  
  myconfig = my_chain_config
  uri  = myconfig["uri"]
  user = myconfig["user"]
  pass = myconfig["<PASSWORD>"]
 
  client.set_auth uri, user, pass
 
  begin
    response = client.post uri, data.to_json, nil
    #$LOG.debug (method(__method__).name) { response }
    response_content = response.content
    $LOG.debug (method(__method__).name) { {"response_content" => response_content} }
    response_json = JSON.parse response_content
    if not response_json["error"].nil?
      #$LOG.debug (method(__method__).name) { response_json["error"] }
      #response_content = response.body
    end
    return response_json
  rescue Exception => e
    print "chain_post error: "
    puts e
  end
end

# params is an array
def chain_command (command:nil, params:nil, timeout:55)
  $LOG.debug (method(__method__).name) { {"parameters"=>method(__method__).parameters.map { |arg| "#{arg[1]} = #{eval arg[1].to_s}" }.join(', ') } }
  if command.nil? or params.nil? 
    return
  end
  #request_data= '{"jsonrpc": "2.0", "method": "blockchain_get_asset", "params":["'+quote.upcase+'"], "id":1}'
  data = {
    "jsonrpc" => "2.0",
    "method"  => command,
    "params"  => params,
    "id"      => 0
  }
  return chain_post data:data, timeout:timeout
end

def fetch_chain (quote="bts", base="cny", max_orders=5)
  chain_fetch quote:quote, base:base, max_orders:max_orders
end

def chain_fetch (quote:"bts", base:"cny", max_orders:5)

  response_json = chain_command command:"blockchain_get_asset", params:[quote.upcase]
  quote_precision = response_json["result"]["precision"]

  response_json = chain_command command:"blockchain_get_asset", params:[base.upcase]
  base_precision = response_json["result"]["precision"]

  response_json = chain_command command:"blockchain_median_feed_price", params:[base.upcase]
  feed_price = response_json["result"]

  response_json = chain_command command:"blockchain_market_order_book", params:[base.upcase, quote.upcase]
  ob = response_json["result"]
  #puts JSON.pretty_generate ob["result"]

  shorts = []
  if (not quote.nil?) and quote.downcase == "bts"
    response_json = chain_command command:"blockchain_market_list_shorts", params:[base.upcase]
    shorts = response_json["result"]
  end
  
  #ask orders and cover orders are in same array. filter invalid short orders here. TODO maybe wrong logic here
  asks = ob[1].delete_if {|e| e["type"] == "cover_order" and
                              Time.parse(e["expiration"]+' +0000') > Time.now and
                              e["market_index"]["order_price"]["ratio"].to_f < feed_price
                              #feed_price > e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision
                       }.sort_by {|e| (e["type"] == "ask_order" and e["market_index"]["order_price"]["ratio"].to_f) or
                                      (e["type"] == "cover_order" and e["market_index"]["order_price"]["ratio"] >= feed_price and 
                                                                      feed_price * 0.9) or
                                      (e["type"] == "cover_order" and feed_price) 
                       }.first(max_orders)
#puts ob[0].concat(shorts)
  bids = ob[0].concat(shorts).sort_by {|e| 
begin
                  (e["type"] == "bid_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                  (e["type"] == "short_order" and ( e["state"]["limit_price"].nil? ? feed_price :
                           [ e["state"]["limit_price"]["ratio"].to_f*quote_precision/base_precision, feed_price ].min ) )
rescue
puts e
end
                       }.reverse.first(max_orders)
  
  #asks_new=Hash[*asks.map["price","volume"]]
  asks_new=[]
  bids_new=[]
  bids.each do |e|
    item = {
      "price"=> ( (e["type"] == "bid_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                  (e["type"] == "short_order" and ( e["state"]["limit_price"].nil? ? feed_price :
                           [ e["state"]["limit_price"]["ratio"].to_f*quote_precision/base_precision, feed_price ].min ) )
                ),
      "volume"=>( e["type"] == "bid_order" ? e["state"]["balance"].to_f/base_precision : e["state"]["balance"].to_f/quote_precision)
    }
    if e["type"] == "bid_order" 
      item["volume"] /= item["price"]
    elsif e["type"] == "short_order"
      item["volume"] /= 2
    end
    bids_new.push item
  end
  asks.each do |e|
    item = {
      "price"=> ((e["type"] == "ask_order" and e["market_index"]["order_price"]["ratio"].to_f*quote_precision/base_precision) or
                 (e["type"] == "cover_order" and e["market_index"]["order_price"]["ratio"] >= feed_price and
                          ( (bids.empty? or feed_price * 0.9 < bids[0]["price"]) ? 
                                 feed_price * 0.9 : bids[0]["price"] + 0.000001 ) ) or # hack price to not match
                 (e["type"] == "cover_order" and feed_price) 
                ),
      "volume"=>e["state"]["balance"].to_f/quote_precision
    }
    #item["volume"] /= item["price"]
    asks_new.push item
  end

  # if there are orders can be matched, wait until they matched.
  if not asks_new[0].nil? and not bids_new[0].nil? and asks_new[0]["price"] <= bids_new[0]["price"]
    asks_new=[]
    bids_new=[]
  end
  
  #return
{'fraction_non_alphanumeric': 0.16442614686607293, 'fraction_numerical': 0.007057637371870274, 'mean_word_length': 3.1215373961218837, 'pattern_counts': {'":': 8, '<': 10, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '27920', 'n_tokens_mistral': 3920, 'n_tokens_neox': 3871, 'n_words': 765}