<commit_before>#! /usr/bin/env python

import SimpleHTTPServer
import BaseHTTPServer
import cgi
import logging
import urllib
import sys, os
import time
from math import log, exp
from thread import start_new_thread
from exchanges import *

try: os.makedirs('logs')
except: pass

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('logs/%d.log' % time.time())
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

formatter = logging.Formatter(fmt = '%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

_port = 2019
_interest = { 'poloniex' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } },
              'ccedk' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } },
              'bitcoincoid' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } }
            }
_nuconfig = '%s/.nu/nu.conf'%os.getenv("HOME") # path to nu.conf
_wrappers = { 'poloniex' : Poloniex(), 'ccedk' : CCEDK(), 'bitcoincoid' : BitcoinCoId() }
_tolerance = 0.08
_sampling = 12
_minpayout = 0.1
_validations = 0
_liquidity = []

keys = {}
price = {'btc' : 0.003666}

_lock = False
def acquire_lock():
  global _lock
  while _lock: pass
  _lock = True
def release_lock():
  global _lock
  _lock = False

def response(errcode = 0, message = 'success'):
  return { 'code' : errcode, 'message' : message }

def register(params):
  ret = response()
  if set(params.keys()) == set(['address', 'key', 'name']):
    user = params['key'][0]
    name = params['name'][0]
    if name in _wrappers:
      if not user in keys:
        acquire_lock()
        keys[user] = { 'name' : params['name'][0], 'address' : params['address'][0], 'balance' : 0.0, 'accepts' : 0, 'units' : {} }
        for unit in _interest[name]:
          keys[user]['units'][unit] = { 'request' : None, 'bid' : [], 'ask' : [], 'rejects' : 0, 'missing' : 0, 'last_error' : "" }
        release_lock()
        logger.info("new user %s: %s" % (user, keys[user]['address']))
      elif keys[user]['address'] != params['address'][0]:
        ret = response(9, "user already exists with different address: %s" % user)
    else:
      ret = response(8, "unknown exchange requested: %s" % name)
  else:
    ret = response(7, "invalid registration data received: %s" % str(params))
  return ret

def liquidity(params):
  ret = response()
  if set(params.keys() + ['user', 'sign', 'unit']) == set(params.keys()):
    user = params.pop('user')[0]
    sign = params.pop('sign')[0]
    unit = params.pop('unit')[0]
    if user in keys:
      if unit in _interest[keys[user]['name']]:
        acquire_lock()
        keys[user]['units'][unit]['request'] = ({ p : v[0] for p,v in params.items() }, sign)
        release_lock()
      else:
        ret = response(12, "%s market not supported on %s" % (unit, keys[user]['name']))
    else:
        ret = response(11, "user not found: %s" % user)
  else:
    ret = response(10, "invalid liquidity data received: %s" % str(params))
  return ret

def poolstats():
  return { 'liquidity' : ([ (0,0) ] + _liquidity)[-1], 'sampling' : _sampling, 'validations' : _validations, 'users' : len(keys.keys()) }

def userstats(user):
  res = { 'name' : keys[user]['name'], 'address' : keys[user]['address'], 'balance' : keys[user]['balance'] }
  res['units'] = {}
  for unit in keys[user]['units']:
    bid = [ x for x in keys[user]['units'][unit]['bid'] if x ]
    ask = [ x for x in keys[user]['units'][unit]['ask'] if x ]
    bid, ask = [[]] + bid, [[]] + ask
    res['units'][unit] = { 'bid' : bid[-1], 'ask' : ask[-1], 
                           'rejects' : keys[user]['units'][unit]['rejects'],
                           'missing' : keys[user]['units'][unit]['missing'],
                           'last_error' :  keys[user]['units'][unit]['last_error'] }
  return res

class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  def do_POST(self):
    if self.path in ['register', 'liquidity']:
      ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
      if ctype == 'application/x-www-form-urlencoded':
        length = int(self.headers.getheader('content-length'))
        params = cgi.parse_qs(self.rfile.read(length), keep_blank_values = 1)
        if self.path == 'liquidity':
          ret = liquidity(params)
        elif self.path == 'register':
          ret = register(params)
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(ret))
      self.end_headers()

  def do_GET(self):
    method = self.path[1:]
    if method == 'status':
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(poolstats()))
      self.end_headers()
    elif method in keys:
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(userstats(method)))
      self.end_headers()
    else:
      self.send_response(404)

  def log_message(self, format, *args): pass

def update_price():
  try:
    ret = json.loads(urllib2.urlopen(urllib2.Request('https://api.bitfinex.com/v1//pubticker/btcusd')).read())
    price['btc'] = 1.0 / float(ret['mid'])
  except:
    logging.error("unable to update price for BTC")

def validate():
  liquidity = { 'bid' : 0.0, 'ask' : 0.0 }
  for user in keys:
    for unit in keys[user]['units']:
      if keys[user]['units'][unit]['request']:
        try:
          orders = _wrappers[keys[user]['name']].validate_request(user, unit, *keys[user]['units'][unit]['request'])
          keys[user]['units'][unit]['request'] = None
        except Exception as e:
          orders = { 'error' : 'exception caught: %s' % str(e)}
        if not 'error' in orders:
          keys[user]['units'][unit]['last_error'] = ""
          valid = { 'bid': [], 'ask' : [] }
          for order in orders:
            if 1.0 - min(order['price'], price[unit]) / max(order['price'], price[unit]) < _tolerance:
              valid[order['type']].append((order['id'], order['amount']))
            else:
              logger.warning("order of deviates too much from current price for user %s at exchange %s on market %s" % (user, keys[user]['name'], unit))
          for side in [ 'bid', 'ask' ]:
            keys[user]['units'][unit][side].append(valid[side])<commit_msg>higher price tolerance<commit_after>#! /usr/bin/env python

import SimpleHTTPServer
import BaseHTTPServer
import cgi
import logging
import urllib
import sys, os
import time
from math import log, exp
from thread import start_new_thread
from exchanges import *

try: os.makedirs('logs')
except: pass

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('logs/%d.log' % time.time())
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)

formatter = logging.Formatter(fmt = '%(asctime)s %(levelname)s: %(message)s', datefmt="%Y/%m/%d-%H:%M:%S")
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)

_port = 2019
_interest = { 'poloniex' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } },
              'ccedk' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } },
              'bitcoincoid' : { 'btc' : { 'rate' : 0.002, 'target' : 100.0 } }
            }
_nuconfig = '%s/.nu/nu.conf'%os.getenv("HOME") # path to nu.conf
_wrappers = { 'poloniex' : Poloniex(), 'ccedk' : CCEDK(), 'bitcoincoid' : BitcoinCoId() }
_tolerance = 0.08
_sampling = 12
_minpayout = 0.1
_validations = 0
_liquidity = []

keys = {}
price = {'btc' : 0.003666}

_lock = False
def acquire_lock():
  global _lock
  while _lock: pass
  _lock = True
def release_lock():
  global _lock
  _lock = False

def response(errcode = 0, message = 'success'):
  return { 'code' : errcode, 'message' : message }

def register(params):
  ret = response()
  if set(params.keys()) == set(['address', 'key', 'name']):
    user = params['key'][0]
    name = params['name'][0]
    if name in _wrappers:
      if not user in keys:
        acquire_lock()
        keys[user] = { 'name' : params['name'][0], 'address' : params['address'][0], 'balance' : 0.0, 'accepts' : 0, 'units' : {} }
        for unit in _interest[name]:
          keys[user]['units'][unit] = { 'request' : None, 'bid' : [], 'ask' : [], 'rejects' : 0, 'missing' : 0, 'last_error' : "" }
        release_lock()
        logger.info("new user %s: %s" % (user, keys[user]['address']))
      elif keys[user]['address'] != params['address'][0]:
        ret = response(9, "user already exists with different address: %s" % user)
    else:
      ret = response(8, "unknown exchange requested: %s" % name)
  else:
    ret = response(7, "invalid registration data received: %s" % str(params))
  return ret

def liquidity(params):
  ret = response()
  if set(params.keys() + ['user', 'sign', 'unit']) == set(params.keys()):
    user = params.pop('user')[0]
    sign = params.pop('sign')[0]
    unit = params.pop('unit')[0]
    if user in keys:
      if unit in _interest[keys[user]['name']]:
        acquire_lock()
        keys[user]['units'][unit]['request'] = ({ p : v[0] for p,v in params.items() }, sign)
        release_lock()
      else:
        ret = response(12, "%s market not supported on %s" % (unit, keys[user]['name']))
    else:
        ret = response(11, "user not found: %s" % user)
  else:
    ret = response(10, "invalid liquidity data received: %s" % str(params))
  return ret

def poolstats():
  return { 'liquidity' : ([ (0,0) ] + _liquidity)[-1], 'sampling' : _sampling, 'validations' : _validations, 'users' : len(keys.keys()) }

def userstats(user):
  res = { 'name' : keys[user]['name'], 'address' : keys[user]['address'], 'balance' : keys[user]['balance'] }
  res['units'] = {}
  for unit in keys[user]['units']:
    bid = [ x for x in keys[user]['units'][unit]['bid'] if x ]
    ask = [ x for x in keys[user]['units'][unit]['ask'] if x ]
    bid, ask = [[]] + bid, [[]] + ask
    res['units'][unit] = { 'bid' : bid[-1], 'ask' : ask[-1], 
                           'rejects' : keys[user]['units'][unit]['rejects'],
                           'missing' : keys[user]['units'][unit]['missing'],
                           'last_error' :  keys[user]['units'][unit]['last_error'] }
  return res

class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  def do_POST(self):
    if self.path in ['register', 'liquidity']:
      ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
      if ctype == 'application/x-www-form-urlencoded':
        length = int(self.headers.getheader('content-length'))
        params = cgi.parse_qs(self.rfile.read(length), keep_blank_values = 1)
        if self.path == 'liquidity':
          ret = liquidity(params)
        elif self.path == 'register':
          ret = register(params)
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(ret))
      self.end_headers()

  def do_GET(self):
    method = self.path[1:]
    if method == 'status':
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(poolstats()))
      self.end_headers()
    elif method in keys:
      self.send_response(200)
      self.send_header('Content-Type', 'application/json')
      self.wfile.write("\n")
      self.wfile.write(json.dumps(userstats(method)))
      self.end_headers()
    else:
      self.send_response(404)

  def log_message(self, format, *args): pass

def update_price():
  try:
    ret = json.loads(urllib2.urlopen(urllib2.Request('https://api.bitfinex.com/v1//pubticker/btcusd')).read())
    price['btc'] = 1.0 / float(ret['mid'])
  except:
    logging.error("unable to update price for BTC")

def validate():
  liquidity = { 'bid' : 0.0, 'ask' : 0.0 }
  for user in keys:
    for unit in keys[user]['units']:
      if keys[user]['units'][unit]['request']:
        try:
          orders = _wrappers[keys[user]['name']].validate_request(user, unit, *keys[user]['units'][unit]['request'])
          keys[user]['units'][unit]['request'] = None
        except Exception as e:
          orders = { 'error' : 'exception caught: %s' % str(e)}
        if not 'error' in orders:
          keys[user]['units'][unit]['last_error'] = ""
          valid = { 'bid': [], 'ask' : [] }
          for order in orders:
            deviation = 1.0 - min(order['price'], price[unit]) / max(order['price'], price[unit])
            if deviation < _tolerance:
              valid[order['type']].append((order['id'], order['amount']))
            else:
              logger.warning("order of deviates too much from current price for user %s at exchange %s on market %s (%.02f < %.2f)" % (user, keys[user]['name'], unit, _tolerance, deviation))
          for side in [ 'bid', 'ask' ]:
            keys[user]['units'][unit][side].append(valid[side])
{'fraction_non_alphanumeric': 0.18563922942206654, 'fraction_numerical': 0.01500038072032285, 'mean_word_length': 3.1894736842105265, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182228', 'n_tokens_mistral': 4533, 'n_tokens_neox': 4420, 'n_words': 1009}