<commit_before>#!/usr/bin/env python

"""
Automated LendingClub investor using P2P-Picks for underwriting
"""

import lendingclub as lc
import p2ppicks as p2p

import datetime as dt
import dateutil.parser as dateparser
import json
import logging
import pprint
import time
import urllib2
import urllib2
from optparse import OptionParser

__all__ = ['AutoInvestor']

class AutoInvestor:
  """
  Auto investor for LendingClub using P2P-Picks to underwrite loans
  https://www.p2p-picks.com/

  self.lc: An instance of the LendingClub API
  self.p2p: An instance of the P2P-Picks API
  """

  def __init__(self, secrets='secrets.json', logfile=None):
    """
    secrets: path to a json file containing sensitive information
    logfile: path a logfile to append logging information
    {
      "lc_api_key": "<KEY>", // Lending Club api key
      "lc_investor_id": 93234531,         // Lending Club investor id
      "lc_portfolio": "MyPortfolio"     // Portfolio name to assign invested loans
      "p2p_key": "87C2FE2B4843AD",    // P2P-Picks API key
      "p2p_secret": "<KEY>",    // P2P-Picks API secret 
      "p2p_sid": "384FBC34D3AB"      // P2P-Picks session ID
    }
    """
    #
    # Set up logging
    #
     
    self.logger = logging.getLogger('AutoInvestor')
    self.logger.setLevel(logging.DEBUG)

    # create formatter
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')

    # Log "info" to file
    if logfile is not None:
      fh = logging.FileHandler(logfile)
      fh.setFormatter(formatter)
      fh.setLevel(logging.INFO)
      self.logger.addHandler(fh)

    # Log to console
    ch = logging.StreamHandler()
    ch.setFormatter(formatter)
    ch.setLevel(logging.DEBUG)
    self.logger.addHandler(ch)

    #
    # Initalize configurations
    # 

    with open(secrets) as f:
      secrets = json.load(f)
      p2p_key = str(secrets['p2p_key'])
      p2p_secret = str(secrets['p2p_secret'])
      p2p_sid = str(secrets['p2p_sid'])
      lc_investor_id = str(secrets['lc_investor_id'])
      lc_api_key = str(secrets['lc_api_key'])

      # Pass lending club secrets to lc.API
      self.lc = lc.API(lc_investor_id, lc_api_key)

      # Pass P2P-Picks secrets to p2p.API
      self.p2p = p2p.API(p2p_key, p2p_secret, p2p_sid)

      # Get portfolio ID from name if it exists
      self.lc_portfolio_id = next((int(p['portfolioId'])
        for p in self.lc.portfolios_owned() 
        if p['portfolioName'] == secrets['lc_portfolio']), None)

      if self.lc_portfolio_id is None:
        self.logger.warning("Portfolio '{}' not found. Not using a portfolio"\
                            .format(secrets['lc_portfolio']))
    #
    # Investment configurations
    #
    self.MIN_RATE = 16.0 # Minimum interest rate
    self.MAX_RATE = 25.0 # Maximum interest rate
    self.PICK_LEVEL = frozenset(['5%'])
    self.AMOUNT_PER_LOAN = 25.0

  def poll(self, fx):
    """
    Generator that polls a function

    fx: a function to poll
    """
    start = dt.datetime.now()
    counter = 0
    while dt.datetime.now() - start < dt.timedelta(minutes=1):
      counter += 1
      try:
        if not counter % 10:
          self.logger.debug('Poll counter: {}'.format(counter))
        yield fx()
      except urllib2.HTTPError as err:
        self.logger.error("HTTPError: {}".format(err.code))
        time.sleep(1)
      except urllib2.URLError as err:
        self.logger.error("URLError: {}".format(err.reason))
        time.sleep(1)
      except (KeyboardInterrupt,SystemExit) as err:
        # We're trying to quit
        raise err
      except Exception as err:
        self.logger.critical("Other exception: {} {}".format(type(err), err))

    raise StopIteration("Polling timeout")

  def wait_for_new_picks(self, start=None):
    """
    Start polling for an update in listed loans
    Must be called before picks update

    start: Time before picks update
    """
    if start is None:
      _, start = self.p2p.picks()

    self.logger.debug("Starting polling picks")

    for picks, timestamp in self.poll(self.p2p.picks):
      if timestamp > start:
        self.logger.info("New picks")
        return picks

    self.logger.error("P2P-Picks polling timeout")
    raise Exception("P2P-Picks polling timeout")

  def wait_for_new_loans(self, start=None):
    """
    Start polling for an update in listed P2P-Picks
    Must be called before picks update

    start: Time before loan update
    """
    if start is None:
      start = dateparser.parse(self.lc.listed_loans()[0]['listD'])

    self.logger.debug("Starting polling loans")

    for loans in self.poll(self.lc.listed_loans):
      timestamp = dateparser.parse(loans[0]['listD'])

      if timestamp > start:
        self.logger.info("New loans")
        return loans

    self.logger.error("Listed loans polling timeout")
    raise Exception("Listed loans polling timeout")

  def invest(self, load_ids):
    """
    Attept to invest in loans by id. Reports
    successful investments to P2P-Picks.

    load_ids: a list of loan ids

    Returns: JSON reponse from lending club
    """
    try:
      # Submit order and report activity to P2P-Picks
      res = self.lc.submit_order(load_ids,
                  self.AMOUNT_PER_LOAN, self.lc_portfolio_id)
      self.p2p.report(res)
      return res
    except urllib2.HTTPError as e:
      self.logger.error(e)<commit_msg>Provide more debugging information on failed invest<commit_after>#!/usr/bin/env python

"""
Automated LendingClub investor using P2P-Picks for underwriting
"""

import lendingclub as lc
import p2ppicks as p2p

import datetime as dt
import dateutil.parser as dateparser
import json
import logging
import pprint
import time
import urllib2
import urllib2
from optparse import OptionParser

__all__ = ['AutoInvestor']

class AutoInvestor:
  """
  Auto investor for LendingClub using P2P-Picks to underwrite loans
  https://www.p2p-picks.com/

  self.lc: An instance of the LendingClub API
  self.p2p: An instance of the P2P-Picks API
  """

  def __init__(self, secrets='secrets.json', logfile=None):
    """
    secrets: path to a json file containing sensitive information
    logfile: path a logfile to append logging information
    {
      "lc_api_key": "<KEY>", // Lending Club api key
      "lc_investor_id": 93234531,         // Lending Club investor id
      "lc_portfolio": "MyPortfolio"     // Portfolio name to assign invested loans
      "p2p_key": "87C2FE2B4843AD",    // P2P-Picks API key
      "p2p_secret": "<KEY>",    // P2P-Picks API secret 
      "p2p_sid": "384FBC34D3AB"      // P2P-Picks session ID
    }
    """
    #
    # Set up logging
    #
     
    self.logger = logging.getLogger('AutoInvestor')
    self.logger.setLevel(logging.DEBUG)

    # create formatter
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')

    # Log "info" to file
    if logfile is not None:
      fh = logging.FileHandler(logfile)
      fh.setFormatter(formatter)
      fh.setLevel(logging.INFO)
      self.logger.addHandler(fh)

    # Log to console
    ch = logging.StreamHandler()
    ch.setFormatter(formatter)
    ch.setLevel(logging.DEBUG)
    self.logger.addHandler(ch)

    #
    # Initalize configurations
    # 

    with open(secrets) as f:
      secrets = json.load(f)
      p2p_key = str(secrets['p2p_key'])
      p2p_secret = str(secrets['p2p_secret'])
      p2p_sid = str(secrets['p2p_sid'])
      lc_investor_id = str(secrets['lc_investor_id'])
      lc_api_key = str(secrets['lc_api_key'])

      # Pass lending club secrets to lc.API
      self.lc = lc.API(lc_investor_id, lc_api_key)

      # Pass P2P-Picks secrets to p2p.API
      self.p2p = p2p.API(p2p_key, p2p_secret, p2p_sid)

      # Get portfolio ID from name if it exists
      self.lc_portfolio_id = next((int(p['portfolioId'])
        for p in self.lc.portfolios_owned() 
        if p['portfolioName'] == secrets['lc_portfolio']), None)

      if self.lc_portfolio_id is None:
        self.logger.warning("Portfolio '{}' not found. Not using a portfolio"\
                            .format(secrets['lc_portfolio']))
    #
    # Investment configurations
    #
    self.MIN_RATE = 16.0 # Minimum interest rate
    self.MAX_RATE = 25.0 # Maximum interest rate
    self.PICK_LEVEL = frozenset(['5%'])
    self.AMOUNT_PER_LOAN = 25.0

  def poll(self, fx):
    """
    Generator that polls a function

    fx: a function to poll
    """
    start = dt.datetime.now()
    counter = 0
    while dt.datetime.now() - start < dt.timedelta(minutes=1):
      counter += 1
      try:
        if not counter % 10:
          self.logger.debug('Poll counter: {}'.format(counter))
        yield fx()
      except urllib2.HTTPError as err:
        self.logger.error("HTTPError: {}".format(err.code))
        time.sleep(1)
      except urllib2.URLError as err:
        self.logger.error("URLError: {}".format(err.reason))
        time.sleep(1)
      except (KeyboardInterrupt,SystemExit) as err:
        # We're trying to quit
        raise err
      except Exception as err:
        self.logger.critical("Other exception: {} {}".format(type(err), err))

    raise StopIteration("Polling timeout")

  def wait_for_new_picks(self, start=None):
    """
    Start polling for an update in listed loans
    Must be called before picks update

    start: Time before picks update
    """
    if start is None:
      _, start = self.p2p.picks()

    self.logger.debug("Starting polling picks")

    for picks, timestamp in self.poll(self.p2p.picks):
      if timestamp > start:
        self.logger.info("New picks")
        return picks

    self.logger.error("P2P-Picks polling timeout")
    raise Exception("P2P-Picks polling timeout")

  def wait_for_new_loans(self, start=None):
    """
    Start polling for an update in listed P2P-Picks
    Must be called before picks update

    start: Time before loan update
    """
    if start is None:
      start = dateparser.parse(self.lc.listed_loans()[0]['listD'])

    self.logger.debug("Starting polling loans")

    for loans in self.poll(self.lc.listed_loans):
      timestamp = dateparser.parse(loans[0]['listD'])

      if timestamp > start:
        self.logger.info("New loans")
        return loans

    self.logger.error("Listed loans polling timeout")
    raise Exception("Listed loans polling timeout")

  def invest(self, load_ids):
    """
    Attept to invest in loans by id. Reports
    successful investments to P2P-Picks.

    load_ids: a list of loan ids

    Returns: JSON reponse from lending club
    """
    try:
      # Submit order and report activity to P2P-Picks
      res = self.lc.submit_order(load_ids,
                  self.AMOUNT_PER_LOAN, self.lc_portfolio_id)
      self.p2p.report(res)
      return res
    except urllib2.HTTPError as e:
      self.logger.error(e)
    except Exception as e:
      self.logger.error(res)
      self.logger.error(e)
      raise

{'fraction_non_alphanumeric': 0.10750228728270814, 'fraction_numerical': 0.01463860933211345, 'mean_word_length': 2.7243611584327088, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '713378', 'n_tokens_mistral': 3730, 'n_tokens_neox': 3521, 'n_words': 930}