<commit_before>#License#
#bitHopper by <NAME> is licensed under a Creative Commons 
# Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import json
import base64
import traceback

import eventlet
httplib2 = eventlet.import_patched('httplib20_7_1')
from eventlet import pools
from eventlet.green import socket

from peak.util import plugins

# Global timeout for sockets in case something leaks
socket.setdefaulttimeout(900)

import webob

class Work():
    def __init__(self, bitHopper):
        self.bitHopper = bitHopper
        self.i = 0
        self.connect_pool = {}
        #pools.Pool(min_size = 2, max_size = 10, create = lambda: httplib2.Http(disable_ssl_certificate_validation=True))

    def get_http(self, address, timeout=0):
        try:
            configured_timeout = self.bitHopper.config.getfloat('main','work_request_timeout')
        except:
            configured_timeout = 2.5
            pass
        if timeout == 0:
            timeout = configured_timeout
        
        if address not in self.connect_pool:
            self.connect_pool[address] =  pools.Pool(min_size = 0, create = lambda: httplib2.Http(disable_ssl_certificate_validation=True, timeout=timeout))
        return self.connect_pool[address].item()

    def jsonrpc_lpcall(self, server, url, lp):
        try:
            #self.i += 1
            #request = json.dumps({'method':'getwork', 'params':[], 'id':self.i}, ensure_ascii = True)
            user, passw, error = self.user_substitution(server, None, None)
            #Check if we are using {USER} or {PASSWORD}
            if error:
                return None
            header = {'Authorization':"Basic " +base64.b64encode(user+ ":" + passw).replace('\n',''), 'user-agent': 'poclbm/20110709', 'Content-Type': 'application/json', 'connection': 'keep-alive'}
            with self.get_http(url, timeout=15*60) as http:
                try:
                    resp, content = http.request( url, 'GET', headers=header)#, body=request)[1] # Returns response dict and content str
                except Exception, e:
                    self.bitHopper.log_dbg('Error with a jsonrpc_lpcall http request')
                    self.bitHopper.log_dbg(e)
                    content = None
            lp.receive(content, server)
            return None
        except Exception, e:
            self.bitHopper.log_dbg('Error in lpcall work.py')
            self.bitHopper.log_dbg(e)
            lp.receive(None, server)
            return None

    def get(self, url, useragent=None):
        """A utility method for getting webpages"""
        if useragent == None:
            try:
                useragent = self.bitHopper.config.get('main', 'work_user_agent')
            except:
                useragent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'
                pass
        #self.bitHopper.log_dbg('user-agent: ' + useragent + ' for ' + str(url) )
        header = {'user-agent':useragent}
        with self.get_http(url) as http:
            try:
                content = http.request( url, 'GET', headers=header)[1] # Returns response dict and content str
            except Exception, e:
                self.bitHopper.log_dbg('Error with a work.get() http request: ' + str(e))
                content = ""
        return content

    def user_substitution(self, server, username, password):
        info = self.bitHopper.pool.get_entry(server)
        if '{USER}' in info['user'] and username is not None:
            user = info['user'].replace('{USER}', username)
        else:
            user = info['user']
        if '{PASSWORD}' in info['pass'] and password is not None:
            passw = info['pass'].replace('{PASSWORD}', password)
        else:
            passw = info['pass']
        if '{USER}' in info['user'] and username is None or '{PASSWORD}' in info['pass'] and password is None:
            error = True
        else:
            error = False
        return user, passw, error

    def jsonrpc_call(self, server, data, client_header={}, username = None, password = None):
        try:
            request = json.dumps({'method':'getwork', 'params':data, 'id':self.i}, ensure_ascii = True)
            self.i += 1
            
            info = self.bitHopper.pool.get_entry(server)
            user, passw, error = self.user_substitution(server, username, password)
            header = {'Authorization':"Basic " +base64.b64encode(user + ":" + passw).replace('\n',''), 'connection': 'keep-alive'}
            header['user-agent'] = 'poclbm/20110709'
            for k,v in client_header.items():
                #Ugly hack to deal with httplib trying to be smart and supplying its own user agent.
                if k.lower() in [ 'user-agent', 'user_agent']:
                    header['user-agent'] = v
                if k.lower() in ['x-mining-extensions', 'x-mining-hashrate']:
                    header[k] = v

            url = "http://" + info['mine_address']
            with self.get_http(url) as http:
                try:
                    resp, content = http.request( url, 'POST', headers=header, body=request)
                except Exception, e:
                    self.bitHopper.log_dbg('jsonrpc_call http error: ' + str(e))
                    return None, None

            #Check for long polling header
            lp = self.bitHopper.lp
            if lp.check_lp(server):
                #bitHopper.log_msg('Inside LP check')
                for k,v in resp.items():
                    if k.lower() == 'x-long-polling':
                        lp.set_lp(v,server)
                        break
        except Exception, e:
            self.bitHopper.log_dbg('jsonrpc_call error: ' + str(e))
            if self.bitHopper.options.debug:
                traceback.print_exc()
            return None, None

        try:
            message = json.loads(content)
            value =  message['result']
            return value, resp
        except Exception, e:
            self.bitHopper.log_dbg( "Error in json decoding, Server probably down")
            self.bitHopper.log_dbg(server)
            self.bitHopper.log_dbg(content)
            return None, None

    def jsonrpc_getwork(self, server, data,  headers={}, username = None, password = <PASSWORD>):
        tries = 0
        work = None
        while work == None:
            if data == [] and tries > 1:
                server = self.bitHopper.get_new_server(server)
            elif data != [] and tries > 1:
                self.bitHopper.get_new_server(server)
            if tries >5:
                return None, {}, 'No Server'
            tries += 1
            try:
                work, server_headers = self.jsonrpc_call(server, data, headers, username, password)
            except Exception, e:
                self.bitHopper.log_dbg( 'caught, inner jsonrpc_call loop')
                self.bitHopper.log_dbg(server)
                self.bitHopper.log_dbg(e)
                work = None
        return work, server_headers, server<commit_msg>Don't retry as much to try and lower getwork spam<commit_after>#License#
#bitHopper by <NAME> is licensed under a Creative Commons 
# Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import json
import base64
import traceback

import eventlet
httplib2 = eventlet.import_patched('httplib20_7_1')
from eventlet import pools
from eventlet.green import socket

from peak.util import plugins

# Global timeout for sockets in case something leaks
socket.setdefaulttimeout(900)

import webob

class Work():
    def __init__(self, bitHopper):
        self.bitHopper = bitHopper
        self.i = 0
        self.connect_pool = {}
        #pools.Pool(min_size = 2, max_size = 10, create = lambda: httplib2.Http(disable_ssl_certificate_validation=True))

    def get_http(self, address, timeout=0):
        try:
            configured_timeout = self.bitHopper.config.getfloat('main','work_request_timeout')
        except:
            configured_timeout = 2.5
            pass
        if timeout == 0:
            timeout = configured_timeout
        
        if address not in self.connect_pool:
            self.connect_pool[address] =  pools.Pool(min_size = 0, create = lambda: httplib2.Http(disable_ssl_certificate_validation=True, timeout=timeout))
        return self.connect_pool[address].item()

    def jsonrpc_lpcall(self, server, url, lp):
        try:
            #self.i += 1
            #request = json.dumps({'method':'getwork', 'params':[], 'id':self.i}, ensure_ascii = True)
            user, passw, error = self.user_substitution(server, None, None)
            #Check if we are using {USER} or {PASSWORD}
            if error:
                return None
            header = {'Authorization':"Basic " +base64.b64encode(user+ ":" + passw).replace('\n',''), 'user-agent': 'poclbm/20110709', 'Content-Type': 'application/json', 'connection': 'keep-alive'}
            with self.get_http(url, timeout=15*60) as http:
                try:
                    resp, content = http.request( url, 'GET', headers=header)#, body=request)[1] # Returns response dict and content str
                except Exception, e:
                    self.bitHopper.log_dbg('Error with a jsonrpc_lpcall http request')
                    self.bitHopper.log_dbg(e)
                    content = None
            lp.receive(content, server)
            return None
        except Exception, e:
            self.bitHopper.log_dbg('Error in lpcall work.py')
            self.bitHopper.log_dbg(e)
            lp.receive(None, server)
            return None

    def get(self, url, useragent=None):
        """A utility method for getting webpages"""
        if useragent == None:
            try:
                useragent = self.bitHopper.config.get('main', 'work_user_agent')
            except:
                useragent = 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)'
                pass
        #self.bitHopper.log_dbg('user-agent: ' + useragent + ' for ' + str(url) )
        header = {'user-agent':useragent}
        with self.get_http(url) as http:
            try:
                content = http.request( url, 'GET', headers=header)[1] # Returns response dict and content str
            except Exception, e:
                self.bitHopper.log_dbg('Error with a work.get() http request: ' + str(e))
                content = ""
        return content

    def user_substitution(self, server, username, password):
        info = self.bitHopper.pool.get_entry(server)
        if '{USER}' in info['user'] and username is not None:
            user = info['user'].replace('{USER}', username)
        else:
            user = info['user']
        if '{PASSWORD}' in info['pass'] and password is not None:
            passw = info['pass'].replace('{PASSWORD}', password)
        else:
            passw = info['pass']
        if '{USER}' in info['user'] and username is None or '{PASSWORD}' in info['pass'] and password is None:
            error = True
        else:
            error = False
        return user, passw, error

    def jsonrpc_call(self, server, data, client_header={}, username = None, password = None):
        try:
            request = json.dumps({'method':'getwork', 'params':data, 'id':self.i}, ensure_ascii = True)
            self.i += 1
            
            info = self.bitHopper.pool.get_entry(server)
            user, passw, error = self.user_substitution(server, username, password)
            header = {'Authorization':"Basic " +base64.b64encode(user + ":" + passw).replace('\n',''), 'connection': 'keep-alive'}
            header['user-agent'] = 'poclbm/20110709'
            for k,v in client_header.items():
                #Ugly hack to deal with httplib trying to be smart and supplying its own user agent.
                if k.lower() in [ 'user-agent', 'user_agent']:
                    header['user-agent'] = v
                if k.lower() in ['x-mining-extensions', 'x-mining-hashrate']:
                    header[k] = v

            url = "http://" + info['mine_address']
            with self.get_http(url) as http:
                try:
                    resp, content = http.request( url, 'POST', headers=header, body=request)
                except Exception, e:
                    self.bitHopper.log_dbg('jsonrpc_call http error: ' + str(e))
                    return None, None

            #Check for long polling header
            lp = self.bitHopper.lp
            if lp.check_lp(server):
                #bitHopper.log_msg('Inside LP check')
                for k,v in resp.items():
                    if k.lower() == 'x-long-polling':
                        lp.set_lp(v,server)
                        break
        except Exception, e:
            self.bitHopper.log_dbg('jsonrpc_call error: ' + str(e))
            if self.bitHopper.options.debug:
                traceback.print_exc()
            return None, None

        try:
            message = json.loads(content)
            value =  message['result']
            return value, resp
        except Exception, e:
            self.bitHopper.log_dbg( "Error in json decoding, Server probably down")
            self.bitHopper.log_dbg(server)
            self.bitHopper.log_dbg(content)
            return None, None

    def jsonrpc_getwork(self, server, data,  headers={}, username = None, password = None):
        tries = 0
        work = None
        while work == None:
            if data == [] and tries > 1:
                server = self.bitHopper.get_new_server(server)
            elif data != [] and tries > 1:
                self.bitHopper.get_new_server(server)
            if tries >2:
                return None, {}, 'No Server'
            tries += 1
            try:
                work, server_headers = self.jsonrpc_call(server, data, headers, username, password)
            except Exception, e:
                self.bitHopper.log_dbg( 'caught, inner jsonrpc_call loop')
                self.bitHopper.log_dbg(server)
                self.bitHopper.log_dbg(e)
                work = None
        return work, server_headers, server
{'fraction_non_alphanumeric': 0.11203989884799101, 'fraction_numerical': 0.009834223096375386, 'mean_word_length': 1.9488400994200497, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '149420', 'n_tokens_mistral': 4229, 'n_tokens_neox': 3984, 'n_words': 1062}