<commit_before># coding=utf-8
import base64
import random
import requests
import json
import logging

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            r = requests.get('http://172.16.31.10:8000/?type=1&count=20')
            #logging.info('contents:%s' % contents.text)
            data = json.loads(r.text)
            if len(data) > 0:
                proxy = random.choice(data)
                ip = proxy.get('ip')
                port = proxy.get('port')
                address = '%s:%s' % (ip, port)
                #request.meta['proxy'] = "http://%s" % proxy['ip_port']
                request.meta['proxy'] = 'http://%s' % address
                logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
        except:
            pass<commit_msg>更改解析代理 IP 格式为 json<commit_after># coding=utf-8
import base64
import random
import requests
import json
import logging

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            r = requests.get('http://172.16.31.10:8000/?type=1&count=20')
            #logging.info('contents:%s' % contents.text)
            data = json.loads(r.text)
            if len(data) > 0:
                proxy = random.choice(data)
                ip = proxy.get('ip')
                port = proxy.get('port')
                address = '%s:%s' % (ip, port)

                request.meta['proxy'] = 'http://%s' % address
                logging.info('********ProxyMiddleware proxy*******:%s' % request.meta['proxy'] )
        except Exception, e:
            logging.warning('ProxyMiddleware Exception:%s' % str(e))

{'fraction_non_alphanumeric': 0.16069295101553166, 'fraction_numerical': 0.023894862604540025, 'mean_word_length': 2.067765567765568, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5976589', 'n_tokens_mistral': 521, 'n_tokens_neox': 488, 'n_words': 101}