<commit_before>import base64
import datetime
from functools import partial
import hashlib
import pickle
from SimpleXMLRPCServer import SimpleXMLRPCDispatcher

import redis

from django.http import HttpResponse, HttpResponseBadRequest
from django.conf import settings
from django.core.urlresolvers import reverse
from django.views.generic import View

from humfrey.utils.views import AuthenticatedView
from humfrey.pingback.longliving.pingback_server import NewPingbackHandler, RetrievedPingbackHandler
from django_conneg.views import HTMLView, JSONPView
from django_conneg.http import HttpResponseSeeOther

def get_redis_client():
    return redis.client.Redis(**settings.REDIS_PARAMS)

class PingbackView(View):
    _QUEUE_KEY = 'pingback.new'

    class PingbackError(Exception): pass
    class AlreadyRegisteredError(PingbackError): pass

    def ping(self, request, source, target):
        client = get_redis_client()

        ping_hash = ''.join('%02x' % (a ^ b) for a, b in zip(*(map(ord, hashlib.sha1(x).digest()) for x in (source, target))))

        data = base64.b64encode(pickle.dumps({
            'hash': ping_hash,
            'source': source,
            'target': target,
            'user_agent': request.META.get('HTTP_USER_AGENT'),
            'remote_addr': request.META.get('REMOTE_ADDR'),
            'date': datetime.datetime.now(),
            'state': 'new',
        }))
        print "PING", data
        stored = client.setnx('pingback:item:%s' % ping_hash, data)
        if stored:
            client.rpush(NewPingbackHandler.QUEUE_NAME, ping_hash)
        else:
            raise self.AlreadyRegisteredError()

class XMLRPCPingbackView(PingbackView):
    _RESPONSE_CODES = {
        'GENERIC_FAULT': 0x0,
        'SOURCE_NOT_FOUND': 0x10,
        'SOURCE_DOESNT_LINK': 0x11,
        'TARGET_NOT_FOUND': 0x20,
        'TARGET_INVALID': 0x21,
        'ALREADY_REGISTERED': 0x30,
        'ACCESS_DENIED': 0x31,
        'BAD_GATEWAY': 0x32,
    }

    def post(self, request):
        dispatcher = SimpleXMLRPCDispatcher(allow_none=False, encoding=None)
        dispatcher.register_function(partial(self.ping, request), 'pingback:ping')

        response = HttpResponse(mimetype="application/xml")
        response.write(dispatcher._marshaled_dispatch(request.raw_post_data))
        return response

    def ping(self, request, sourceURI, targetURI):
        try:
            super(XMLRPCPingbackView, self).ping(request, sourceURI, targetURI)
        except self.AlreadyRegisteredError:
            return self._RESPONSE_CODES['ALREADY_REGISTERED']
        except self.PingbackError:
            return self._RESPONSE_CODES['GENERIC_FAULT']
        except Exception:
            raise
        else:
            return "OK"

class RESTfulPingbackView(PingbackView):
    def post(self, request):
        try:
            self.ping(request, request.POST['source'], request.POST['target'])
        except Exception:
            return HttpResponseBadRequest()
        else:
            response = HttpResponse()
            response.status_code = 202
            return response

class ModerationView(HTMLView, JSONPView, AuthenticatedView):
    def initial_context(self, request):
        client = get_redis_client()
        pingback_hashes = ['pingback:item:%s' % s for s in client.smembers(RetrievedPingbackHandler.PENDING_QUEUE_NAME)]
        if pingback_hashes:
            pingbacks = client.mget(pingback_hashes)
            pingbacks = [pickle.loads(base64.b64decode(p)) for p in pingbacks]
            pingbacks.sort(key=lambda d: d['date'])
        else:
            pingbacks = []
        return {
            'client': client,
            'pingbacks': pingbacks,
        }

    def handle_GET(self, request, context):
        return self.render(request, context, 'pingback/moderation')

    def handle_POST(self, request, context):
        client = context['client']
        for k in request.POST:
            ping_hash, action = k.split(':')[-1], request.POST[k]
            key_name = 'pingback:item:%s' % ping_hash
            if action in ('accept', 'reject') and not (k.startswith('action:') and client.srem('pingback.pending', ping_hash)):
                continue
            data = pickle.loads(base64.b64decode(client.get(key_name)))
            if action == 'accept':
                data['state'] = 'accepted'
                client.rpush('pingback:accepted', ping_hash)
            else:
                data['state'] = 'rejected'
            client.set(key_name, base64.b64encode(pickle.dumps(data)))
            client.expire(key_name, 3600 * 24 * 7)
        return HttpResponseSeeOther(reverse('pingback:moderation'))<commit_msg>Finished migrating pingback stuff to newer features.<commit_after>import base64
import datetime
from functools import partial
import hashlib
import pickle
from SimpleXMLRPCServer import SimpleXMLRPCDispatcher

import redis

from django.http import HttpResponse, HttpResponseBadRequest
from django.conf import settings
from django.core.urlresolvers import reverse
from django.views.generic import View

from humfrey.utils.views import AuthenticatedView, RedisView
from humfrey.pingback.longliving.pingback_server import NewPingbackHandler, RetrievedPingbackHandler
from django_conneg.views import HTMLView, JSONPView
from django_conneg.http import HttpResponseSeeOther

class PingbackView(RedisView):
    _QUEUE_KEY = 'pingback.new'

    class PingbackError(Exception): pass
    class AlreadyRegisteredError(PingbackError): pass

    def ping(self, request, source, target):
        if source == target:
            return HttpResponseBadRequest()

        client = self.get_redis_client()

        ping_hash = ''.join('%02x' % (a ^ b) for a, b in zip(*(map(ord, hashlib.sha1(x).digest()) for x in (source, target))))

        data = base64.b64encode(pickle.dumps({
            'hash': ping_hash,
            'source': source,
            'target': target,
            'user_agent': request.META.get('HTTP_USER_AGENT'),
            'remote_addr': request.META.get('REMOTE_ADDR'),
            'date': datetime.datetime.now(),
            'state': 'new',
        }))

        stored = client.setnx('pingback:item:%s' % ping_hash, data)
        if stored:
            client.rpush(NewPingbackHandler.QUEUE_NAME, ping_hash)
        else:
            raise self.AlreadyRegisteredError()

class XMLRPCPingbackView(PingbackView):
    _RESPONSE_CODES = {
        'GENERIC_FAULT': 0x0,
        'SOURCE_NOT_FOUND': 0x10,
        'SOURCE_DOESNT_LINK': 0x11,
        'TARGET_NOT_FOUND': 0x20,
        'TARGET_INVALID': 0x21,
        'ALREADY_REGISTERED': 0x30,
        'ACCESS_DENIED': 0x31,
        'BAD_GATEWAY': 0x32,
    }

    def post(self, request):
        dispatcher = SimpleXMLRPCDispatcher(allow_none=False, encoding=None)
        dispatcher.register_function(partial(self.ping, request), 'pingback:ping')

        response = HttpResponse(mimetype="application/xml")
        response.write(dispatcher._marshaled_dispatch(request.raw_post_data))
        return response

    def ping(self, request, sourceURI, targetURI):
        try:
            super(XMLRPCPingbackView, self).ping(request, sourceURI, targetURI)
        except self.AlreadyRegisteredError:
            return self._RESPONSE_CODES['ALREADY_REGISTERED']
        except self.PingbackError:
            return self._RESPONSE_CODES['GENERIC_FAULT']
        except Exception:
            raise
        else:
            return "OK"

class RESTfulPingbackView(PingbackView):
    def post(self, request):
        try:
            self.ping(request, request.POST['source'], request.POST['target'])
        except Exception:
            return HttpResponseBadRequest()
        else:
            response = HttpResponse()
            response.status_code = 202
            return response

class ModerationView(HTMLView, JSONPView, AuthenticatedView, RedisView):
    def common(self, request):
        client = self.get_redis_client()
        pingback_hashes = ['pingback:item:%s' % s for s in client.smembers(RetrievedPingbackHandler.PENDING_QUEUE_NAME)]
        if pingback_hashes:
            pingbacks = client.mget(pingback_hashes)
            pingbacks = [pickle.loads(base64.b64decode(p)) for p in pingbacks]
            pingbacks.sort(key=lambda d: d['date'])
        else:
            pingbacks = []
        return {
            'client': client,
            'pingbacks': pingbacks,
        }

    def get(self, request):
        context = self.common(request)
        return self.render(request, context, 'pingback/moderation')

    def post(self, request):
        context = self.common(request)
        client = context['client']
        for k in request.POST:
            ping_hash, action = k.split(':')[-1], request.POST[k]
            key_name = 'pingback:item:%s' % ping_hash
            if action in ('accept', 'reject') and not (k.startswith('action:') and client.srem('pingback.pending', ping_hash)):
                continue
            data = pickle.loads(base64.b64decode(client.get(key_name)))
            if action == 'accept':
                data['state'] = 'accepted'
                client.rpush('pingback:accepted', ping_hash)
            else:
                data['state'] = 'rejected'
            client.set(key_name, base64.b64encode(pickle.dumps(data)))
            client.expire(key_name, 3600 * 24 * 7)
        return HttpResponseSeeOther(reverse('pingback:moderation'))
{'fraction_non_alphanumeric': 0.10431007095202796, 'fraction_numerical': 0.01164884041088637, 'mean_word_length': 2.767052253689669, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7358835', 'n_tokens_mistral': 2940, 'n_tokens_neox': 2791, 'n_words': 585}