<commit_before>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# SvnPubSub - Simple Push Notification of Subversion commits
#
# Based on the theory behind the Live Journal Atom Streaming Service:
#   <http://atom.services.livejournal.com/>
#
# Instead of using a complicated XMPP/AMPQ/JMS/super messaging service,
# we have simple HTTP GETs and PUTs to get data in and out.
#
# Currently supports both XML and JSON serialization.
#
# Example Sub clients:
#   curl -sN http://127.0.0.1:2069/commits
#   curl -sN http://127.0.0.1:2069/commits/svn/*
#   curl -sN http://127.0.0.1:2069/commits/svn
#   curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68
#   curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
#
#   curl -sN http://127.0.0.1:2069/metadata
#   curl -sN http://127.0.0.1:2069/metadata/svn/*
#   curl -sN http://127.0.0.1:2069/metadata/svn
#   curl -sN http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68
#   curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68
#
#   URLs are constructed from 3 parts:
#       /${notification}/${optional_type}/${optional_repository}
#
#   Notifications can be sent for commits or metadata (e.g., revprop) changes.
#   If the type is included in the URL, you will only get notifications of that type.
#   The type can be * and then you will receive notifications of any type.
#
#   If the repository is included in the URL, you will only receive
#   messages about that repository.  The repository can be * and then you
#   will receive messages about all repositories.
#
# Example Pub clients:
#   curl -T revinfo.json -i http://127.0.0.1:2069/commits
#
# TODO:
#   - Add Real access controls (not just 127.0.0.1)
#   - Document PUT format
#   - Convert to twisted.python.log




try:
    import simplejson as json
except ImportError:
    import json

import sys

import twisted
from twisted.internet import reactor
from twisted.internet import defer
from twisted.web import server
from twisted.web import resource
from twisted.python import log

import time

class Notification(object):
    def __init__(self, r):
        self.__dict__.update(r)
        if not self.check_value('repository'):
            raise ValueError('Invalid Repository Value')
        if not self.check_value('type'):
            raise ValueError('Invalid Type Value')
        if not self.check_value('format'):
            raise ValueError('Invalid Format Value')
        if not self.check_value('id'):
            raise ValueError('Invalid ID Value')

    def check_value(self, k):
        return hasattr(self, k) and self.__dict__[k]

    def render(self):
        raise NotImplementedError

    def render_log(self):
        raise NotImplementedError

class Commit(Notification):
    KIND = 'COMMIT'

    def render(self):
        obj = {'commit': {}}
        obj['commit'].update(self.__dict__)
        return json.dumps(obj)

    def render_log(self):
        try:
            paths_changed = " %d paths changed" % len(self.changed)
        except:
            paths_changed = ""
        return "commit %s:%s repo '%s' id '%s'%s" % (
            self.type, self.format, self.repository, self.id,
            paths_changed)

class Metadata(Notification):
    KIND = 'METADATA'

    def render(self):
        obj = {'metadata': {}}
        obj['metadata'].update(self.__dict__)
        return json.dumps(obj)

    def render_log(self):
        return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % (
            self.type, self.format, self.repository, self.id,
            self.revprop.name)


HEARTBEAT_TIME = 15

class Client(object):
    def __init__(self, pubsub, r, kind, type, repository):
        self.pubsub = pubsub
        r.notifyFinish().addErrback(self.finished)
        self.r = r
        self.kind = kind
        self.type = type
        self.repository = repository
        self.alive = True
        log.msg("OPEN: %s:%d (%d clients online)"% (r.getClientIP(), r.client.port, pubsub.cc()+1))

    def finished(self, reason):
        self.alive = False
        log.msg("CLOSE: %s:%d (%d clients online)"% (self.r.getClientIP(), self.r.client.port, self.pubsub.cc()))
        try:
            self.pubsub.remove(self)
        except ValueError:
            pass

    def interested_in(self, notification):
        if self.kind != notification.KIND:
            return False<commit_msg>* tools/server-side/svnpubsub/svnpubsub/server.py   (Metadata.render_log): Fix lookup of revprop name in notification.<commit_after>#!/usr/bin/env python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# SvnPubSub - Simple Push Notification of Subversion commits
#
# Based on the theory behind the Live Journal Atom Streaming Service:
#   <http://atom.services.livejournal.com/>
#
# Instead of using a complicated XMPP/AMPQ/JMS/super messaging service,
# we have simple HTTP GETs and PUTs to get data in and out.
#
# Currently supports both XML and JSON serialization.
#
# Example Sub clients:
#   curl -sN http://127.0.0.1:2069/commits
#   curl -sN http://127.0.0.1:2069/commits/svn/*
#   curl -sN http://127.0.0.1:2069/commits/svn
#   curl -sN http://127.0.0.1:2069/commits/*/13f79535-47bb-0310-9956-ffa450edef68
#   curl -sN http://127.0.0.1:2069/commits/svn/13f79535-47bb-0310-9956-ffa450edef68
#
#   curl -sN http://127.0.0.1:2069/metadata
#   curl -sN http://127.0.0.1:2069/metadata/svn/*
#   curl -sN http://127.0.0.1:2069/metadata/svn
#   curl -sN http://127.0.0.1:2069/metadata/*/13f79535-47bb-0310-9956-ffa450edef68
#   curl -sN http://127.0.0.1:2069/metadata/svn/13f79535-47bb-0310-9956-ffa450edef68
#
#   URLs are constructed from 3 parts:
#       /${notification}/${optional_type}/${optional_repository}
#
#   Notifications can be sent for commits or metadata (e.g., revprop) changes.
#   If the type is included in the URL, you will only get notifications of that type.
#   The type can be * and then you will receive notifications of any type.
#
#   If the repository is included in the URL, you will only receive
#   messages about that repository.  The repository can be * and then you
#   will receive messages about all repositories.
#
# Example Pub clients:
#   curl -T revinfo.json -i http://127.0.0.1:2069/commits
#
# TODO:
#   - Add Real access controls (not just 127.0.0.1)
#   - Document PUT format
#   - Convert to twisted.python.log




try:
    import simplejson as json
except ImportError:
    import json

import sys

import twisted
from twisted.internet import reactor
from twisted.internet import defer
from twisted.web import server
from twisted.web import resource
from twisted.python import log

import time

class Notification(object):
    def __init__(self, r):
        self.__dict__.update(r)
        if not self.check_value('repository'):
            raise ValueError('Invalid Repository Value')
        if not self.check_value('type'):
            raise ValueError('Invalid Type Value')
        if not self.check_value('format'):
            raise ValueError('Invalid Format Value')
        if not self.check_value('id'):
            raise ValueError('Invalid ID Value')

    def check_value(self, k):
        return hasattr(self, k) and self.__dict__[k]

    def render(self):
        raise NotImplementedError

    def render_log(self):
        raise NotImplementedError

class Commit(Notification):
    KIND = 'COMMIT'

    def render(self):
        obj = {'commit': {}}
        obj['commit'].update(self.__dict__)
        return json.dumps(obj)

    def render_log(self):
        try:
            paths_changed = " %d paths changed" % len(self.changed)
        except:
            paths_changed = ""
        return "commit %s:%s repo '%s' id '%s'%s" % (
            self.type, self.format, self.repository, self.id,
            paths_changed)

class Metadata(Notification):
    KIND = 'METADATA'

    def render(self):
        obj = {'metadata': {}}
        obj['metadata'].update(self.__dict__)
        return json.dumps(obj)

    def render_log(self):
        return "metadata %s:%s repo '%s' id '%s' revprop '%s'" % (
            self.type, self.format, self.repository, self.id,
            self.revprop['name'])


HEARTBEAT_TIME = 15

class Client(object):
    def __init__(self, pubsub, r, kind, type, repository):
        self.pubsub = pubsub
        r.notifyFinish().addErrback(self.finished)
        self.r = r
        self.kind = kind
        self.type = type
        self.repository = repository
        self.alive = True
        log.msg("OPEN: %s:%d (%d clients online)"% (r.getClientIP(), r.client.port, pubsub.cc()+1))

    def finished(self, reason):
        self.alive = False
        log.msg("CLOSE: %s:%d (%d clients online)"% (self.r.getClientIP(), self.r.client.port, self.pubsub.cc()))
        try:
            self.pubsub.remove(self)
        except ValueError:
            pass

    def interested_in(self, notification):
        if self.kind != notification.KIND:
            return False
{'fraction_non_alphanumeric': 0.11417397939717665, 'fraction_numerical': 0.04044257916825639, 'mean_word_length': 2.8905380333951762, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 24, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1453146', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3233, 'n_words': 998}