<commit_before>import re
import json
import token
import tokenize
from StringIO import StringIO

from opencollab import wiki

import idiokit
from idiokit import timer
from abusehelper.core import bot, rules
from abusehelper.core.startup import Bot, StartupBot
from abusehelper.contrib.opencollab.crypto import decrypt, DecryptionError

TYPES = {"True":True, "true":True, "False":False, "false":False, "None":None,
         "none":None}

TOKENS = {"AND":rules.AND, "OR":rules.OR, "NOT":rules.NOT,
          "MATCH":rules.MATCH, "NETBLOCK":rules.NETBLOCK,
          "ANYTHING":rules.ANYTHING}

def startswith(string, words):
    for word in words:
        if string.startswith(word):
            return True
    return False

def rmlink(value):
    if value.startswith("[[") and value.endswith("]]"):
        return value[2:-2]
    return value

def get_rule_tokens(string):
    tokens = list()
    rl = StringIO(string).readline
    try:
        for ttype, tstring, start, end, string in tokenize.generate_tokens(rl):
            if ttype == token.ENDMARKER:
                break
            elif ttype not in [token.NAME,token.OP,token.STRING,token.NUMBER]:
                return
            tokens.append((ttype, tstring))
    except tokenize.TokenError, e:
        return
    return tokens

def parse_rule(tokens):
    def find_args(tokens, end):
        args = list()
        while tokens[0][1] != end:
            parsed = parse_rule(tokens)
            if parsed:
                args.append(parsed)
        tokens.pop(0)
        return tokens, args

    ttype, tstring = tokens.pop(0)
    if ttype == token.NAME:
        if tstring in TOKENS:
            ttype, parenthesis = tokens.pop(0)
            tokens, args = find_args(tokens, ")")
            return TOKENS[tstring](*args)
        elif tstring == "re":
            ttype, parenthesis = tokens.pop(0)
            tokens, args = find_args(tokens, ")")
            args.append(re.U)
            return re.compile(*args)
    elif ttype == token.OP and tstring == "[":
        tokens, values = find_args(tokens, "]")
        return values
    elif ttype in [token.STRING, token.NUMBER]:
        try:
            tstring = int(tstring)
        except ValueError:
            if tstring.startswith('"') and tstring.endswith('"') or \
               tstring.startswith("'") and tstring.endswith("'"):
                tstring = tstring[1:-1]
        return tstring

class WikiConfigInterface:
    default_metas = dict()

    @idiokit.stream
    def configs(self):
        self.collab = wiki.GraphingWiki(self.collab_url,
            ssl_verify_cert=not self.collab_ignore_cert,
            ssl_ca_certs=self.collab_extra_ca_certs)

        self.collab.authenticate(self.collab_user, self.collab_password)

        while True:
            self.log.info("Reading configs")
            if self.defaults_page:
                previous_defaults = self.default_metas.copy()
                pages = self.get_metas(self.defaults_page)
                for page, metas in pages.iteritems():
                    metas = self.parse_page(page, metas)
                    metas = self.parse_metas(metas)
                    self.default_metas.update(metas)
                if self.default_metas != previous_defaults:
                    self.log.info("Defaults values changed on page %r",
                                  self.defaults_page)
                
            pages = self.get_metas(self.category)
            if pages:
                bots = self.parse_pages(pages)
                if bots:
                    yield idiokit.send(bots)
            yield timer.sleep(self.poll_interval)

    def get_metas(self, query):
        try:
            pages = self.collab.getMeta(query)
        except wiki.WikiFailure as fail:
            self.log.error("getMeta failed: {0!r}".format(fail))<commit_msg>opencollab.wikistartup: Fixing WikiConfigInterface with wikiruntime (does not have or even need the defaults_page argument)<commit_after>import re
import json
import token
import tokenize
from StringIO import StringIO

from opencollab import wiki

import idiokit
from idiokit import timer
from abusehelper.core import bot, rules
from abusehelper.core.startup import Bot, StartupBot
from abusehelper.contrib.opencollab.crypto import decrypt, DecryptionError

TYPES = {"True":True, "true":True, "False":False, "false":False, "None":None,
         "none":None}

TOKENS = {"AND":rules.AND, "OR":rules.OR, "NOT":rules.NOT,
          "MATCH":rules.MATCH, "NETBLOCK":rules.NETBLOCK,
          "ANYTHING":rules.ANYTHING}

def startswith(string, words):
    for word in words:
        if string.startswith(word):
            return True
    return False

def rmlink(value):
    if value.startswith("[[") and value.endswith("]]"):
        return value[2:-2]
    return value

def get_rule_tokens(string):
    tokens = list()
    rl = StringIO(string).readline
    try:
        for ttype, tstring, start, end, string in tokenize.generate_tokens(rl):
            if ttype == token.ENDMARKER:
                break
            elif ttype not in [token.NAME,token.OP,token.STRING,token.NUMBER]:
                return
            tokens.append((ttype, tstring))
    except tokenize.TokenError, e:
        return
    return tokens

def parse_rule(tokens):
    def find_args(tokens, end):
        args = list()
        while tokens[0][1] != end:
            parsed = parse_rule(tokens)
            if parsed:
                args.append(parsed)
        tokens.pop(0)
        return tokens, args

    ttype, tstring = tokens.pop(0)
    if ttype == token.NAME:
        if tstring in TOKENS:
            ttype, parenthesis = tokens.pop(0)
            tokens, args = find_args(tokens, ")")
            return TOKENS[tstring](*args)
        elif tstring == "re":
            ttype, parenthesis = tokens.pop(0)
            tokens, args = find_args(tokens, ")")
            args.append(re.U)
            return re.compile(*args)
    elif ttype == token.OP and tstring == "[":
        tokens, values = find_args(tokens, "]")
        return values
    elif ttype in [token.STRING, token.NUMBER]:
        try:
            tstring = int(tstring)
        except ValueError:
            if tstring.startswith('"') and tstring.endswith('"') or \
               tstring.startswith("'") and tstring.endswith("'"):
                tstring = tstring[1:-1]
        return tstring

class WikiConfigInterface:
    default_metas = dict()

    @idiokit.stream
    def configs(self):
        self.collab = wiki.GraphingWiki(self.collab_url,
            ssl_verify_cert=not self.collab_ignore_cert,
            ssl_ca_certs=self.collab_extra_ca_certs)

        self.collab.authenticate(self.collab_user, self.collab_password)

        while True:
            self.log.info("Reading configs")
            if getattr(self, "defaults_page", None):
                previous_defaults = self.default_metas.copy()
                pages = self.get_metas(self.defaults_page)
                for page, metas in pages.iteritems():
                    metas = self.parse_page(page, metas)
                    metas = self.parse_metas(metas)
                    self.default_metas.update(metas)
                if self.default_metas != previous_defaults:
                    self.log.info("Defaults values changed on page %r",
                                  self.defaults_page)
                
            pages = self.get_metas(self.category)
            if pages:
                bots = self.parse_pages(pages)
                if bots:
                    yield idiokit.send(bots)
            yield timer.sleep(self.poll_interval)

    def get_metas(self, query):
        try:
            pages = self.collab.getMeta(query)
        except wiki.WikiFailure as fail:
            self.log.error("getMeta failed: {0!r}".format(fail))
{'fraction_non_alphanumeric': 0.10454429410262957, 'fraction_numerical': 0.002808271636456472, 'mean_word_length': 2.0846456692913384, 'pattern_counts': {'":': 28, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '968380', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2300, 'n_words': 530}