<commit_before>#encoding=utf-8
from __future__ import unicode_literals
import re
import urllib
from bs4 import BeautifulSoup
from danmicholoparser import DanmicholoParser, DanmicholoParseError


class Rule(object):

    def __init__(self):
        pass

    def iszero(self, f):
        f = float(f)
        return (f > -0.1 and f < 0.1)
    
    def add_points(self, rev, points, ptype, txt, pmax, include_zero = False):

        ab = rev.article.get_points(ptype)
        ab_raw = rev.article.get_points(ptype, ignore_max = True)
        
        if pmax > 0.0 and self.iszero(ab - pmax) :
            # we have reached max
            if points < 0.0 and  ab_raw + points < pmax:
                rev.points.append([pmax - ab_raw - points, ptype, txt, points])
            else:
                rev.points.append([0.0, ptype, txt, points])

        elif pmax > 0.0 and ab + points > pmax:
            # reaching max
            rev.points.append([pmax - ab, ptype, txt + ' &gt; maks', points])

        #elif not self.iszero(revpoints):
        else:
            if self.iszero(points) and not include_zero:
                return
            rev.points.append([points, ptype, txt, points])


class NewPageRule(Rule):
    
    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)

    def test(self, rev):
        if rev.new and not rev.article.redirect:
            rev.points.append([self.points, 'newpage', 'ny side'])


class StubRule(Rule):

    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)

    def is_stub(self, text):
        """ Checks if a given text is a stub """

        dp = DanmicholoParser(text, debug = False)
        for tname, templ in dp.templates.iteritems():
            if tname.find('stubb') != -1 or tname.find('spire') != -1:
                return True
        return False

    def test(self, rev):
        try:
            if self.is_stub(rev.parenttext) and not self.is_stub(rev.text):
                rev.points.append([self.points, 'stub', 'avstubbing'])
        except DanmicholoParseError as e:
            rev.article.errors.append('Problem ved parsing av [%s rev. %d] : %s' % (rev.get_link(), rev.revid, e.msg))
    

class QualiRule(Rule):

    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)
    
    def test(self, rev):
        if self.iszero(rev.article.get_points('quali')):
            rev.points.append([self.points, 'quali', 'kvalifisert'])


class ByteRule(Rule):

    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)


    def test(self, rev):
        revpoints = rev.bytes * self.points
        self.add_points(rev, revpoints, 'byte', '%.f bytes' % rev.bytes, self.maxpoints, include_zero = True)


class WordRule(Rule):

    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)

    def get_wordcount(self, txt):
        dp = DanmicholoParser(txt)
        return len(dp.maintext.split())

    def test(self, rev):

        try:
            nwords = self.get_wordcount(rev.text)
        except DanmicholoParseError as e:
            rev.article.errors.append('Det oppstod et problem ved parsing av revisjon %d som kan ha påvirket ordtellingen for denne revisjonen: %s' % (rev.revid,e.msg))
            nwords = -1
        
        try:
            nwords_p = self.get_wordcount(rev.parenttext)
        except DanmicholoParseError as e:
            rev.article.errors.append('Det oppstod et problem ved parsing av foreldrerevisjon %d som kan ha påvirket ordtellingen for denne revisjonen: %s' % (rev.parentid,e.msg))
            nwords_p = -1

        if nwords == -1 or nwords_p == -1:
            # oh noes!
            pass
        else:
            words = nwords - nwords_p
            revpoints = words * self.points
            self.add_points(rev, revpoints, 'word', '%.f ord' % words, self.maxpoints)


class ImageRule(Rule):
    
    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)

    def get_imagecount(self, txt):
        return len(re.findall(r'(?:\.svg|\.png|\.jpg)', txt, flags = re.IGNORECASE))

    def test(self, rev):
       
        nimages = self.get_imagecount(rev.text)
        nimages_p = self.get_imagecount(rev.parenttext)
        imgs = nimages - nimages_p

        revpoints = imgs * self.points
        self.add_points(rev, revpoints, 'image', '%d bilde%s' % (imgs, 'r' if imgs > 1 else ''), self.maxpoints)
 

class RefRule(Rule):
    
    def __init__(self, sourcepoints, refpoints):
        """
        sourcepoints: points for adding new sources
        refpoints: points for referring to existing sources
        """
        Rule.__init__(self)
        self.sourcepoints = float(sourcepoints)
        self.refpoints = float(refpoints)
            
    def test(self, rev):
        """
        While BeautifulSoup and its parsers are robust, (unknown) tags with unquoted arguments seems to be an issue.

        Let's first define a function to make things clearer:
        >>> def f(str):
        >>>     return ''.join([unicode(tag) for tag in BeautifulSoup(str, 'lxml').findAll('body')[0].contents])

        Now, here is an unexpected result: the ref-tag is not read as closed and continue to eat the remaining text!
        >>> f('<ref name=XYZ/>Mer tekst her')
        <<< u'<ref name="XYZ/">Mer tekst her</ref>'

        Add a space before / and we get the expected result:
        >>> f('<ref name=XYZ />Mer tekst her')<commit_msg>gir ikke minuspoeng for fjerning av bilder<commit_after>#encoding=utf-8
from __future__ import unicode_literals
import re
import urllib
from bs4 import BeautifulSoup
from danmicholoparser import DanmicholoParser, DanmicholoParseError


class Rule(object):

    def __init__(self):
        pass

    def iszero(self, f):
        f = float(f)
        return (f > -0.1 and f < 0.1)
    
    def add_points(self, rev, points, ptype, txt, pmax, include_zero = False):

        ab = rev.article.get_points(ptype)
        ab_raw = rev.article.get_points(ptype, ignore_max = True)
        
        if pmax > 0.0 and self.iszero(ab - pmax) :
            # we have reached max
            if points < 0.0 and  ab_raw + points < pmax:
                rev.points.append([pmax - ab_raw - points, ptype, txt, points])
            else:
                rev.points.append([0.0, ptype, txt, points])

        elif pmax > 0.0 and ab + points > pmax:
            # reaching max
            rev.points.append([pmax - ab, ptype, txt + ' &gt; maks', points])

        #elif not self.iszero(revpoints):
        else:
            if self.iszero(points) and not include_zero:
                return
            rev.points.append([points, ptype, txt, points])


class NewPageRule(Rule):
    
    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)

    def test(self, rev):
        if rev.new and not rev.article.redirect:
            rev.points.append([self.points, 'newpage', 'ny side'])


class StubRule(Rule):

    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)

    def is_stub(self, text):
        """ Checks if a given text is a stub """

        dp = DanmicholoParser(text, debug = False)
        for tname, templ in dp.templates.iteritems():
            if tname.find('stubb') != -1 or tname.find('spire') != -1:
                return True
        return False

    def test(self, rev):
        try:
            if self.is_stub(rev.parenttext) and not self.is_stub(rev.text):
                rev.points.append([self.points, 'stub', 'avstubbing'])
        except DanmicholoParseError as e:
            rev.article.errors.append('Problem ved parsing av [%s rev. %d] : %s' % (rev.get_link(), rev.revid, e.msg))
    

class QualiRule(Rule):

    def __init__(self, points):
        Rule.__init__(self)
        self.points = float(points)
    
    def test(self, rev):
        if self.iszero(rev.article.get_points('quali')):
            rev.points.append([self.points, 'quali', 'kvalifisert'])


class ByteRule(Rule):

    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)


    def test(self, rev):
        revpoints = rev.bytes * self.points
        self.add_points(rev, revpoints, 'byte', '%.f bytes' % rev.bytes, self.maxpoints, include_zero = True)


class WordRule(Rule):

    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)

    def get_wordcount(self, txt):
        dp = DanmicholoParser(txt)
        return len(dp.maintext.split())

    def test(self, rev):

        try:
            nwords = self.get_wordcount(rev.text)
        except DanmicholoParseError as e:
            rev.article.errors.append('Det oppstod et problem ved parsing av revisjon %d som kan ha påvirket ordtellingen for denne revisjonen: %s' % (rev.revid,e.msg))
            nwords = -1
        
        try:
            nwords_p = self.get_wordcount(rev.parenttext)
        except DanmicholoParseError as e:
            rev.article.errors.append('Det oppstod et problem ved parsing av foreldrerevisjon %d som kan ha påvirket ordtellingen for denne revisjonen: %s' % (rev.parentid,e.msg))
            nwords_p = -1

        if nwords == -1 or nwords_p == -1:
            # oh noes!
            pass
        else:
            words = nwords - nwords_p
            revpoints = words * self.points
            self.add_points(rev, revpoints, 'word', '%.f ord' % words, self.maxpoints)


class ImageRule(Rule):
    
    def __init__(self, points, maxpoints = -1):
        Rule.__init__(self)
        self.points = float(points)
        self.maxpoints = float(maxpoints)

    def get_imagecount(self, txt):
        return len(re.findall(r'(?:\.svg|\.png|\.jpg)', txt, flags = re.IGNORECASE))

    def test(self, rev):
       
        nimages = self.get_imagecount(rev.text)
        nimages_p = self.get_imagecount(rev.parenttext)
        imgs = nimages - nimages_p

        if imgs > 0:
            revpoints = imgs * self.points
            self.add_points(rev, revpoints, 'image', '%d bilde%s' % (imgs, 'r' if imgs > 1 else ''), self.maxpoints)
 

class RefRule(Rule):
    
    def __init__(self, sourcepoints, refpoints):
        """
        sourcepoints: points for adding new sources
        refpoints: points for referring to existing sources
        """
        Rule.__init__(self)
        self.sourcepoints = float(sourcepoints)
        self.refpoints = float(refpoints)
            
    def test(self, rev):
        """
        While BeautifulSoup and its parsers are robust, (unknown) tags with unquoted arguments seems to be an issue.

        Let's first define a function to make things clearer:
        >>> def f(str):
        >>>     return ''.join([unicode(tag) for tag in BeautifulSoup(str, 'lxml').findAll('body')[0].contents])

        Now, here is an unexpected result: the ref-tag is not read as closed and continue to eat the remaining text!
        >>> f('<ref name=XYZ/>Mer tekst her')
        <<< u'<ref name="XYZ/">Mer tekst her</ref>'

        Add a space before / and we get the expected result:
        >>> f('<ref name=XYZ />Mer tekst her')
{'fraction_non_alphanumeric': 0.11639571082670357, 'fraction_numerical': 0.004410238671739883, 'mean_word_length': 2.231349538977368, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2671313', 'n_tokens_mistral': 3691, 'n_tokens_neox': 3578, 'n_words': 980}