<commit_before>#!/usr/bin/env python
import daemon
import lxml.html
import math
import os.path
import sys
import string
import pickle
import tornado.escape
import tornado.httpclient
import tornado.httpserver
import tornado.ioloop
import tornado.web


class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/measure", MeasureHandler)
        ]
        settings = dict(
			debug=True,
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static")
        )
        tornado.web.Application.__init__(self, handlers, **settings)


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("index.html")


class Page(object):
    words = frozenset([w.strip() for w in open("liste.txt")])

    def __init__(self, body):
        self.doc = lxml.html.fromstring(body.decode('utf8'))

    def absolute_links(self, url):
        self.doc.make_links_absolute(url)
        base = lxml.html.Element('base', dict(href=url))
        self.doc.head.append(base)

    def compute_score(self):
        txt = self.doc.text_content()
        for punct in string.punctuation:
            txt = txt.replace(punct," ")
        txt = txt.split()
        count = len(filter(lambda x: x in self.words, txt))
        return count / math.log10(2 + len(txt))

    def inject_score(self, score):
        #txt = '<div style="position:fixed;top:0px;left:0px;background-color:green;width=100%%;height=4px"><span style="background-color:red;width=(%.1f)%%;" >trollscore&nbsp;"</span></div>' % score * 10 
        #if score > 10.0:
        #    txt = '/!\\ ' + txt
        trolldiv = lxml.html.Element('span')<commit_msg>cleanup comments remove debug<commit_after>#!/usr/bin/env python
import daemon
import lxml.html
import math
import os.path
import sys
import string
import tornado.escape
import tornado.httpclient
import tornado.httpserver
import tornado.ioloop
import tornado.web


class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/measure", MeasureHandler)
        ]
        settings = dict(
			debug=False,
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static")
        )
        tornado.web.Application.__init__(self, handlers, **settings)


class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("index.html")


class Page(object):
    words = frozenset([w.strip() for w in open("liste.txt")])

    def __init__(self, body):
        self.doc = lxml.html.fromstring(body.decode('utf8'))

    def absolute_links(self, url):
        self.doc.make_links_absolute(url)
        base = lxml.html.Element('base', dict(href=url))
        self.doc.head.append(base)

    def compute_score(self):
        txt = self.doc.text_content()
        for punct in string.punctuation:
            txt = txt.replace(punct," ")
        txt = txt.split()
        count = len(filter(lambda x: x in self.words, txt))
        return count / math.log10(2 + len(txt))

    def inject_score(self, score):
        trolldiv = lxml.html.Element('span')
{'fraction_non_alphanumeric': 0.1362126245847176, 'fraction_numerical': 0.006040471156750227, 'mean_word_length': 2.896470588235294, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '454787', 'n_tokens_mistral': 1073, 'n_tokens_neox': 1025, 'n_words': 210}