<commit_before>from flask import Flask, request,  url_for
import datetime
import reflector
import datetime
import md5, random
import os
import flask
import statsd

from frozone import conf

app = Flask(__name__)


from logging import FileHandler
fh = FileHandler(filename=os.path.join(conf.remote_e2repo, 'e2repo.log'))
app.logger.addHandler(fh)


def getfilehash(moduletxt):
    '''from a html5 string get some sort of hash '''
    h = random.randint(1,1000)
    return h

def callstatsd(dottedcounter):
    ''' '''
    c = statsd.StatsClient(conf.statsd_host, conf.statsd_port)
    c.incr(dottedcounter)
    #todo: really return c and keep elsewhere for efficieny I suspect

def qlog(msg):
    d = datetime.datetime.today().isoformat()
    fo = open('/tmp/e2repo.log','a')
    fo.write('%s %s \n' % (d, msg))
    fo.close()


def asjson(pyobj):
    '''just placeholder '''
    return 'JSON:: %s' % repr(pyobj)

def gettime():
    return datetime.datetime.today().isoformat()
<commit_msg>spelling<commit_after>from flask import Flask, request,  url_for
import datetime
import reflector
import datetime
import md5, random
import os
import flask
import statsd

from frozone import conf

app = Flask(__name__)
REPO = conf.remote_e2repo


from logging import FileHandler
fh = FileHandler(filename=os.path.join(REPO, 'e2repo.log'))
app.logger.addHandler(fh)


def getfilehash(moduletxt):
    '''from a html5 string get some sort of hash '''
    h = random.randint(1,1000)
    return h

def callstatsd(dottedcounter):
    ''' '''
    c = statsd.StatsClient(conf.statsd_host, conf.statsd_port)
    c.incr(dottedcounter)
    #todo: really return c and keep elsewhere for efficieny I suspect

def qlog(msg):
    d = datetime.datetime.today().isoformat()
    fo = open('/tmp/e2repo.log','a')
    fo.write('%s %s \n' % (d, msg))
    fo.close()


def asjson(pyobj):
    '''just placeholder '''
    return 'JSON:: %s' % repr(pyobj)

def gettime():
    return datetime.datetime.today().isoformat()

{'fraction_non_alphanumeric': 0.12644836272040302, 'fraction_numerical': 0.010075566750629723, 'mean_word_length': 4.066326530612245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1151329', 'n_tokens_mistral': 696, 'n_tokens_neox': 663, 'n_words': 184}