<commit_before>#!/usr/bin/env python

"""
:copyright: (c) 2014-2015 by <NAME>
:license: MIT, see LICENSE for more details.

A Flask web service to handle the dynamic bits of
the indie-stats.com site.
"""

import os, sys
import json
import time
import uuid
import urllib
import logging
import datetime

import ninka
import redis
import requests

from bearlib.config import Config
from bearlib.tools import baseDomain
from urlparse import urlparse, ParseResult
from flask import Flask, request, redirect, render_template, session, flash, jsonify

from flask.ext.wtf import Form
from wtforms import TextField, HiddenField, BooleanField
from wtforms.validators import Required
from flask_restful import reqparse, abort, Resource, Api

sys.path.append(os.path.dirname(__file__))

from domains import Domain, Domains


class LoginForm(Form):
    me           = TextField('me', validators = [ Required() ])
    client_id    = HiddenField('client_id')
    redirect_uri = HiddenField('redirect_uri')
    from_uri     = HiddenField('from_uri')

class DomainForm(Form):
    excluded  = BooleanField('excluded')
    client_id = HiddenField('client_id')

class DomainNotFoundForm(Form):
    client_id = HiddenField('client_id')

class IndexForm(Form):
    client_id = HiddenField('client_id')

# check for uwsgi, use PWD if present or getcwd() if not
_uwsgi = __name__.startswith('uwsgi')
if _uwsgi:
    _ourPath    = os.getenv('PWD', None)
    _configFile = '/etc/indie-stats.cfg'
else:
    _ourPath    = os.getcwd()
    _configFile = os.path.join(_ourPath, 'indie-stats.cfg')

# check for uwsgi, use PWD if present or getcwd() if not
_uwsgi = __name__.startswith('uwsgi')
if _uwsgi:
    _ourPath    = os.path.dirname(__name__.replace('uwsgi_file_', '').replace('_', '/'))
    _configFile = '/etc/indie-stats.cfg'
    if not os.path.exists('/etc/indie-stats.cfg'):
        _configFile = os.path.join(_ourPath, 'indie-stats.cfg')
else:
    _ourPath    = os.getcwd()
    _configFile = os.path.join(_ourPath, 'indie-stats.cfg')

app = Flask(__name__)
app.config['SECRET_KEY'] = 'foo'  # replaced downstream
api = Api(app)
cfg = None
db  = None

parser = reqparse.RequestParser()
parser.add_argument('domain')

class DomainList(Resource):
    def get(self):
        app.logger.info('apiDomainList')
        l = db.get('cache-domain-list')
        if l is None:
            domainList = Domains(cfg.dataPath, cfg.domainPath, cfg.domains)
            l = domainList.keys()
            db.set('cache-domain-list', json.dumps(l))
        else:
            l = json.loads(l)
        return l

    def post(self):
        args = parser.parse_args()
        domain = args['domain']
        found = False
        domainList = os.path.join(cfg['domainPath'], 'domain_list.txt')
        if os.path.exists(domainList):
            with open(domainList, 'r') as h:
                for line in h.readlines():
                    d = line.strip().encode('utf8')
                    if d == domain:
                        found = True
        if not found:
            app.logger.info('%s not found in domain list' % domain)
            with open(domainList, 'a+') as h:
                h.write('%s\n' % domain)
            d = Domain(domain, cfg.domainPath)
            d.store()
            db.delete('cache-domain-list')
            return { "domain": domain, "result": "Domain has been added to the tracking list" }, 201
        else:
            return { "domain": domain, "result": "Domain is already being tracked" }, 200
<commit_msg>remove POST for domains endpoint as I realized it had no auth check<commit_after>#!/usr/bin/env python

"""
:copyright: (c) 2014-2015 by <NAME>
:license: MIT, see LICENSE for more details.

A Flask web service to handle the dynamic bits of
the indie-stats.com site.
"""

import os, sys
import json
import time
import uuid
import urllib
import logging
import datetime

import ninka
import redis
import requests

from bearlib.config import Config
from bearlib.tools import baseDomain
from urlparse import urlparse, ParseResult
from flask import Flask, request, redirect, render_template, session, flash, jsonify

from flask.ext.wtf import Form
from wtforms import TextField, HiddenField, BooleanField
from wtforms.validators import Required
from flask_restful import reqparse, abort, Resource, Api

sys.path.append(os.path.dirname(__file__))

from domains import Domain, Domains


class LoginForm(Form):
    me           = TextField('me', validators = [ Required() ])
    client_id    = HiddenField('client_id')
    redirect_uri = HiddenField('redirect_uri')
    from_uri     = HiddenField('from_uri')

class DomainForm(Form):
    excluded  = BooleanField('excluded')
    client_id = HiddenField('client_id')

class DomainNotFoundForm(Form):
    client_id = HiddenField('client_id')

class IndexForm(Form):
    client_id = HiddenField('client_id')

# check for uwsgi, use PWD if present or getcwd() if not
_uwsgi = __name__.startswith('uwsgi')
if _uwsgi:
    _ourPath    = os.getenv('PWD', None)
    _configFile = '/etc/indie-stats.cfg'
else:
    _ourPath    = os.getcwd()
    _configFile = os.path.join(_ourPath, 'indie-stats.cfg')

# check for uwsgi, use PWD if present or getcwd() if not
_uwsgi = __name__.startswith('uwsgi')
if _uwsgi:
    _ourPath    = os.path.dirname(__name__.replace('uwsgi_file_', '').replace('_', '/'))
    _configFile = '/etc/indie-stats.cfg'
    if not os.path.exists('/etc/indie-stats.cfg'):
        _configFile = os.path.join(_ourPath, 'indie-stats.cfg')
else:
    _ourPath    = os.getcwd()
    _configFile = os.path.join(_ourPath, 'indie-stats.cfg')

app = Flask(__name__)
app.config['SECRET_KEY'] = 'foo'  # replaced downstream
api = Api(app)
cfg = None
db  = None

parser = reqparse.RequestParser()
parser.add_argument('domain')

class DomainList(Resource):
    def get(self):
        app.logger.info('apiDomainList')
        l = db.get('cache-domain-list')
        if l is None:
            domainList = Domains(cfg.dataPath, cfg.domainPath, cfg.domains)
            l = domainList.keys()
            db.set('cache-domain-list', json.dumps(l))
        else:
            l = json.loads(l)
        return l

    # def post(self):
    #     args = parser.parse_args()
    #     domain = args['domain']
    #     found = False
    #     domainList = os.path.join(cfg['domainPath'], 'domain_list.txt')
    #     if os.path.exists(domainList):
    #         with open(domainList, 'r') as h:
    #             for line in h.readlines():
    #                 d = line.strip().encode('utf8')
    #                 if d == domain:
    #                     found = True
    #     if not found:
    #         app.logger.info('%s not found in domain list' % domain)
    #         with open(domainList, 'a+') as h:
    #             h.write('%s\n' % domain)
    #         d = Domain(domain, cfg.domainPath)
    #         d.store()
    #         db.delete('cache-domain-list')
    #         return { "domain": domain, "result": "Domain has been added to the tracking list" }, 201
    #     else:
    #         return { "domain": domain, "result": "Domain is already being tracked" }, 200

{'fraction_non_alphanumeric': 0.12152289969092442, 'fraction_numerical': 0.004214667041303737, 'mean_word_length': 2.852272727272727, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4688705', 'n_tokens_mistral': 2323, 'n_tokens_neox': 2247, 'n_words': 604}