<commit_before>"""Main file"""
from flask import Flask, redirect, request, Response
app = Flask(__name__)
import json
import sys
from jinja2 import FileSystemLoader
from jinja2.environment import Environment

env = Environment()
env.loader = FileSystemLoader('.')


def printerr(msg):
	sys.stderr.write(msg + "\n")

# Load settings
try:
	settings_file = open("components/settings.json")
	settings = json.load(settings_file)
except Exception, e:
	printerr("ERROR: Can't load components/settings.json")
	exit(1)

# Load modules
modules = []

for mod in settings["modules"]:
	try:
		m = reload(__import__(mod))
	except:
		printerr("Can't import module '%s'. Aborting." % mod)
		exit(2)
	try:
		c = getattr(m, mod)
		modules.append(c(settings, app))
	except AttributeError:
		app.logger.warning("No operations!")
cachedDocuments = {}
print modules


@app.route("/")
def root_url():
	return redirect(settings["root"], code=302)


@app.route("/<path:path>", methods=['POST', 'GET', 'HEAD', 'PUT', 'DELETE'])
def catch_all(path):
	mime = request.accept_mimetypes.best
	localUri = "%s%s" % (settings["ns"]["local"], path)
	# Check if URI is mirrored, otherwise originUri == localUri
	originUri = localUri
	if settings["mirrored"] is True:
		originUri = "%s%s" % (settings["ns"]["origin"], path)
	localUri = localUri.replace("/cp/", "/")
	originUri = originUri.replace("/cp/", "/")
	# Store .html, .ttl, .json URLs that are not present in triple store.
	if localUri in cachedDocuments.keys():
		originUri = cachedDocuments[localUri]["originUri"]
	c = ""
	r = {"originUri": originUri, "localUri": localUri, "mimetype": mime, "request": request}
	for module in modules:
		response = module.test(r)
		response["request"] = request
		if response["accepted"] is True:
			if localUri != response["url"]:
				# cachedDocuments[response["url"]] = response
				# cachedDocuments[response["url"]]["localUri"] = localUri
				# cachedDocuments[response["url"]]["originUri"] = originUri
				# cachedDocuments[response["url"]]["mime"] = mime<commit_msg>Added CORS, based on http://flask.pocoo.org/snippets/56/<commit_after>"""Main file"""
from flask import Flask, redirect, request, Response, make_response, current_app
app = Flask(__name__)
import json
import sys
from functools import update_wrapper
from datetime import timedelta
from jinja2 import FileSystemLoader
from jinja2.environment import Environment

env = Environment()
env.loader = FileSystemLoader('.')



def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, basestring):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, basestring):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = current_app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and request.method == 'OPTIONS':
                resp = current_app.make_default_options_response()
            else:
                resp = make_response(f(*args, **kwargs))
            if not attach_to_all and request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

def printerr(msg):
	sys.stderr.write(msg + "\n")

# Load settings
try:
	settings_file = open("components/settings.json")
	settings = json.load(settings_file)
except Exception, e:
	printerr("ERROR: Can't load components/settings.json")
	exit(1)

# Load modules
modules = []

for mod in settings["modules"]:
	try:
		m = reload(__import__(mod))
	except:
		printerr("Can't import module '%s'. Aborting." % mod)
		exit(2)
	try:
		c = getattr(m, mod)
		modules.append(c(settings, app))
	except AttributeError:
		app.logger.warning("No operations!")
cachedDocuments = {}
print modules


@app.route("/")
@crossdomain(origin='*')
def root_url():
	return redirect(settings["root"], code=302)


@app.route("/<path:path>", methods=['POST', 'GET', 'HEAD', 'PUT', 'DELETE'])
@crossdomain(origin='*')
def catch_all(path):
	mime = request.accept_mimetypes.best
	localUri = "%s%s" % (settings["ns"]["local"], path)
	# Check if URI is mirrored, otherwise originUri == localUri
	originUri = localUri
	if settings["mirrored"] is True:
		originUri = "%s%s" % (settings["ns"]["origin"], path)
	localUri = localUri.replace("/cp/", "/")
	originUri = originUri.replace("/cp/", "/")
	# Store .html, .ttl, .json URLs that are not present in triple store.
	if localUri in cachedDocuments.keys():
		originUri = cachedDocuments[localUri]["originUri"]
	c = ""
	r = {"originUri": originUri, "localUri": localUri, "mimetype": mime, "request": request}
	for module in modules:
		response = module.test(r)
		response["request"] = request
		if response["accepted"] is True:
			if localUri != response["url"]:
				# cachedDocuments[response["url"]] = response
				# cachedDocuments[response["url"]]["localUri"] = localUri
				# cachedDocuments[response["url"]]["originUri"] = originUri
				# cachedDocuments[response["url"]]["mime"] = mime
{'fraction_non_alphanumeric': 0.14769975786924938, 'fraction_numerical': 0.0036319612590799033, 'mean_word_length': 4.046247818499127, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7077885', 'n_tokens_mistral': 1888, 'n_tokens_neox': 1778, 'n_words': 460}