<commit_before>""" simple """

# python imports
import re
import datetime
import os
from functools import wraps
from unicodedata import normalize
from os import urandom
from base64 import b32encode

# web stuff and markdown imports
import markdown
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug.security import check_password_hash
from flask import render_template, request, Flask, flash, redirect, url_for, \
    abort, jsonify, Response, make_response
from werkzeug.contrib.cache import FileSystemCache, NullCache
from werkzeug.utils import secure_filename
import json
from flask import send_from_directory

try:
    import pygments
    from pygments.formatters import HtmlFormatter
except ImportError:
    pygments = None

app = Flask(__name__)
app.config.from_object('settings')
app.secret_key = app.config["SECRET_KEY"]

UPLOAD_FOLDER = 'uploads/'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER


db = SQLAlchemy(app)
cache_directory = os.path.dirname(__file__)
try:
    cache = FileSystemCache(os.path.join(cache_directory, "cache"))
except Exception, e:
    print "Could not create cache folder, caching will be disabled."
    print "Error: %s" % e
    cache = NullCache()

_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')

extensions = ['fenced_code', 'toc']
if pygments is not None:
    extensions.append('codehilite')

MARKDOWN_PARSER = markdown.Markdown(extensions=extensions,
                                    output_format="html5")


class Post(db.Model):
    def __init__(self, title=None, created_at=None):
        if title:
            self.title = title
            self.slug = slugify(title)
        if created_at:
            self.created_at = created_at
            self.updated_at = created_at

    __tablename__ = "posts"
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String())
    slug = db.Column(db.String(), unique=True)
    text = db.Column(db.String(), default="")
    draft = db.Column(db.Boolean(), index=True, default=True)
    views = db.Column(db.Integer(), default=0)
    created_at = db.Column(db.DateTime, index=True)
    updated_at = db.Column(db.DateTime)

    def render_content(self):
        _cached = cache.get("post_%s"%self.id)
        if _cached is not None:
            return _cached
        text = MARKDOWN_PARSER.convert(self.text)
        cache.set("post_%s"%self.id, text)
        return text

    def set_content(self, content):
        cache.delete("post_%s"%self.id)
        self.text = content

    def get_content(self): return self.text


try:
    db.create_all()
except Exception:
    pass


def is_admin():
    auth = request.authorization
    if not auth or not (auth.username == app.config["ADMIN_USERNAME"]
                        and check_password_hash(app.config["ADMIN_PASSWORD"], 
                                                auth.password)):
        return False
    return True


def requires_authentication(func):
    """ function decorator for handling authentication """
    @wraps(func)
    def _auth_decorator(*args, **kwargs):
        """ does the wrapping """
        if not is_admin():
            return Response("Could not authenticate you", 
                            401, 
                            {"WWW-Authenticate": 'Basic realm="Login Required"'})
        return func(*args, **kwargs)

    return _auth_decorator


@app.route("/")
def index():
    """ Index Page. Here is where the magic starts """
    page = request.args.get("page", 0, type=int)
    posts_master = db.session.query(Post)\
        .filter_by(draft=False).order_by(Post.created_at.desc())
    
    posts_count = posts_master.count()

    posts = posts_master\
        .limit(app.config["POSTS_PER_PAGE"])\
        .offset(page * app.config["POSTS_PER_PAGE"]) .all()

    # Sorry for the verbose names, but this seemed like a sensible
    # thing to do.
    last_possible_post_on_page = page * app.config["POSTS_PER_PAGE"] + \
                                 app.config["POSTS_PER_PAGE"]<commit_msg>Cast POSTS_PER_PAGE to integer to prevent 500 error on index page.<commit_after>""" simple """

# python imports
import re
import datetime
import os
from functools import wraps
from unicodedata import normalize
from os import urandom
from base64 import b32encode

# web stuff and markdown imports
import markdown
from flask.ext.sqlalchemy import SQLAlchemy
from werkzeug.security import check_password_hash
from flask import render_template, request, Flask, flash, redirect, url_for, \
    abort, jsonify, Response, make_response
from werkzeug.contrib.cache import FileSystemCache, NullCache
from werkzeug.utils import secure_filename
import json
from flask import send_from_directory

try:
    import pygments
    from pygments.formatters import HtmlFormatter
except ImportError:
    pygments = None

app = Flask(__name__)
app.config.from_object('settings')
app.secret_key = app.config["SECRET_KEY"]

UPLOAD_FOLDER = 'uploads/'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER


db = SQLAlchemy(app)
cache_directory = os.path.dirname(__file__)
try:
    cache = FileSystemCache(os.path.join(cache_directory, "cache"))
except Exception, e:
    print "Could not create cache folder, caching will be disabled."
    print "Error: %s" % e
    cache = NullCache()

_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+')

extensions = ['fenced_code', 'toc']
if pygments is not None:
    extensions.append('codehilite')

MARKDOWN_PARSER = markdown.Markdown(extensions=extensions,
                                    output_format="html5")


class Post(db.Model):
    def __init__(self, title=None, created_at=None):
        if title:
            self.title = title
            self.slug = slugify(title)
        if created_at:
            self.created_at = created_at
            self.updated_at = created_at

    __tablename__ = "posts"
    id = db.Column(db.Integer, primary_key=True)
    title = db.Column(db.String())
    slug = db.Column(db.String(), unique=True)
    text = db.Column(db.String(), default="")
    draft = db.Column(db.Boolean(), index=True, default=True)
    views = db.Column(db.Integer(), default=0)
    created_at = db.Column(db.DateTime, index=True)
    updated_at = db.Column(db.DateTime)

    def render_content(self):
        _cached = cache.get("post_%s"%self.id)
        if _cached is not None:
            return _cached
        text = MARKDOWN_PARSER.convert(self.text)
        cache.set("post_%s"%self.id, text)
        return text

    def set_content(self, content):
        cache.delete("post_%s"%self.id)
        self.text = content

    def get_content(self): return self.text


try:
    db.create_all()
except Exception:
    pass


def is_admin():
    auth = request.authorization
    if not auth or not (auth.username == app.config["ADMIN_USERNAME"]
                        and check_password_hash(app.config["ADMIN_PASSWORD"], 
                                                auth.password)):
        return False
    return True


def requires_authentication(func):
    """ function decorator for handling authentication """
    @wraps(func)
    def _auth_decorator(*args, **kwargs):
        """ does the wrapping """
        if not is_admin():
            return Response("Could not authenticate you", 
                            401, 
                            {"WWW-Authenticate": 'Basic realm="Login Required"'})
        return func(*args, **kwargs)

    return _auth_decorator


@app.route("/")
def index():
    """ Index Page. Here is where the magic starts """
    page = request.args.get("page", 0, type=int)
    posts_master = db.session.query(Post)\
        .filter_by(draft=False).order_by(Post.created_at.desc())
    
    posts_count = posts_master.count()

    posts = posts_master\
        .limit(app.config["POSTS_PER_PAGE"])\
        .offset(page * int(app.config["POSTS_PER_PAGE"])) .all()

    # Sorry for the verbose names, but this seemed like a sensible
    # thing to do.
    last_possible_post_on_page = page * app.config["POSTS_PER_PAGE"] + \
                                 app.config["POSTS_PER_PAGE"]
{'fraction_non_alphanumeric': 0.11825473991249393, 'fraction_numerical': 0.0027953330092367525, 'mean_word_length': 2.9297994269340975, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6358865', 'n_tokens_mistral': 2571, 'n_tokens_neox': 2504, 'n_words': 621}