<commit_before>from __future__ import with_statement
import hashlib
import logging
import os.path
import sqlite3
import sys
import util

import tornado.httpserver
import tornado.ioloop
import tornado.web

logging.basicConfig(stream=sys.stdout, level=logging.INFO)
log = logging.getLogger('firefly_ui_server')


def shorten(state, db_conn):
    """Returns the b58encoded id of the provided full firefly
    state description. Creates a new entry for state if one
    doesn't already exist in the db.
    """
    state = unicode(state, 'utf_8')
    state_hash = buffer(hashlib.sha1(state.encode('utf_8')).digest())
    row = db_conn.execute("select id from states where state_hash=?", (state_hash,)).fetchone()
    stateid = row[0] if row else db_conn.execute("insert into states(state, state_hash) values (?, ?)", (state, state_hash)).lastrowid
    return util.b58encode(stateid)


class IndexHandler(tornado.web.RequestHandler):
    """Serves the basic dashboard page"""

    def get(self):
        """Handle the default case of just hitting the index."""
        embed = False
        if self.get_argument('embed', '') == 'true':
            embed = True
        env = {
            'url_path_prefix': self.application.settings['url_path_prefix'],
            'data_servers': self.application.settings['data_servers'],
            'embedded': embed}

        self.render("templates/index.html", **env)

class TokenHandler(tornado.web.RequestHandler):
    """Generate tokens"""

    def get(self):
        self.set_header("Content-Type", "text/plain")
        self.write(util.generate_access_token(self.application.settings['secret_key']))

class ShortenHandler(tornado.web.RequestHandler):
    """Stores state data and returns an ID for later retrieval"""

    def post(self):
        conn = self.application.settings['db_connection']
        stateid = shorten(self.request.body, conn)
        self.set_header("Content-Type", "text/plain")
        self.write(stateid)

class ExpandHandler(tornado.web.RequestHandler):
    """Retrieves state data given an ID"""

    def get(self, b58id):
        conn = self.application.settings['db_connection']
        try:
            stateid = util.b58decode(b58id)
        except:
            raise tornado.web.HTTPError(404)
        row = conn.execute("select state from states where id=?", (stateid,)).fetchone()

        if row:
            self.set_header("Content-Type", "application/json; charset=UTF-8")
            self.write(row[0])
        else:
            raise tornado.web.HTTPError(404)

class RedirectHandler(tornado.web.RequestHandler):
    """Redirects to graph display given a full state description.
    Useful for embedding graphs to show programatically supplied values.
    """

    def get(self, serialized):
        fragment = shorten(serialized, self.application.settings['db_connection'])
        url = self.application.settings['url_path_prefix']
        if self.get_argument('embed', '') == 'true':
            url += "?embed=true"
        self.redirect(url + '#!' + fragment)

class NameHandler(tornado.web.RequestHandler):
    """Handles storing and retrieving named dashboards"""

    def post(self, name):
        conn = self.application.settings['db_connection']
        stateid = util.b58decode(unicode(self.request.body, 'utf_8'))
        conn.execute("insert or replace into names (name, stateid) values (?, ?)", (name, stateid))

        # TODO: make the content-type json
        self.set_header("Content-Type", "text/plain")
        self.write('{success: true}')

    def get(self, name):
        conn = self.application.settings['db_connection']
        state = conn.execute("select state from states where id = (select stateid from names where name = ?)", (name,)).fetchone()

        if state:
            self.set_header("Content-Type", "application/json; charset=UTF-8")
            self.write(state[0])
        else:
            self.set_header("Content-Type", "text/plain")
            self.write('{success: false}')

def initialize_ui_server(config, secret_key=None, ioloop=None):
    if not ioloop:
        ioloop = tornado.ioloop.IOLoop.instance()

    # connect to the database
    conn = sqlite3.connect(config['db_file'], isolation_level=None)
    conn.execute("create table if not exists states (id integer primary key autoincrement, state text not null, state_hash blob not null)")
    conn.execute("create index if not exists hash_idx on states(state_hash)")
    conn.execute("create table if not exists names (id integer primary key autoincrement, name text not null, stateid text not null)")
    conn.execute("create unique index if not exists name_idx on names (name)")

    config["static_path"] = os.path.join(os.path.join(*os.path.split(__file__)[:-1]), 'static')
    config["db_connection"] = conn
    config["static_url_prefix"] = os.path.join(config["url_path_prefix"], "static") + "/"
    config["secret_key"] = secret_key

    # init the application instance
    application = tornado.web.Application([
        (r"/", IndexHandler),
        (r"/token", TokenHandler),
        (r"/shorten", ShortenHandler),
        (r"/expand/(.*)", ExpandHandler),
        (r"/redirect/(.*)", RedirectHandler),
        (r"/named/(.*)", NameHandler),
        (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": config['static_path']}),
    ], **config)

    # start the main server
    http_server = tornado.httpserver.HTTPServer(application, io_loop=ioloop)
    http_server.listen(config["port"])

    log.info('Firefly UI server started on port %d' % config["port"])<commit_msg>Move to using redirects for names<commit_after>from __future__ import with_statement
import hashlib
import logging
import os.path
import sqlite3
import sys
import util

import tornado.httpserver
import tornado.ioloop
import tornado.web

logging.basicConfig(stream=sys.stdout, level=logging.INFO)
log = logging.getLogger('firefly_ui_server')


def shorten(state, db_conn):
    """Returns the b58encoded id of the provided full firefly
    state description. Creates a new entry for state if one
    doesn't already exist in the db.
    """
    state = unicode(state, 'utf_8')
    state_hash = buffer(hashlib.sha1(state.encode('utf_8')).digest())
    row = db_conn.execute("select id from states where state_hash=?", (state_hash,)).fetchone()
    stateid = row[0] if row else db_conn.execute("insert into states(state, state_hash) values (?, ?)", (state, state_hash)).lastrowid
    return util.b58encode(stateid)


class IndexHandler(tornado.web.RequestHandler):
    """Serves the basic dashboard page"""

    def get(self):
        """Handle the default case of just hitting the index."""
        embed = False
        if self.get_argument('embed', '') == 'true':
            embed = True
        env = {
            'url_path_prefix': self.application.settings['url_path_prefix'],
            'data_servers': self.application.settings['data_servers'],
            'embedded': embed}

        self.render("templates/index.html", **env)

class TokenHandler(tornado.web.RequestHandler):
    """Generate tokens"""

    def get(self):
        self.set_header("Content-Type", "text/plain")
        self.write(util.generate_access_token(self.application.settings['secret_key']))

class ShortenHandler(tornado.web.RequestHandler):
    """Stores state data and returns an ID for later retrieval"""

    def post(self):
        conn = self.application.settings['db_connection']
        stateid = shorten(self.request.body, conn)
        self.set_header("Content-Type", "text/plain")
        self.write(stateid)

class ExpandHandler(tornado.web.RequestHandler):
    """Retrieves state data given an ID"""

    def get(self, b58id):
        conn = self.application.settings['db_connection']
        try:
            stateid = util.b58decode(b58id)
        except:
            raise tornado.web.HTTPError(404)
        row = conn.execute("select state from states where id=?", (stateid,)).fetchone()

        if row:
            self.set_header("Content-Type", "application/json; charset=UTF-8")
            self.write(row[0])
        else:
            raise tornado.web.HTTPError(404)

class RedirectHandler(tornado.web.RequestHandler):
    """Redirects to graph display given a full state description.
    Useful for embedding graphs to show programatically supplied values.
    """

    def get(self, serialized):
        fragment = shorten(serialized, self.application.settings['db_connection'])
        url = self.application.settings['url_path_prefix']
        if self.get_argument('embed', '') == 'true':
            url += "?embed=true"
        self.redirect(url + '#!' + fragment)

class NameHandler(tornado.web.RequestHandler):
    """Handles storing and retrieving named dashboards"""

    def post(self, name):
        conn = self.application.settings['db_connection']
        stateid = util.b58decode(unicode(self.request.body, 'utf_8'))
        conn.execute("insert or replace into names (name, stateid) values (?, ?)", (name, stateid))

        self.set_header("Content-Type", "application/json")
        self.write('{success: true}')

    def get(self, name):
        conn = self.application.settings['db_connection']
        stateid = conn.execute("select stateid from names where name = ?", (name,)).fetchone()

        if stateid:
            b58id = util.b58encode(stateid[0])
            self.redirect('/?incoming=%s#!%s' % (name, b58id))
        else:
            self.redirect('/')

def initialize_ui_server(config, secret_key=None, ioloop=None):
    if not ioloop:
        ioloop = tornado.ioloop.IOLoop.instance()

    # connect to the database
    conn = sqlite3.connect(config['db_file'], isolation_level=None)
    conn.execute("create table if not exists states (id integer primary key autoincrement, state text not null, state_hash blob not null)")
    conn.execute("create index if not exists hash_idx on states(state_hash)")
    conn.execute("create table if not exists names (id integer primary key autoincrement, name text not null, stateid integer not null)")
    conn.execute("create unique index if not exists name_idx on names (name)")

    config["static_path"] = os.path.join(os.path.join(*os.path.split(__file__)[:-1]), 'static')
    config["db_connection"] = conn
    config["static_url_prefix"] = os.path.join(config["url_path_prefix"], "static") + "/"
    config["secret_key"] = secret_key

    # init the application instance
    application = tornado.web.Application([
        (r"/", IndexHandler),
        (r"/token", TokenHandler),
        (r"/shorten", ShortenHandler),
        (r"/expand/(.*)", ExpandHandler),
        (r"/redirect/(.*)", RedirectHandler),
        (r"/named/(.*)", NameHandler),
        (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": config['static_path']}),
    ], **config)

    # start the main server
    http_server = tornado.httpserver.HTTPServer(application, io_loop=ioloop)
    http_server.listen(config["port"])

    log.info('Firefly UI server started on port %d' % config["port"])
{'fraction_non_alphanumeric': 0.13329706202393907, 'fraction_numerical': 0.005894087776568734, 'mean_word_length': 3.390525477707006, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3583544', 'n_tokens_mistral': 3284, 'n_tokens_neox': 3163, 'n_words': 797}