<commit_before>"""
This file, vislab/ui/app.py, is the updated data and results viewing ui.
The older code for the same tasks is in vislab/app.py
We start with the Pinterest data here.

TODO
- Display the number of results on the page.
- Switch to getting data from a mongo database instead of loading df.
"""
import flask
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import vislab.datasets
import pymongo

app = flask.Flask(__name__)
style_names = vislab.datasets.flickr.underscored_style_names

@app.route('/')
def index():
     return flask.redirect(flask.url_for(
        'data', dataset_name='pinterest', style_name='style_Pastel',
        pins_per_user=5, page=1)
    )


@app.route('/data/<dataset_name>/<style_name>/<int:pins_per_user>/<int:page>')
def data(dataset_name, style_name, pins_per_user, page):
    client = pymongo.MongoClient('localhost', 27017)
    if dataset_name == 'flickr':
        db = client['ui_dfs']['flickr_df']
    elif dataset_name == 'pinterest':
        db = client['ui_dfs']['pins_df']
    else:
        raise Exception("Unknown dataset")

    results_per_page = 7 * 20

    # Filter on style.
    if style_name != 'all':
        cursor = db.find({style_name: True})
    else:
        cursor = db.find()

    # Filter on pins per user
    # TODO: bring this back
    # df = df.groupby('username').head(pins_per_user)
    # df.set_index(df.index.get_level_values(1), inplace=True)

    # Paginate
    num_results = cursor.count()
    num_pages = num_results / results_per_page
    start = page * results_per_page
    end = min(num_results, start + results_per_page)
    results = cursor[start:end]

    # Set filter options
    select_options = [
        ('dataset', ['flickr', 'pinterest'], dataset_name),
        ('style', ['all'] + style_names, style_name),
        ('pins_per_user', [1, 5, 100], pins_per_user),
        ('page', range(1, num_pages), page),<commit_msg>added code to populate db if needed<commit_after>"""
This file, vislab/ui/app.py, is the updated data and results viewing ui.
The older code for the same tasks is in vislab/app.py
We start with the Pinterest data here.

TODO
- Display the number of results on the page.
- Switch to getting data from a mongo database instead of loading df.
"""
import flask
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
import vislab.datasets
import pymongo
import numpy

app = flask.Flask(__name__)
style_names = vislab.datasets.flickr.underscored_style_names

@app.route('/')
def index():
     return flask.redirect(flask.url_for(
        'data', dataset_name='pinterest', style_name='style_Pastel',
        pins_per_user=5, page=1)
    )
def insert_dfs(db, collection):
    print "Inserting df for {}".format(collection)
    if collection == 'flickr_df':
        df = vislab.datasets.flickr.get_df()
    elif collection == 'pins_df':
        df = vislab.datasets.pinterest.get_pins_80k_df()
    for i in range(df.shape[0]):
        d = df.iloc[i].to_dict()
        for k, v in d.iteritems():
            if type(d[k]) is numpy.bool_:
                d[k] = bool(d[k])
        db.insert(d)

@app.route('/data/<dataset_name>/<style_name>/<int:pins_per_user>/<int:page>')
def data(dataset_name, style_name, pins_per_user, page):
    client = pymongo.MongoClient('localhost', 27017)
    if dataset_name == 'flickr':
        db = client['ui_dfs']['flickr_df']
        if db.count() == 0:
            insert_dfs(db, 'flickr_df')
    elif dataset_name == 'pinterest':
        db = client['ui_dfs']['pins_df']
        if db.count() == 0:
            insert_dfs(db, 'pins_df')
    else:
        raise Exception("Unknown dataset")

    results_per_page = 7 * 20

    # Filter on style.
    if style_name != 'all':
        cursor = db.find({style_name: True})
    else:
        cursor = db.find()

    # Filter on pins per user
    # TODO: bring this back
    # df = df.groupby('username').head(pins_per_user)
    # df.set_index(df.index.get_level_values(1), inplace=True)

    # Paginate
    num_results = cursor.count()
    num_pages = num_results / results_per_page
    start = page * results_per_page
    end = min(num_results, start + results_per_page)
    results = cursor[start:end]

    # Set filter options
    select_options = [
        ('dataset', ['flickr', 'pinterest'], dataset_name),
        ('style', ['all'] + style_names, style_name),
        ('pins_per_user', [1, 5, 100], pins_per_user),
        ('page', range(1, num_pages), page),
{'fraction_non_alphanumeric': 0.12573802755302865, 'fraction_numerical': 0.008528318390553248, 'mean_word_length': 3.3110273327049953, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3562425', 'n_tokens_mistral': 1583, 'n_tokens_neox': 1492, 'n_words': 391}