<commit_before>import socket
import database
import flask
import logging
import time
import json

from logging.handlers import SysLogHandler
from datetime import datetime

from flask import Flask
from ConfigParser import ConfigParser
from functools import wraps
from cassandra.cluster import Cluster as CassandraCluster
from cassandra.auth import PlainTextAuthProvider

from util import TIME_BOT_COMPETITION_START
from util import timeline
from util import cursor
from util import make_json_response
from util import not_implemented
from util import temporal
from util import get_time_anchor
from util import get_current_virtual_time
from util import translate_alpha_time_to_virtual_time
from util import translate_virtual_time_to_alpha_time
from util import disabled_beta
from util import nearest_scan
from util import beta_predicate_tweets
from util import beta_predicate_users
from util import we_are_out_of_beta

from formatter import UserFormatter, TweetFormatter, GuessFormatter, EdgeFormatter
from search import Search

from detectionteam import DetectionTeam
from guess import Guess
from guess_user import GuessUser
from tuser import TUser
from tweet import Tweet
from scan import Scan
from bot import Bot
from tweet_entity import TweetEntity

config = ConfigParser()
config.read('configuration.ini')

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://%s:%s@/pacsocial?host=%s' % (config.get('postgresql', 'username'), config.get('postgresql', 'password'), config.get('postgresql', 'socket'))
app.config['SQLALCHEMY_ECHO'] = False
database.db.init_app(app)

def cassandrafied(f):
    @wraps(f)
    def decorator(*args, **kwargs):
        cluster = CassandraCluster([config.get('cassandra', 'contact')], auth_provider=PlainTextAuthProvider(username=config.get('cassandra', 'username'), password=config.get('cassandra', 'password')), executor_threads=50)
        session = cluster.connect('smisc')
        kwargs['cassandra_cluster'] = session
        try:
            return f(*args, **kwargs)
        finally:
            session.shutdown()
            cluster.shutdown()

    return decorator

def require_passcode(f):
    @wraps(f)
    def decorator(*args, **kwargs):
        if 'Authorization' not in flask.request.headers:
            return flask.make_response('', 401)

        password = flask.request.headers['Authorization'].replace('Bearer ', '')
        team = DetectionTeam.query.filter(DetectionTeam.password == password).first()

        if team is None:
            return flask.make_response('', 403)

        kwargs['team_id'] = team.id

        return f(*args, **kwargs)
            
    return decorator

@app.route('/clock', methods=['GET'], defaults={'vtime': None})
@app.route('/clock/<vtime>', methods=['GET'])
@make_json_response
@temporal
def show_clock(vtime):
    time_str = lambda t: datetime.fromtimestamp(t+(3600*-8)).strftime("%b %d %Y, %I:%M:%S %p PDT")

    virtual = vtime
    anchor = get_time_anchor()
    min_vtime = TIME_BOT_COMPETITION_START

    return json.dumps({
        "now": {
            "alpha": translate_virtual_time_to_alpha_time(vtime),
            "alpha_str":time_str(translate_virtual_time_to_alpha_time(vtime)),
            "virtual": vtime,
            "virtual_str": time_str(vtime)
        },
        "minimum": {
            "alpha": translate_virtual_time_to_alpha_time(min_vtime),
            "alpha_str":time_str(translate_virtual_time_to_alpha_time(min_vtime)),
            "virtual": min_vtime,
            "virtual_str": time_str(min_vtime)
        },
        "anchor": anchor,
        "alpha_str": time_str(anchor)
    })

@app.route('/edges/near/<vtime>/followers/<user_id>', methods=['GET'])
@app.route('/edges/followers/<user_id>', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@timeline
@nearest_scan(Scan.SCAN_TYPE_FOLLOWERS)
@cassandrafied
def timeless_list_followers(cassandra_cluster, vtime, user_id, max_id, since_id, since_count, max_scan_id, min_scan_id):
    user = beta_predicate_users(TUser.query.filter(TUser.user_id == user_id)).first()

    if user is not None:
        id_condition = ""
        ids = []

        wanted_min_id = since_id+1
        wanted_max_id = max_id+1

        if min_scan_id is not None and max_scan_id is not None:
            wanted_min_id = max(since_id+1, min_scan_id)
            wanted_max_id = min(max_id+1, max_scan_id)
        elif min_scan_id is not None and max_scan_id is None:
            wanted_min_id = max(since_id+1, min_scan_id)
        elif min_scan_id is None and max_scan_id is not None:
            wanted_max_id = min(max_id+1, max_scan_id)

        conds = [user_id]

        id_condition = 'id >= %s'
        conds.append(wanted_min_id)

        if wanted_max_id != float('+inf'):
            id_condition += ' and id < %s'
            conds.append(wanted_max_id)

        rows = cassandra_cluster.execute("SELECT id, to_user, from_user, \"timestamp\" FROM tuser_tuser WHERE to_user = %s AND " + id_condition + " ORDER BY id DESC LIMIT " + str(since_count), tuple(conds))
        formatter = EdgeFormatter()
        return json.dumps(formatter.format(rows))
    else:
        return flask.make_response('', 404)

@app.route('/edges/explore/<vtime>/<from_user>/to/<to_user>', methods=['GET'])
@make_json_response
@temporal
@timeline
@nearest_scan(Scan.SCAN_TYPE_FOLLOWERS)
@cassandrafied
def timeless_explore_edges(cassandra_cluster, vtime, from_user, to_user, max_id, since_id, since_count, max_scan_id, min_scan_id):
    to_user = beta_predicate_users(TUser.query.filter(TUser.user_id == to_user)).first()

    if to_user is not None:
        id_condition = ""
        ids = []

        wanted_max_id = max_id+1

        if min_scan_id is not None and max_scan_id is not None:
            wanted_max_id = min(max_id+1, max_scan_id)
        elif min_scan_id is None and max_scan_id is not None:
            wanted_max_id = min(max_id+1, max_scan_id)

        conds = [to_user.user_id, from_user]

        if wanted_max_id != float('+inf'):
            id_condition += 'AND id < %s'
            conds.append(wanted_max_id)

        logging.info(id_condition)
        logging.info(conds)

        rows = cassandra_cluster.execute("SELECT id, to_user, from_user, \"timestamp\" FROM tuser_tuser_inspect WHERE to_user = %s AND from_user = %s " + id_condition + " ORDER BY id DESC LIMIT " + str(since_count), tuple(conds))
        formatter = EdgeFormatter()
        return json.dumps(formatter.format(rows))
    else:
        return flask.make_response('', 404)

@app.route('/user/near/<vtime>', methods=['GET'])
@app.route('/user', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@cursor
@nearest_scan(Scan.SCAN_TYPE_USER)
def list_users(vtime, cursor_size, offset, max_scan_id, min_scan_id):
    users = beta_predicate_users(TUser.query.filter(
        TUser.id >= min_scan_id,
        TUser.id <= max_scan_id
    )).order_by(TUser.id.desc()).limit(cursor_size).offset(offset).all()
    formatter = UserFormatter()
    return json.dumps(formatter.format(users))

@app.route('/user/near/<vtime>/<user_id>', methods=['GET'])
@app.route('/user/<user_id>', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@nearest_scan(Scan.SCAN_TYPE_USER)
def show_user(vtime, user_id, max_scan_id, min_scan_id):<commit_msg>Revert "Fix edges explore should return all prior results"<commit_after>import socket
import database
import flask
import logging
import time
import json

from logging.handlers import SysLogHandler
from datetime import datetime

from flask import Flask
from ConfigParser import ConfigParser
from functools import wraps
from cassandra.cluster import Cluster as CassandraCluster
from cassandra.auth import PlainTextAuthProvider

from util import TIME_BOT_COMPETITION_START
from util import timeline
from util import cursor
from util import make_json_response
from util import not_implemented
from util import temporal
from util import get_time_anchor
from util import get_current_virtual_time
from util import translate_alpha_time_to_virtual_time
from util import translate_virtual_time_to_alpha_time
from util import disabled_beta
from util import nearest_scan
from util import beta_predicate_tweets
from util import beta_predicate_users
from util import we_are_out_of_beta

from formatter import UserFormatter, TweetFormatter, GuessFormatter, EdgeFormatter
from search import Search

from detectionteam import DetectionTeam
from guess import Guess
from guess_user import GuessUser
from tuser import TUser
from tweet import Tweet
from scan import Scan
from bot import Bot
from tweet_entity import TweetEntity

config = ConfigParser()
config.read('configuration.ini')

app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://%s:%s@/pacsocial?host=%s' % (config.get('postgresql', 'username'), config.get('postgresql', 'password'), config.get('postgresql', 'socket'))
app.config['SQLALCHEMY_ECHO'] = False
database.db.init_app(app)

def cassandrafied(f):
    @wraps(f)
    def decorator(*args, **kwargs):
        cluster = CassandraCluster([config.get('cassandra', 'contact')], auth_provider=PlainTextAuthProvider(username=config.get('cassandra', 'username'), password=config.get('cassandra', 'password')), executor_threads=50)
        session = cluster.connect('smisc')
        kwargs['cassandra_cluster'] = session
        try:
            return f(*args, **kwargs)
        finally:
            session.shutdown()
            cluster.shutdown()

    return decorator

def require_passcode(f):
    @wraps(f)
    def decorator(*args, **kwargs):
        if 'Authorization' not in flask.request.headers:
            return flask.make_response('', 401)

        password = flask.request.headers['Authorization'].replace('Bearer ', '')
        team = DetectionTeam.query.filter(DetectionTeam.password == password).first()

        if team is None:
            return flask.make_response('', 403)

        kwargs['team_id'] = team.id

        return f(*args, **kwargs)
            
    return decorator

@app.route('/clock', methods=['GET'], defaults={'vtime': None})
@app.route('/clock/<vtime>', methods=['GET'])
@make_json_response
@temporal
def show_clock(vtime):
    time_str = lambda t: datetime.fromtimestamp(t+(3600*-8)).strftime("%b %d %Y, %I:%M:%S %p PDT")

    virtual = vtime
    anchor = get_time_anchor()
    min_vtime = TIME_BOT_COMPETITION_START

    return json.dumps({
        "now": {
            "alpha": translate_virtual_time_to_alpha_time(vtime),
            "alpha_str":time_str(translate_virtual_time_to_alpha_time(vtime)),
            "virtual": vtime,
            "virtual_str": time_str(vtime)
        },
        "minimum": {
            "alpha": translate_virtual_time_to_alpha_time(min_vtime),
            "alpha_str":time_str(translate_virtual_time_to_alpha_time(min_vtime)),
            "virtual": min_vtime,
            "virtual_str": time_str(min_vtime)
        },
        "anchor": anchor,
        "alpha_str": time_str(anchor)
    })

@app.route('/edges/near/<vtime>/followers/<user_id>', methods=['GET'])
@app.route('/edges/followers/<user_id>', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@timeline
@nearest_scan(Scan.SCAN_TYPE_FOLLOWERS)
@cassandrafied
def timeless_list_followers(cassandra_cluster, vtime, user_id, max_id, since_id, since_count, max_scan_id, min_scan_id):
    user = beta_predicate_users(TUser.query.filter(TUser.user_id == user_id)).first()

    if user is not None:
        id_condition = ""
        ids = []

        wanted_min_id = since_id+1
        wanted_max_id = max_id+1

        if min_scan_id is not None and max_scan_id is not None:
            wanted_min_id = max(since_id+1, min_scan_id)
            wanted_max_id = min(max_id+1, max_scan_id)
        elif min_scan_id is not None and max_scan_id is None:
            wanted_min_id = max(since_id+1, min_scan_id)
        elif min_scan_id is None and max_scan_id is not None:
            wanted_max_id = min(max_id+1, max_scan_id)

        conds = [user_id]

        id_condition = 'id >= %s'
        conds.append(wanted_min_id)

        if wanted_max_id != float('+inf'):
            id_condition += ' and id < %s'
            conds.append(wanted_max_id)

        rows = cassandra_cluster.execute("SELECT id, to_user, from_user, \"timestamp\" FROM tuser_tuser WHERE to_user = %s AND " + id_condition + " ORDER BY id DESC LIMIT " + str(since_count), tuple(conds))
        formatter = EdgeFormatter()
        return json.dumps(formatter.format(rows))
    else:
        return flask.make_response('', 404)

@app.route('/edges/explore/<vtime>/<from_user>/to/<to_user>', methods=['GET'])
@make_json_response
@temporal
@timeline
@nearest_scan(Scan.SCAN_TYPE_FOLLOWERS)
@cassandrafied
def timeless_explore_edges(cassandra_cluster, vtime, from_user, to_user, max_id, since_id, since_count, max_scan_id, min_scan_id):
    to_user = beta_predicate_users(TUser.query.filter(TUser.user_id == to_user)).first()

    if to_user is not None:
        id_condition = ""
        ids = []

        wanted_min_id = since_id+1
        wanted_max_id = max_id+1

        if min_scan_id is not None and max_scan_id is not None:
            wanted_min_id = max(since_id+1, min_scan_id)
            wanted_max_id = min(max_id+1, max_scan_id)
        elif min_scan_id is not None and max_scan_id is None:
            wanted_min_id = max(since_id+1, min_scan_id)
        elif min_scan_id is None and max_scan_id is not None:
            wanted_max_id = min(max_id+1, max_scan_id)

        conds = [to_user.user_id, from_user]

        id_condition = 'id >= %s'
        conds.append(wanted_min_id)

        if wanted_max_id != float('+inf'):
            id_condition += ' and id < %s'
            conds.append(wanted_max_id)

        logging.info(id_condition)
        logging.info(conds)

        rows = cassandra_cluster.execute("SELECT id, to_user, from_user, \"timestamp\" FROM tuser_tuser_inspect WHERE to_user = %s AND from_user = %s AND " + id_condition + " ORDER BY id DESC LIMIT " + str(since_count), tuple(conds))
        formatter = EdgeFormatter()
        return json.dumps(formatter.format(rows))
    else:
        return flask.make_response('', 404)

@app.route('/user/near/<vtime>', methods=['GET'])
@app.route('/user', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@cursor
@nearest_scan(Scan.SCAN_TYPE_USER)
def list_users(vtime, cursor_size, offset, max_scan_id, min_scan_id):
    users = beta_predicate_users(TUser.query.filter(
        TUser.id >= min_scan_id,
        TUser.id <= max_scan_id
    )).order_by(TUser.id.desc()).limit(cursor_size).offset(offset).all()
    formatter = UserFormatter()
    return json.dumps(formatter.format(users))

@app.route('/user/near/<vtime>/<user_id>', methods=['GET'])
@app.route('/user/<user_id>', methods=['GET'], defaults={'vtime': None})
@make_json_response
@temporal
@nearest_scan(Scan.SCAN_TYPE_USER)
def show_user(vtime, user_id, max_scan_id, min_scan_id):
{'fraction_non_alphanumeric': 0.11344873501997337, 'fraction_numerical': 0.0039280958721704395, 'mean_word_length': 3.8470474346563406, 'pattern_counts': {'":': 24, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339035', 'n_tokens_mistral': 4992, 'n_tokens_neox': 4835, 'n_words': 1073}