<commit_before>import json
import os
import time

from klein import Klein
from twisted.web.static import File
import zmq

from db import Database
from docgetter import get_docs
from ipgetter import get_ip
import msg
from servicegetters import got_cluster, got_slave, got_notebook
import settings

app = Klein()
slave_db = Database()

def zocket_send(**kwargs):
    zcontext = zmq.Context()
    socket = zcontext.socket(zmq.PUSH)
    socket.bind("ipc:///tmp/sock")
    socket.send_json(kwargs)
    
def get_request_par(req,par):
    return req.args.get(bytes(par,'utf-8'), False)[0].decode('utf-8')
    
@app.route('/', branch=True)
def root(request):
    return File("./static/")

@app.route('/api/getdocs')
def all_the_docs(reques):
    return json.dumps({'allthedocs':get_docs()})

@app.route('/api/status')
def status(request):

    result = {}
    
    result['virtual'] = json.loads(os.environ.get('VBOX','false')) == 'true'
        
    if result['virtual']:
        try:
            with open('ip.json', 'r') as ipfile:
                result['ext_ip'] = json.loads(ipfile.read())['ip'].split('.')
            result['realip'] = True
        except:
            result['realip'] = False
    else:
        result['realip'] = False
              
    with open('id.json', 'r') as idfile:
        result['appid'] = json.loads(idfile.read())['id']
        
    ip = get_ip()
    if ip:
        result['network'] = True
        result['ip'] = ip.split('.')
        result['slave_ip'] = got_slave(ip)
        result['master_owner'] = got_cluster(ip)
        result['pysparknotebook'] = got_notebook()
        if result['master_owner'] or result['slave_ip']:
            result['status'] = 'active'
        else:
            result['status'] = 'dormant'
    else:
        result['status'] = 'error'
            
    return json.dumps(result)
    
def valid_ip(ip): 
    try:
        chunks = ip.split('.')
        return len(chunks) == 4 and all([c.isnumeric and 0 <= int(c) <= 255 for c in chunks])
    except:
        return False

@app.route('/api/testip')
def test_ip(request):
    result = {'okay':False}
    try:
        external_ip = get_request_par(request,'ip')
    except:
        external_ip = False
    with open('id.json', 'r') as idfile:
        true_id = json.loads(idfile.read())['id']
    try:    
        app_id = get_request_par(request,'id')
    except:
        add_id = False
    if app_id == true_id:
        result = {'okay':True}
        with open('ip.json', 'w') as ipfile:
            ipfile.write(json.dumps({'ip':external_ip}))
    callback = get_request_par(request,'callback')
    return callback+'('+json.dumps(result)+');'    
    
@app.route('/api/startcluster')
def start_master(request):
    result = {'okay':False}
    ip = get_ip()
    if not got_cluster(ip):
        if not got_slave(ip):
            zocket_send(msg=msg.STARTMASTER,ip=ip)
            slave_db.create_db().addCallback(post_purge)
            result = {'okay': True, 'ip':ip}
        else:
            result['error'] = "This node is already a Spark slave."
    else:<commit_msg>	modified:   ananke/app.py<commit_after>import json
import os
import time

from klein import Klein
from twisted.web.static import File
import zmq

from db import Database
from docgetter import get_docs
from ipgetter import get_ip
import msg
from servicegetters import got_cluster, got_slave, got_notebook
import settings

app = Klein()
slave_db = Database()

def zocket_send(**kwargs):
    zcontext = zmq.Context()
    socket = zcontext.socket(zmq.PUSH)
    socket.bind("ipc:///tmp/sock")
    socket.send_json(kwargs)
    
def get_request_par(req,par):
    return req.args.get(bytes(par,'utf-8'), False)[0].decode('utf-8')
    
@app.route('/', branch=True)
def root(request):
    return File("./static/")

@app.route('/api/getdocs')
def all_the_docs(reques):
    return json.dumps({'allthedocs':get_docs()})

@app.route('/api/status')
def status(request):

    result = {}
    
    result['virtual'] = json.loads(os.environ.get('VBOX','false')) == 'true'
        
    if result['virtual']:
        try:
            with open('ip.json', 'r') as ipfile:
                result['ext_ip'] = json.loads(ipfile.read())['ip'].split('.')
            result['realip'] = True
        except:
            result['realip'] = False
    else:
        result['realip'] = False
              
    with open('id.json', 'r') as idfile:
        result['appid'] = json.loads(idfile.read())['id']
        
    ip = get_ip()
    if ip:
        result['network'] = True
        result['ip'] = ip.split('.')
        result['slave_ip'] = got_slave(ip)
        result['master_owner'] = got_cluster(ip)
        result['pysparknotebook'] = got_notebook()
        if result['master_owner'] or result['slave_ip']:
            result['status'] = 'active'
        else:
            result['status'] = 'dormant'
    else:
        result['status'] = 'error'
            
    return json.dumps(result)
    
def valid_ip(ip): 
    try:
        chunks = ip.split('.')
        return len(chunks) == 4 and all([c.isnumeric and 0 <= int(c) <= 255 for c in chunks])
    except:
        return False

@app.route('/api/testip')
def test_ip(request):
    result = {'okay':False}
    try:
        external_ip = get_request_par(request,'ip')
    except:
        external_ip = False
    with open('id.json', 'r') as idfile:
        true_id = json.loads(idfile.read())['id']
    try:    
        app_id = get_request_par(request,'id')
    except:
        add_id = False
    if app_id == true_id:
        result = {'okay':True}
        with open('ip.json', 'w') as ipfile:
            ipfile.write(json.dumps({'ip':external_ip}))
    callback = get_request_par(request,'callback')
    return callback+'('+json.dumps(result)+');'    
    
@app.route('/api/startcluster')
def start_master(request):
    result = {'okay':False}
    vbox = os.environ.get('VBOX','false')
        if vbox == 'false':
            ip = get_ip()
        else:
            try:
                with open('ip.json', 'r') as ipfile:
                    ip = json.loads(ipfile.read())['ip']
                except:
                    ip = '127.0.0.1'
            
    if not got_cluster(ip):
        if not got_slave(ip):
            zocket_send(msg=msg.STARTMASTER,ip=ip)
            slave_db.create_db().addCallback(post_purge)
            result = {'okay': True, 'ip':ip}
        else:
            result['error'] = "This node is already a Spark slave."
    else:
{'fraction_non_alphanumeric': 0.15528430160692214, 'fraction_numerical': 0.003399258343634116, 'mean_word_length': 2.4596472474612505, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3569309', 'n_tokens_mistral': 2133, 'n_tokens_neox': 2125, 'n_words': 448}