<commit_before>"""`main` is the top level module for the blueprint indexer Flask app)"""

from google.appengine.api import urlfetch
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.ext import blobstore, ndb
from flask import Flask, render_template, request, make_response, redirect, url_for
from starmade import Blueprint
from struct import Struct
from werkzeug import parse_options_header
import json
import math
import logging
import starmade
import urllib
import zipfile

app = Flask(__name__)


class Secrets(ndb.Model):
    """Datastore Entity for Private keys"""
    secret_key = ndb.StringProperty(indexed=False)

millnames=['', 'K', 'M', 'B', 'T']
def millify(n):
    """Converts a number into it's short scale abbreviation

    From here: http://stackoverflow.com/a/3155023/153407
    """
    n = float(n)
    millidx = max(0,min(len(millnames)-1,
                  int(math.floor(math.log10(abs(n))/3))))
    return '%.0f %s'%(n/10**(3*millidx),millnames[millidx])

@app.route("/upload")
def upload():
    uploadUri = blobstore.create_upload_url('/submit',
                                            gs_bucket_name='blueprints')
    return render_template('upload.html', uploadUri=uploadUri)

@app.route("/submit", methods=['POST'])
def submit():
    if request.method == 'POST':

        # Check reCaptcha before uploading
        recaptcha = request.form['g-recaptcha-response']
        url = 'https://www.google.com/recaptcha/api/siteverify'
        recaptcha_secrets = ndb.Key(Secrets, 'recaptcha').get()
        form_fields = {
            "secret": recaptcha_secrets.secret_key,
            "response": recaptcha,
            "remoteip": request.remote_addr
        }
        form_data = urllib.urlencode(form_fields)
        verify_recaptcha = urlfetch.fetch(url=url,
            payload=form_data,
            method=urlfetch.POST,
            headers={'Content-Type': 'application/x-www-form-urlencoded'})
        verify_content = json.loads(verify_recaptcha.content)

        # Bypassing check since it seems like blobstore IP gets set here
        # making it impossible to confirm recaptcha in this manner :(
#        if (verify_content.get('success'), False):
#            return request.remote_addr+' Sorry, reCaptcha failed'+ verify_recaptcha.content, 403

        # reCaptcha has passed if we get to here - proceed with upload
        f = request.files['file']
        power_recharge = starmade.valid_power(request.form['power_recharge'])
        power_capacity = starmade.valid_power(request.form['power_capacity'])
        blueprint_title = None
        header = f.headers['Content-Type']
        parsed_header = parse_options_header(header)
        blob_key = parsed_header[1]['blob-key']

        blue_key = process_blueprint(blob_key, blueprint_title, power_recharge,
                                     power_capacity)

        return render_template('finished_upload.html',
                               blue_key=blue_key.urlsafe())

def process_blueprint(blob_key, blueprint_title, power_recharge=0,
                      power_capacity=0, blue_key=None):
    blob_info = blobstore.get(blob_key)
    blob = blob_info.open()

    with zipfile.ZipFile(file=blob, mode="r") as zip_file:
        for filename in (name for name in zip_file.namelist()
                         if name.endswith('/header.smbph')
                            and name.count('/') <= 1):
            blueprint_title = filename[:filename.find("/")].replace("_", " ")
            header_blob = zip_file.open(filename)
            return process_header(blob_key, header_blob, blueprint_title,
                                  power_recharge, power_capacity, blue_key)

def process_header(blob_key, blob, blueprint_title, power_recharge=0,
                   power_capacity=0, blue_key=None):
    version_struct = Struct('>i')
    ver = version_struct.unpack(blob.read(version_struct.size))[0]
    if ver > 65535:
        endian = '<'
        ver = ver<<24&0xFF000000|ver<<8&0xFF0000|ver>>8&0xFF00|ver>>24&0xFF
    else:
        endian = '>'
    header_struct = Struct(endian + 'I3f3fi')
    block_struct = Struct(endian + 'hi') # BlockID<short>, blockCount<int>

    result = []
    result = header_struct.unpack(blob.read(header_struct.size))

    entity_type = result[0]

    # -2 since core-only blueprint gives 2, -1 respectively.
    length = int(result[6] - result[3]) - 2
    width = int(result[4] - result[1]) - 2
    height = int(result[5] - result[2]) - 2

    context = {
       "title": blueprint_title,
       "version": ver,
       "entity": entity_type,
       "length": length,
       "width": width,
       "height": height,
       "power_recharge": {"base": 1},
       "power_capacity": {"base": 50000},
       "power_usage": {},
       "thrust": "None",
       "shields": {"capacity": 220, "recharge": 0},
       "systems" : {"medical": 0, "factory": 0}
    }

    ship_dimensions = context['length'] + context['width'] + context['height']
    element_count = result[7]
    element_list = []
    total_block_count = 0
    total_mass = 0
    power_recharge_rating = 0
    complex_systems = {"salvage": 0, "astrotech": 0, "power_drain": 0,
                       "power_supply": 0, "shield_drain": 0, "shield_supply": 0}

    for element in xrange(0, element_count):
        new_element = block_struct.unpack(blob.read(block_struct.size))
        element_list.append([new_element])
        block_id = new_element[0]
        block_count = new_element[1]
        total_block_count += block_count
        total_mass += block_count * starmade.NON_STANDARD_MASS.get(block_id, 0.1)
        if block_id == 2: # Power Block
            power_output = starmade.calc_power_output(block_count,
                                                      ship_dimensions)
            ideal_generator = round(power_output, 1)
            if power_recharge > ideal_generator or power_recharge == 0:
               context['power_recharge']['ideal_generator'] = ideal_generator
               context['power_efficieny_gauge'] = 100.0
               power_recharge_rating = ideal_generator
            else:
               power_efficieny_gauge = power_recharge / ideal_generator
               context['power_efficiency_gauge'] = round(power_efficieny_gauge * 100.0,1)
               context['power_recharge']['power_recharge'] = power_recharge
               power_recharge_rating = power_recharge
        elif block_id == 331: # Power Capacitor
            power_storage = starmade.calc_power_capacity(block_count)
            ideal_capacitor = round(power_storage, 1)
            if entity_type == 0:
                base_capacity = 50000
            else:
                base_capacity = 0
            context['ideal_capacitor'] = ideal_capacitor
            if power_capacity > ideal_capacitor + base_capacity or power_capacity == 0:
               context['power_capacity']['ideal_capacitor'] = ideal_capacitor
            else:
               context['power_capacity']['power_capacity'] = power_capacity
               power_capacity_gauge = power_capacity / ideal_capacitor
               context['power_capacity_efficiency_gauge'] = round(power_capacity_gauge * 100.0,1)
        elif block_id == 8: # Thruster Block
            context['thrust'] = round(starmade.calc_thrust(block_count),1)
            context['power_usage']['thruster'] = round(-starmade.calc_thrust_power(block_count), 0)
        elif block_id == 3: # Shield Capacitor Block
            context['shields']['capacity'] = round(starmade.calc_shield_capacity(block_count), 0)
        elif block_id == 478: # Shield Recharger Block
            shield_power_standby = starmade.calc_shield_power(block_count)
            shield_power_active = starmade.calc_shield_power(block_count, True)
            shield_recharge = starmade.calc_shield_recharge(block_count)
            context['shields']['recharge'] = round(shield_recharge, 0)
            context['power_recharge']['shields'] = -round(shield_power_standby, 0)
            context['power_usage']['shield_recharge'] = -round(shield_power_active, 0)
        elif block_id == 15: # Radar Jamming
            context['systems']['radar_jamming'] = block_count
        elif block_id == 22: # Cloaking
            context['systems']['cloaking'] = block_count
        elif block_id == 291: # Faction
            context['systems']['faction'] = block_count
        elif block_id == 347: # Shop
            context['systems']['shop'] = block_count
        elif block_id == 94: # Plex Undeathinator
            context['systems']['plexundeathinator'] = block_count
        elif block_id >= 211 and block_id <= 215: # Factory equipment
            context['systems']['factory'] += block_count
        elif block_id == 121: # AI
            context['systems']['bobby_ai'] = block_count<commit_msg>Fixed custom power capacity to account for base power<commit_after>"""`main` is the top level module for the blueprint indexer Flask app)"""

from google.appengine.api import urlfetch
from google.appengine.datastore.datastore_query import Cursor
from google.appengine.ext import blobstore, ndb
from flask import Flask, render_template, request, make_response, redirect, url_for
from starmade import Blueprint
from struct import Struct
from werkzeug import parse_options_header
import json
import math
import logging
import starmade
import urllib
import zipfile

app = Flask(__name__)


class Secrets(ndb.Model):
    """Datastore Entity for Private keys"""
    secret_key = ndb.StringProperty(indexed=False)

millnames=['', 'K', 'M', 'B', 'T']
def millify(n):
    """Converts a number into it's short scale abbreviation

    From here: http://stackoverflow.com/a/3155023/153407
    """
    n = float(n)
    millidx = max(0,min(len(millnames)-1,
                  int(math.floor(math.log10(abs(n))/3))))
    return '%.0f %s'%(n/10**(3*millidx),millnames[millidx])

@app.route("/upload")
def upload():
    uploadUri = blobstore.create_upload_url('/submit',
                                            gs_bucket_name='blueprints')
    return render_template('upload.html', uploadUri=uploadUri)

@app.route("/submit", methods=['POST'])
def submit():
    if request.method == 'POST':

        # Check reCaptcha before uploading
        recaptcha = request.form['g-recaptcha-response']
        url = 'https://www.google.com/recaptcha/api/siteverify'
        recaptcha_secrets = ndb.Key(Secrets, 'recaptcha').get()
        form_fields = {
            "secret": recaptcha_secrets.secret_key,
            "response": recaptcha,
            "remoteip": request.remote_addr
        }
        form_data = urllib.urlencode(form_fields)
        verify_recaptcha = urlfetch.fetch(url=url,
            payload=form_data,
            method=urlfetch.POST,
            headers={'Content-Type': 'application/x-www-form-urlencoded'})
        verify_content = json.loads(verify_recaptcha.content)

        # Bypassing check since it seems like blobstore IP gets set here
        # making it impossible to confirm recaptcha in this manner :(
#        if (verify_content.get('success'), False):
#            return request.remote_addr+' Sorry, reCaptcha failed'+ verify_recaptcha.content, 403

        # reCaptcha has passed if we get to here - proceed with upload
        f = request.files['file']
        power_recharge = starmade.valid_power(request.form['power_recharge'])
        power_capacity = starmade.valid_power(request.form['power_capacity'])
        blueprint_title = None
        header = f.headers['Content-Type']
        parsed_header = parse_options_header(header)
        blob_key = parsed_header[1]['blob-key']

        blue_key = process_blueprint(blob_key, blueprint_title, power_recharge,
                                     power_capacity)

        return render_template('finished_upload.html',
                               blue_key=blue_key.urlsafe())

def process_blueprint(blob_key, blueprint_title, power_recharge=0,
                      power_capacity=0, blue_key=None):
    blob_info = blobstore.get(blob_key)
    blob = blob_info.open()

    with zipfile.ZipFile(file=blob, mode="r") as zip_file:
        for filename in (name for name in zip_file.namelist()
                         if name.endswith('/header.smbph')
                            and name.count('/') <= 1):
            blueprint_title = filename[:filename.find("/")].replace("_", " ")
            header_blob = zip_file.open(filename)
            return process_header(blob_key, header_blob, blueprint_title,
                                  power_recharge, power_capacity, blue_key)

def process_header(blob_key, blob, blueprint_title, power_recharge=0,
                   power_capacity=0, blue_key=None):
    version_struct = Struct('>i')
    ver = version_struct.unpack(blob.read(version_struct.size))[0]
    if ver > 65535:
        endian = '<'
        ver = ver<<24&0xFF000000|ver<<8&0xFF0000|ver>>8&0xFF00|ver>>24&0xFF
    else:
        endian = '>'
    header_struct = Struct(endian + 'I3f3fi')
    block_struct = Struct(endian + 'hi') # BlockID<short>, blockCount<int>

    result = []
    result = header_struct.unpack(blob.read(header_struct.size))

    entity_type = result[0]

    # -2 since core-only blueprint gives 2, -1 respectively.
    length = int(result[6] - result[3]) - 2
    width = int(result[4] - result[1]) - 2
    height = int(result[5] - result[2]) - 2

    context = {
       "title": blueprint_title,
       "version": ver,
       "entity": entity_type,
       "length": length,
       "width": width,
       "height": height,
       "power_recharge": {"base": 1},
       "power_capacity": {"base": 50000},
       "power_usage": {},
       "thrust": "None",
       "shields": {"capacity": 220, "recharge": 0},
       "systems" : {"medical": 0, "factory": 0}
    }

    ship_dimensions = context['length'] + context['width'] + context['height']
    element_count = result[7]
    element_list = []
    total_block_count = 0
    total_mass = 0
    power_recharge_rating = 0
    complex_systems = {"salvage": 0, "astrotech": 0, "power_drain": 0,
                       "power_supply": 0, "shield_drain": 0, "shield_supply": 0}

    for element in xrange(0, element_count):
        new_element = block_struct.unpack(blob.read(block_struct.size))
        element_list.append([new_element])
        block_id = new_element[0]
        block_count = new_element[1]
        total_block_count += block_count
        total_mass += block_count * starmade.NON_STANDARD_MASS.get(block_id, 0.1)
        if block_id == 2: # Power Block
            power_output = starmade.calc_power_output(block_count,
                                                      ship_dimensions)
            ideal_generator = round(power_output, 1)
            if power_recharge > ideal_generator or power_recharge == 0:
               context['power_recharge']['ideal_generator'] = ideal_generator
               context['power_efficieny_gauge'] = 100.0
               power_recharge_rating = ideal_generator
            else:
               power_efficieny_gauge = power_recharge / ideal_generator
               context['power_efficiency_gauge'] = round(power_efficieny_gauge * 100.0,1)
               context['power_recharge']['power_recharge'] = power_recharge
               power_recharge_rating = power_recharge
        elif block_id == 331: # Power Capacitor
            power_storage = starmade.calc_power_capacity(block_count)
            ideal_capacitor = round(power_storage, 1)
            if entity_type == 0:
                base_capacity = 50000
            else:
                base_capacity = 0
            context['ideal_capacitor'] = ideal_capacitor
            power_capacity -= base_capacity
            if power_capacity > ideal_capacitor or power_capacity == 0:
               context['power_capacity']['ideal_capacitor'] = ideal_capacitor
            else:
               context['power_capacity']['power_capacity'] = power_capacity
               power_capacity_gauge = power_capacity / ideal_capacitor
               context['power_capacity_efficiency_gauge'] = round(power_capacity_gauge * 100.0,1)
        elif block_id == 8: # Thruster Block
            context['thrust'] = round(starmade.calc_thrust(block_count),1)
            context['power_usage']['thruster'] = round(-starmade.calc_thrust_power(block_count), 0)
        elif block_id == 3: # Shield Capacitor Block
            context['shields']['capacity'] = round(starmade.calc_shield_capacity(block_count), 0)
        elif block_id == 478: # Shield Recharger Block
            shield_power_standby = starmade.calc_shield_power(block_count)
            shield_power_active = starmade.calc_shield_power(block_count, True)
            shield_recharge = starmade.calc_shield_recharge(block_count)
            context['shields']['recharge'] = round(shield_recharge, 0)
            context['power_recharge']['shields'] = -round(shield_power_standby, 0)
            context['power_usage']['shield_recharge'] = -round(shield_power_active, 0)
        elif block_id == 15: # Radar Jamming
            context['systems']['radar_jamming'] = block_count
        elif block_id == 22: # Cloaking
            context['systems']['cloaking'] = block_count
        elif block_id == 291: # Faction
            context['systems']['faction'] = block_count
        elif block_id == 347: # Shop
            context['systems']['shop'] = block_count
        elif block_id == 94: # Plex Undeathinator
            context['systems']['plexundeathinator'] = block_count
        elif block_id >= 211 and block_id <= 215: # Factory equipment
            context['systems']['factory'] += block_count
        elif block_id == 121: # AI
            context['systems']['bobby_ai'] = block_count
{'fraction_non_alphanumeric': 0.11353146459529438, 'fraction_numerical': 0.0182370820668693, 'mean_word_length': 2.7084116050928824, 'pattern_counts': {'":': 52, '<': 21, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3964056', 'n_tokens_mistral': 5546, 'n_tokens_neox': 5173, 'n_words': 1068}