<commit_before>from datetime import datetime
from tempfile import TemporaryFile
from zipfile import ZipFile
from enum import Enum
import hashlib
import os

from flask import Blueprint, render_template, request, flash, redirect, g, current_app, url_for, abort, make_response
from flask.ext.babel import _, lazy_gettext as l_
from redis.exceptions import ConnectionError
from werkzeug.exceptions import BadRequest

from skylines.database import db
from skylines.frontend.forms import UploadForm, UploadUpdateForm
from skylines.lib import files
from skylines.lib.util import pressure_alt_to_qnh_alt
from skylines.lib.decorators import login_required
from skylines.lib.md5 import file_md5
from skylines.lib.sql import query_to_sql
from skylines.lib.xcsoar_ import flight_path, analyse_flight
from skylines.model import User, Flight, IGCFile, Airspace
from skylines.model.airspace import get_airspace_infringements
from skylines.model.event import create_flight_notifications
from skylines.worker import tasks

from geoalchemy2.shape import from_shape
from sqlalchemy.sql import literal_column
from shapely.geometry import MultiLineString, LineString

try:
    import mapscript
    import pyproj
    mapscript_available = True
except ImportError:
    mapscript_available = False

import xcsoar

upload_blueprint = Blueprint('upload', 'skylines')


class UploadStatus(Enum):
    SUCCESS = 0
    DUPLICATE = 1  # _('Duplicate file')
    MISSING_DATE = 2  # _('Date missing in IGC file')
    PARSER_ERROR = 3  # _('Failed to parse file')
    NO_FLIGHT = 4  # _('No flight found in file')
    FLIGHT_IN_FUTURE = 5  # _('Date of flight in future')


def IterateFiles(name, f):
    try:
        z = ZipFile(f, 'r')
    except:
        # if f is not a ZipFile

        # reset the pointer to the top of the file
        # (the ZipFile constructor might have moved it!)
        f.seek(0)
        yield name, f
    else:
        # if f is a ZipFile
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')


def IterateUploadFiles(upload):
    if isinstance(upload, unicode):
        # the Chromium browser sends an empty string if no file is selected
        if not upload:
            return

        # some Android versions send the IGC file as a string, not as
        # a file
        with TemporaryFile() as f:
            f.write(upload.encode('UTF-8'))
            f.seek(0)
            yield 'direct.igc', f

    elif isinstance(upload, list):
        for x in upload:
            for name, f in IterateUploadFiles(x):
                yield name, f

    else:
        for x in IterateFiles(upload.filename, upload):
            yield x


def _encode_flight_path(fp, qnh):
    # Reduce to 1000 points maximum with equal spacing
    shortener = int(max(1, len(fp) / 1000))

    barogram_h = xcsoar.encode([pressure_alt_to_qnh_alt(fix.pressure_altitude, qnh) for fix in fp[::shortener]],
                               method="signed")
    barogram_t = xcsoar.encode([fix.seconds_of_day for fix in fp[::shortener]], method="signed")
    enl = xcsoar.encode([fix.enl if fix.enl is not None else 0 for fix in fp[::shortener]], method="signed")
    elevations_h = xcsoar.encode([fix.elevation if fix.elevation is not None else -1000 for fix in fp[::shortener]], method="signed")

    return dict(barogram_h=barogram_h, barogram_t=barogram_t,
                enl=enl, elevations_h=elevations_h,
                igc_start_time=fp[0].datetime, igc_end_time=fp[-1].datetime)


@upload_blueprint.route('/', methods=('GET', 'POST'))
@login_required(l_("You have to login to upload flights."))
def index():
    if request.values.get('stage', type=int) == 1:
        # Parse update form
        num_flights = request.values.get('num_flights', 0, type=int)

        flights = []
        flight_id_list = []
        form_error = False

        for prefix in range(1, num_flights + 1):
            name = request.values.get('{}-name'.format(prefix))

            try:
                status = UploadStatus(request.values.get('{}-status'.format(prefix), type=int))
            except ValueError:
                raise BadRequest('Status unknown')

            flight, fp, form = check_update_form(prefix, status)

            if fp:
                trace = _encode_flight_path(fp, flight.qnh)
                infringements = get_airspace_infringements(fp, qnh=flight.qnh)
            else:
                trace = None
                infringements = {}

            cache_key = None

            if form and not infringements:
                # remove airspace field from form if no airspace infringements found
                del form.airspace_usage

            elif form and infringements:
                # if airspace infringements found create cache_key from flight id and user id
                cache_key = hashlib.sha1(str(flight.id) + '_' + str(g.current_user.id)).hexdigest()

            airspace = db.session.query(Airspace) \
                                 .filter(Airspace.id.in_(infringements.keys())) \
                                 .all()

            flights.append((name, flight, status, str(prefix), trace, airspace, cache_key, form))

            if form and form.validate_on_submit():
                _update_flight(flight.id,
                               fp,
                               form.model_id.data,
                               form.registration.data,
                               form.competition_id.data,
                               form.takeoff_time.data,
                               form.scoring_start_time.data,
                               form.scoring_end_time.data,
                               form.landing_time.data,
                               form.pilot_id.data, form.pilot_name.data,
                               form.co_pilot_id.data, form.co_pilot_name.data)
                flight_id_list.append(flight.id)
            elif form:
                form_error = True

        if form_error:
            return render_template(
                'upload/result.jinja', num_flights=num_flights, flights=flights, success=True)
        elif flight_id_list:
            flash(_('Your flight(s) have been successfully published.'))
            return redirect(url_for('flights.list', ids=','.join(str(x) for x in flight_id_list)))
        else:
            return redirect(url_for('flights.today'))

    else:
        # Create/parse file selection form
        form = UploadForm(pilot=g.current_user.id)

        if form.validate_on_submit():
            return index_post(form)

        return render_template('upload/form.jinja', form=form)


def index_post(form):
    user = g.current_user

    pilot_id = form.pilot.data if form.pilot.data != 0 else None
    pilot = pilot_id and User.get(int(pilot_id))
    pilot_id = pilot and pilot.id

    club_id = (pilot and pilot.club_id) or user.club_id

    flights = []
    success = False

    prefix = 0
    for name, f in IterateUploadFiles(form.file.raw_data):<commit_msg>views/upload: Fix function name<commit_after>from datetime import datetime
from tempfile import TemporaryFile
from zipfile import ZipFile
from enum import Enum
import hashlib
import os

from flask import Blueprint, render_template, request, flash, redirect, g, current_app, url_for, abort, make_response
from flask.ext.babel import _, lazy_gettext as l_
from redis.exceptions import ConnectionError
from werkzeug.exceptions import BadRequest

from skylines.database import db
from skylines.frontend.forms import UploadForm, UploadUpdateForm
from skylines.lib import files
from skylines.lib.util import pressure_alt_to_qnh_alt
from skylines.lib.decorators import login_required
from skylines.lib.md5 import file_md5
from skylines.lib.sql import query_to_sql
from skylines.lib.xcsoar_ import flight_path, analyse_flight
from skylines.model import User, Flight, IGCFile, Airspace
from skylines.model.airspace import get_airspace_infringements
from skylines.model.event import create_flight_notifications
from skylines.worker import tasks

from geoalchemy2.shape import from_shape
from sqlalchemy.sql import literal_column
from shapely.geometry import MultiLineString, LineString

try:
    import mapscript
    import pyproj
    mapscript_available = True
except ImportError:
    mapscript_available = False

import xcsoar

upload_blueprint = Blueprint('upload', 'skylines')


class UploadStatus(Enum):
    SUCCESS = 0
    DUPLICATE = 1  # _('Duplicate file')
    MISSING_DATE = 2  # _('Date missing in IGC file')
    PARSER_ERROR = 3  # _('Failed to parse file')
    NO_FLIGHT = 4  # _('No flight found in file')
    FLIGHT_IN_FUTURE = 5  # _('Date of flight in future')


def iterate_files(name, f):
    try:
        z = ZipFile(f, 'r')
    except:
        # if f is not a ZipFile

        # reset the pointer to the top of the file
        # (the ZipFile constructor might have moved it!)
        f.seek(0)
        yield name, f
    else:
        # if f is a ZipFile
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')


def iterate_upload_files(upload):
    if isinstance(upload, unicode):
        # the Chromium browser sends an empty string if no file is selected
        if not upload:
            return

        # some Android versions send the IGC file as a string, not as
        # a file
        with TemporaryFile() as f:
            f.write(upload.encode('UTF-8'))
            f.seek(0)
            yield 'direct.igc', f

    elif isinstance(upload, list):
        for x in upload:
            for name, f in iterate_upload_files(x):
                yield name, f

    else:
        for x in iterate_files(upload.filename, upload):
            yield x


def _encode_flight_path(fp, qnh):
    # Reduce to 1000 points maximum with equal spacing
    shortener = int(max(1, len(fp) / 1000))

    barogram_h = xcsoar.encode([pressure_alt_to_qnh_alt(fix.pressure_altitude, qnh) for fix in fp[::shortener]],
                               method="signed")
    barogram_t = xcsoar.encode([fix.seconds_of_day for fix in fp[::shortener]], method="signed")
    enl = xcsoar.encode([fix.enl if fix.enl is not None else 0 for fix in fp[::shortener]], method="signed")
    elevations_h = xcsoar.encode([fix.elevation if fix.elevation is not None else -1000 for fix in fp[::shortener]], method="signed")

    return dict(barogram_h=barogram_h, barogram_t=barogram_t,
                enl=enl, elevations_h=elevations_h,
                igc_start_time=fp[0].datetime, igc_end_time=fp[-1].datetime)


@upload_blueprint.route('/', methods=('GET', 'POST'))
@login_required(l_("You have to login to upload flights."))
def index():
    if request.values.get('stage', type=int) == 1:
        # Parse update form
        num_flights = request.values.get('num_flights', 0, type=int)

        flights = []
        flight_id_list = []
        form_error = False

        for prefix in range(1, num_flights + 1):
            name = request.values.get('{}-name'.format(prefix))

            try:
                status = UploadStatus(request.values.get('{}-status'.format(prefix), type=int))
            except ValueError:
                raise BadRequest('Status unknown')

            flight, fp, form = check_update_form(prefix, status)

            if fp:
                trace = _encode_flight_path(fp, flight.qnh)
                infringements = get_airspace_infringements(fp, qnh=flight.qnh)
            else:
                trace = None
                infringements = {}

            cache_key = None

            if form and not infringements:
                # remove airspace field from form if no airspace infringements found
                del form.airspace_usage

            elif form and infringements:
                # if airspace infringements found create cache_key from flight id and user id
                cache_key = hashlib.sha1(str(flight.id) + '_' + str(g.current_user.id)).hexdigest()

            airspace = db.session.query(Airspace) \
                                 .filter(Airspace.id.in_(infringements.keys())) \
                                 .all()

            flights.append((name, flight, status, str(prefix), trace, airspace, cache_key, form))

            if form and form.validate_on_submit():
                _update_flight(flight.id,
                               fp,
                               form.model_id.data,
                               form.registration.data,
                               form.competition_id.data,
                               form.takeoff_time.data,
                               form.scoring_start_time.data,
                               form.scoring_end_time.data,
                               form.landing_time.data,
                               form.pilot_id.data, form.pilot_name.data,
                               form.co_pilot_id.data, form.co_pilot_name.data)
                flight_id_list.append(flight.id)
            elif form:
                form_error = True

        if form_error:
            return render_template(
                'upload/result.jinja', num_flights=num_flights, flights=flights, success=True)
        elif flight_id_list:
            flash(_('Your flight(s) have been successfully published.'))
            return redirect(url_for('flights.list', ids=','.join(str(x) for x in flight_id_list)))
        else:
            return redirect(url_for('flights.today'))

    else:
        # Create/parse file selection form
        form = UploadForm(pilot=g.current_user.id)

        if form.validate_on_submit():
            return index_post(form)

        return render_template('upload/form.jinja', form=form)


def index_post(form):
    user = g.current_user

    pilot_id = form.pilot.data if form.pilot.data != 0 else None
    pilot = pilot_id and User.get(int(pilot_id))
    pilot_id = pilot and pilot.id

    club_id = (pilot and pilot.club_id) or user.club_id

    flights = []
    success = False

    prefix = 0
    for name, f in iterate_upload_files(form.file.raw_data):
{'fraction_non_alphanumeric': 0.08672240328751595, 'fraction_numerical': 0.0051013178404421145, 'mean_word_length': 2.2575582737133626, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2956126', 'n_tokens_mistral': 4303, 'n_tokens_neox': 4073, 'n_words': 1112}