<commit_before>from tg import expose, request, redirect, flash
from tg.i18n import ugettext as _, lazy_ugettext as l_
from repoze.what.predicates import has_permission
from tw.forms import TableForm
from tw.forms.fields import FileField, SingleSelectField
from skylines.lib.base import BaseController
from skylines import files
from skylines.model import DBSession, User, Flight
from skylines.lib.md5 import file_md5
from skylines.lib.analysis import analyse_flight

class PilotSelectField(SingleSelectField):
    def update_params(self, d):
        users = DBSession.query(User).filter(User.club_id == request.identity['user'].club_id)
        options = [(None, '[unspecified]')] + \
                  [(user.user_id, user) for user in users]
        d['options'] = options
        return SingleSelectField.update_params(self, d)

upload_form = TableForm('upload_form', submit_text="Upload", action='do', children=[
    FileField('file', label_text="IGC or ZIP file"),
    PilotSelectField('pilot', label_text="Pilot"),
])

def IterateFiles(name, f):
    from zipfile import ZipFile
    try:
        z = ZipFile(f, 'r')
    except:
        # if f is not a ZipFile

        # reset the pointer to the top of the file
        # (the ZipFile constructor might have moved it!)
        f.seek(0)
        yield name, f
    else:
        # if f is a ZipFile
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')

class UploadController(BaseController):
    allow_only = has_permission('upload',
                                msg=l_("You don't have permission to upload flights."))

    @expose('skylines.templates.generic.form')
    def index(self):
        return dict(page='upload', title=_("Upload Flight"),
                    form=upload_form,
                    values=dict(pilot=request.identity['user'].user_id))

    @expose('skylines.templates.upload.result')
    def do(self, file, pilot):
        user = request.identity['user']

        flights = []

        for name, f in IterateFiles(file.filename, file.file):
            filename = files.sanitise_filename(name)

            # check if the file already exists
            md5 = file_md5(f)
            other = Flight.by_md5(md5)
            if other:
                flights.append((name, other, _('Duplicate file')))
                continue

            f.seek(0)

            filename = files.add_file(filename, f)

            flight = Flight()
            flight.owner = user
            flight.filename = filename
            flight.md5 = md5
            flight.club_id = user.club_id
            flight.pilot_id = pilot

            if not analyse_flight(flight):
                files.delete_file(filename)
                flights.append((name, None, _('Failed to parse file')))
                continue

            flights.append((name, flight, None))
            DBSession.add(flight)

        DBSession.flush()

        return dict(page='upload', flights=flights)

    @expose('skylines.templates.upload.result')<commit_msg>controllers.upload: disallow upload of files without a flight<commit_after>from tg import expose, request, redirect, flash
from tg.i18n import ugettext as _, lazy_ugettext as l_
from repoze.what.predicates import has_permission
from tw.forms import TableForm
from tw.forms.fields import FileField, SingleSelectField
from skylines.lib.base import BaseController
from skylines import files
from skylines.model import DBSession, User, Flight
from skylines.lib.md5 import file_md5
from skylines.lib.analysis import analyse_flight

class PilotSelectField(SingleSelectField):
    def update_params(self, d):
        users = DBSession.query(User).filter(User.club_id == request.identity['user'].club_id)
        options = [(None, '[unspecified]')] + \
                  [(user.user_id, user) for user in users]
        d['options'] = options
        return SingleSelectField.update_params(self, d)

upload_form = TableForm('upload_form', submit_text="Upload", action='do', children=[
    FileField('file', label_text="IGC or ZIP file"),
    PilotSelectField('pilot', label_text="Pilot"),
])

def IterateFiles(name, f):
    from zipfile import ZipFile
    try:
        z = ZipFile(f, 'r')
    except:
        # if f is not a ZipFile

        # reset the pointer to the top of the file
        # (the ZipFile constructor might have moved it!)
        f.seek(0)
        yield name, f
    else:
        # if f is a ZipFile
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')

class UploadController(BaseController):
    allow_only = has_permission('upload',
                                msg=l_("You don't have permission to upload flights."))

    @expose('skylines.templates.generic.form')
    def index(self):
        return dict(page='upload', title=_("Upload Flight"),
                    form=upload_form,
                    values=dict(pilot=request.identity['user'].user_id))

    @expose('skylines.templates.upload.result')
    def do(self, file, pilot):
        user = request.identity['user']

        flights = []

        for name, f in IterateFiles(file.filename, file.file):
            filename = files.sanitise_filename(name)

            # check if the file already exists
            md5 = file_md5(f)
            other = Flight.by_md5(md5)
            if other:
                flights.append((name, other, _('Duplicate file')))
                continue

            f.seek(0)

            filename = files.add_file(filename, f)

            flight = Flight()
            flight.owner = user
            flight.filename = filename
            flight.md5 = md5
            flight.club_id = user.club_id
            flight.pilot_id = pilot

            if not analyse_flight(flight):
                files.delete_file(filename)
                flights.append((name, None, _('Failed to parse file')))
                continue

            if not flight.takeoff_time or not flight.landing_time:
                files.delete_file(filename)
                flights.append((name, None, _('No flight found in file')))
                continue

            flights.append((name, flight, None))
            DBSession.add(flight)

        DBSession.flush()

        return dict(page='upload', flights=flights)

    @expose('skylines.templates.upload.result')
{'fraction_non_alphanumeric': 0.10233644859813085, 'fraction_numerical': 0.004049844236760125, 'mean_word_length': 2.332122470160872, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4601670', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1798, 'n_words': 477}