<commit_before>from tg import expose, request, redirect, flash
from tg.i18n import ugettext as _, lazy_ugettext as l_
from repoze.what.predicates import has_permission
from skylines.lib.base import BaseController
from skylines import files
from skylines.model import DBSession, Flight
from skylines.lib.analysis import analyse_flight

def IterateFiles(name, f):
    from zipfile import ZipFile
    try:
        z = ZipFile(f, 'r')
    except:
        f.seek(0)
        yield name, f
    else:
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')

class UploadController(BaseController):
    allow_only = has_permission('upload',
                                msg=l_("You don't have permission to upload flights."))

    @expose('skylines.templates.upload.index')
    def index(self):
        return dict(page = 'upload')

    @expose('skylines.templates.upload.result')
    def do(self, file):
        user = request.identity['user']

        flights = []

        for name, f in IterateFiles(file.filename, file.file):
            filename = files.sanitise_filename(name)
            filename = files.add_file(filename, f)

            flight = Flight()
            flight.owner = request.identity['user']
            flight.filename = filename
            flight.club_id = user.club_id

            if not analyse_flight(flight):
                files.delete_file(filename)
                flights.append((name, None, _('Failed to parse file')))
                continue

            other = flight.by_md5(flight.md5)
            if other:
                files.delete_file(filename)
                flights.append((name, other, _('Duplicate file')))
                continue

            flights.append((name, flight, None))
            DBSession.add(flight)

        DBSession.flush()

        return dict(page='upload', flights=flights)

    @expose('skylines.templates.upload.result')
    def test(self):
        return dict(page='upload', flights=[('foo.igc', None, None),('bar.igc', None, 'Error!')])
<commit_msg>controllers/upload: Added comments to IterateFiles()<commit_after>from tg import expose, request, redirect, flash
from tg.i18n import ugettext as _, lazy_ugettext as l_
from repoze.what.predicates import has_permission
from skylines.lib.base import BaseController
from skylines import files
from skylines.model import DBSession, Flight
from skylines.lib.analysis import analyse_flight

def IterateFiles(name, f):
    from zipfile import ZipFile
    try:
        z = ZipFile(f, 'r')
    except:
        # if f is not a ZipFile

        # reset the pointer to the top of the file
        # (the ZipFile constructor might have moved it!)
        f.seek(0)
        yield name, f
    else:
        # if f is a ZipFile
        for info in z.infolist():
            if info.file_size > 0:
                yield info.filename, z.open(info.filename, 'r')

class UploadController(BaseController):
    allow_only = has_permission('upload',
                                msg=l_("You don't have permission to upload flights."))

    @expose('skylines.templates.upload.index')
    def index(self):
        return dict(page = 'upload')

    @expose('skylines.templates.upload.result')
    def do(self, file):
        user = request.identity['user']

        flights = []

        for name, f in IterateFiles(file.filename, file.file):
            filename = files.sanitise_filename(name)
            filename = files.add_file(filename, f)

            flight = Flight()
            flight.owner = request.identity['user']
            flight.filename = filename
            flight.club_id = user.club_id

            if not analyse_flight(flight):
                files.delete_file(filename)
                flights.append((name, None, _('Failed to parse file')))
                continue

            other = flight.by_md5(flight.md5)
            if other:
                files.delete_file(filename)
                flights.append((name, other, _('Duplicate file')))
                continue

            flights.append((name, flight, None))
            DBSession.add(flight)

        DBSession.flush()

        return dict(page='upload', flights=flights)

    @expose('skylines.templates.upload.result')
    def test(self):
        return dict(page='upload', flights=[('foo.igc', None, None),('bar.igc', None, 'Error!')])

{'fraction_non_alphanumeric': 0.10579972813774355, 'fraction_numerical': 0.002718622564567286, 'mean_word_length': 2.2752225519287834, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4487917', 'n_tokens_mistral': 1293, 'n_tokens_neox': 1239, 'n_words': 323}