<commit_before>import os
import mimetypes

from flask import Blueprint, Flask, current_app, url_for
from flask import render_template, send_file, abort

from lopypi.store import PackageStore
from lopypi.pypi import PyPI


views = Blueprint('proxy', __name__, template_folder="templates")


@views.route("/simple/", methods=['GET'])
def list_packages():
    """
    """
    pypi = current_app.config['pypi']

    packages = (dict(name=name,
                     uri=url_for('proxy.list_files', package=name))
                for name in pypi.list_packages())
    return render_template("package_list.html", packages=packages)


@views.route("/simple/<package>/", methods=['GET'])
def list_files(package):
    """
    """
    pypi = current_app.config['pypi']
    store = current_app.config['cached_file_store']

    files = list(pypi.list_files(package))
    map(lambda f: f.update({'uri': url_for('proxy.get_file',
                                           package=package,
                                           filename=f['filename'])}),
        files)
    return render_template("file_list.html",
                           package=package,
                           files=files)


@views.route("/packages/<package>/<filename>", methods=["GET"])
def get_file(package, filename):
    """
    """
    pypi = current_app.config['pypi']
    store = current_app.config['cached_file_store']

    f = store.get_file(package, filename)

    if not f:
        uri = pypi.get_file_uri(package, filename)
        if not uri:
            abort(404)
        # TODO should be async
        # TODO fix simultaneous downloads of same packae
        # TODO give option of redirecting instead of saving
        with urllib2.urlopen(uri) as download:
            store.add_file(package, filename, download)

        f = store.get_file(package, filename)

    content_type, _ = mimetypes.guess_type(filename)
    if content_type is None and filename.endswith(".egg"):
        content_type = "application/zip"

    return send_file(f, mimetype=content_type)


def create_app(package_dir, pypi_url):
    app = Flask(__name__)
    if not os.path.isdir(package_dir):
        os.makedirs(package_dir)
    app.config['cached_file_store'] = PackageStore(package_dir)
    app.config['pypi'] = PyPI(pypi_url)
    app.register_blueprint(views)<commit_msg>Initial (read f'ugly) implementation of get_file function<commit_after>import os
import mimetypes
import tempfile
import requests
import shutil

from flask import Blueprint, Flask, current_app, url_for
from flask import render_template, send_file, abort, Response

from lopypi.store import PackageStore
from lopypi.pypi import PyPI


views = Blueprint('proxy', __name__, template_folder="templates")


@views.route("/simple/", methods=['GET'])
def list_packages():
    """
    """
    pypi = current_app.config['pypi']

    packages = list(pypi.list_packages())
    for package in packages:
        package['uri'] = url_for('proxy.list_files', package=package['name'])
    return render_template("package_list.html", packages=packages)


@views.route("/simple/<package>/", methods=['GET'])
def list_files(package):
    """
    """
    pypi = current_app.config['pypi']
    store = current_app.config['cached_file_store']

    files = list(pypi.list_files(package))
    for f in files:
        f['uri'] = url_for('proxy.get_file',
                           package=package,
                           filename=f['filename'])

    return render_template("file_list.html",
                           package=package,
                           files=files)


@views.route("/packages/<package>/<filename>", methods=["GET"])
def get_file(package, filename):
    """
    """
    pypi = current_app.config['pypi']
    store = current_app.config['cached_file_store']

    content_type, _ = mimetypes.guess_type(filename)
    if content_type is None and filename.endswith(".egg"):
        content_type = "application/zip"

    f = store.get_file(package, filename)
    if f:
        # file is already cached locally.  Send it
        return send_file(f, mimetype=content_type)

    ## request file from pypi and stream it to client and disk
    # get url of remote file
    files = pypi.list_files(package)
    file_info = filter(lambda f: f['filename'] == filename, files)
    if not len(file_info):
        abort(404)
    file_info = file_info[0]
    uri = file_info['remote_uri']

    # start download
    download = requests.get(uri, stream=True)
    if download.status_code != 200:
        # TODO
        download.close()
        abort(download.status_code)

    def generate():
        # TODO this is fucking ugly
        data_iter = download.iter_content(1024)
        outfile = tempfile.NamedTemporaryFile(delete=False)
        try:
            for data in data_iter:
                outfile.write(data)
                yield data
        finally:
            for data in data_iter:
                outfile.write(data)
            outfile.seek(0)

            # TODO verify checksum
            store.add_file(package, filename, outfile)

    response = Response(generate(), mimetype=content_type)
    if 'content-length' in download.headers:
        response.content_length = download.headers['content-length']

    return response


def create_app(package_dir, pypi_url):
    app = Flask(__name__)
    if not os.path.isdir(package_dir):
        os.makedirs(package_dir)
    app.config['cached_file_store'] = PackageStore(package_dir)
    app.config['pypi'] = PyPI(pypi_url)
    app.register_blueprint(views)
{'fraction_non_alphanumeric': 0.11491971856395454, 'fraction_numerical': 0.002886523543207649, 'mean_word_length': 2.8208132322536184, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596804', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1622, 'n_words': 360}