<commit_before>from pywb.utils.timeutils import timestamp_now
from pywb.utils.loaders import BlockLoader

from webagg.utils import StreamIter, chunk_encode_iter
from recorder.warcwriter import SimpleTempWARCWriter

from webrecorder.basecontroller import BaseController

from bottle import response
from six.moves.urllib.parse import quote
from six import iteritems
from collections import OrderedDict
import json


# ============================================================================
class DownloadController(BaseController):
    def __init__(self, app, jinja_env, manager, config):
        super(DownloadController, self).__init__(app, jinja_env, manager, config)
        self.paths = config['url_templates']
        self.download_filename = config['download_paths']['filename']
        self.warc_key_templ = config['warc_key_templ']

    def init_routes(self):
        @self.app.get('/<user>/<coll>/<rec>/$download')
        def logged_in_download_rec_warc(user, coll, rec):

            return self.handle_download(user, coll, rec)

        @self.app.get('/<user>/<coll>/$download')
        def logged_in_download_coll_warc(user, coll):

            return self.handle_download(user, coll, '*')

    def create_warcinfo(self, creator, title, metadata, filename):
        info = OrderedDict([
                ('software', 'Webrecorder Platform v2.5'),
                ('format', 'WARC File Format 1.0'),
                ('creator', creator),
                ('isPartOf', title),
                ('json-metadata', json.dumps(metadata)),
               ])

        wi_writer = SimpleTempWARCWriter()
        wi_writer.write_record(wi_writer.create_warcinfo_record(filename, info))
        return wi_writer.get_buffer()

    def create_coll_warcinfo(self, user, collection, filename=''):
        metadata = {'desc': collection['desc'],
                    'title': collection['title']}

        title = quote(collection['title'])
        return self.create_warcinfo(user, title, metadata, filename)

    def create_rec_warcinfo(self, user, collection, recording, filename=''):
        metadata = {'pages': self.manager.list_pages(user,
                                                     collection['id'],
                                                     recording['id']),
                    'title': recording['title'],
                   }

        title = quote(collection['title']) + '/' + quote(recording['title'])
        return self.create_warcinfo(user, title, metadata, filename)

    def handle_download(self, user, coll, rec):
        collection = self.manager.get_collection(user, coll, rec)
        if not collection:
            self._raise_error(404, 'Collection not found',
                              id=coll)

        now = timestamp_now()

        name = collection['id']
        if rec != '*':
            rec_list = rec.split(',')
            if len(rec_list) == 1:
                name = rec
            else:
                name += '-' + rec
        else:<commit_msg>download metadata: set type explicitly 'collection' and 'recording' (#204)<commit_after>from pywb.utils.timeutils import timestamp_now
from pywb.utils.loaders import BlockLoader

from webagg.utils import StreamIter, chunk_encode_iter
from recorder.warcwriter import SimpleTempWARCWriter

from webrecorder.basecontroller import BaseController

from bottle import response
from six.moves.urllib.parse import quote
from six import iteritems
from collections import OrderedDict
import json


# ============================================================================
class DownloadController(BaseController):
    def __init__(self, app, jinja_env, manager, config):
        super(DownloadController, self).__init__(app, jinja_env, manager, config)
        self.paths = config['url_templates']
        self.download_filename = config['download_paths']['filename']
        self.warc_key_templ = config['warc_key_templ']

    def init_routes(self):
        @self.app.get('/<user>/<coll>/<rec>/$download')
        def logged_in_download_rec_warc(user, coll, rec):

            return self.handle_download(user, coll, rec)

        @self.app.get('/<user>/<coll>/$download')
        def logged_in_download_coll_warc(user, coll):

            return self.handle_download(user, coll, '*')

    def create_warcinfo(self, creator, title, metadata, filename):
        info = OrderedDict([
                ('software', 'Webrecorder Platform v2.5'),
                ('format', 'WARC File Format 1.0'),
                ('creator', creator),
                ('isPartOf', title),
                ('json-metadata', json.dumps(metadata)),
               ])

        wi_writer = SimpleTempWARCWriter()
        wi_writer.write_record(wi_writer.create_warcinfo_record(filename, info))
        return wi_writer.get_buffer()

    def create_coll_warcinfo(self, user, collection, filename=''):
        metadata = {'desc': collection['desc'],
                    'title': collection['title'],
                    'type': 'collection',
                   }

        title = quote(collection['title'])
        return self.create_warcinfo(user, title, metadata, filename)

    def create_rec_warcinfo(self, user, collection, recording, filename=''):
        metadata = {'pages': self.manager.list_pages(user,
                                                     collection['id'],
                                                     recording['id']),
                    'title': recording['title'],
                    'type': 'recording',
                   }

        title = quote(collection['title']) + '/' + quote(recording['title'])
        return self.create_warcinfo(user, title, metadata, filename)

    def handle_download(self, user, coll, rec):
        collection = self.manager.get_collection(user, coll, rec)
        if not collection:
            self._raise_error(404, 'Collection not found',
                              id=coll)

        now = timestamp_now()

        name = collection['id']
        if rec != '*':
            rec_list = rec.split(',')
            if len(rec_list) == 1:
                name = rec
            else:
                name += '-' + rec
        else:
{'fraction_non_alphanumeric': 0.1376799870571105, 'fraction_numerical': 0.0030739362562692122, 'mean_word_length': 2.3873972602739726, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1552851', 'n_tokens_mistral': 1670, 'n_tokens_neox': 1576, 'n_words': 365}