<commit_before># -*- coding: utf-8 -*-
import os
import re
import config
import zipfile
import crypto_util
import uuid
import tempfile
import subprocess
from cStringIO import StringIO

import logging
log = logging.getLogger(__name__)

from werkzeug import secure_filename

VALIDATE_FILENAME = re.compile(
    "^(?P<index>\d+)\-[a-z0-9-_]+?(?P<file_type>msg|doc\.zip|)\.gpg$").match


class PathException(Exception):

    '''An exception raised by `store.verify` when it encounters a bad path. A path
    can be bad when it is not absolute, not normalized, not within
    `config.STORE_DIR`, or doesn't match the filename format.
    '''
    pass


def verify(p):
    '''Assert that the path is absolute, normalized, inside `config.STORE_DIR`, and
    matches the filename format.
    '''
    if not os.path.isabs(config.STORE_DIR):
        raise PathException("config.STORE_DIR(%s) is not absolute" % (
            config.STORE_DIR, ))

    # os.path.abspath makes the path absolute and normalizes '/foo/../bar' to
    # '/bar', etc. We have to check that the path is normalized before checking
    # that it starts with the `config.STORE_DIR` or else a malicious actor could
    # append a bunch of '../../..' to access files outside of the store.
    if not p == os.path.abspath(p):
        raise PathException("The path is not absolute and/or normalized")

    # Check that the path p is in config.STORE_DIR
    if os.path.relpath(p, config.STORE_DIR).startswith('..'):
        raise PathException("Invalid directory %s" % (p, ))

    if os.path.isfile(p):
        filename = os.path.basename(p)
        ext = os.path.splitext(filename)[-1]
        if filename == '_FLAG':
            return True
        if ext != '.gpg':
            # if there's an extension, verify it's a GPG
            raise PathException("Invalid file extension %s" % (ext, ))
        if not VALIDATE_FILENAME(filename):
            raise PathException("Invalid filename %s" % (filename, ))


def path(*s):
    '''Get the normalized, absolute file path, within `config.STORE_DIR`.'''
    joined = os.path.join(os.path.abspath(config.STORE_DIR), *s)
    absolute = os.path.abspath(joined)
    verify(absolute)
    return absolute


def get_bulk_archive(filenames, zip_directory=''):
    zip_file = tempfile.NamedTemporaryFile(prefix='tmp_securedrop_bulk_dl_',
                                           dir=config.TEMP_DIR,
                                           delete=False)
    with zipfile.ZipFile(zip_file, 'w') as zip:
        for filename in filenames:
            verify(filename)
            zip.write(filename, arcname=os.path.join(
                zip_directory,
                os.path.basename(filename)
            ))
    return zip_file


def save_file_submission(sid, count, journalist_filename, filename, stream):
    sanitized_filename = secure_filename(filename)

    s = StringIO()
    with zipfile.ZipFile(s, 'w') as zf:
        zf.writestr(sanitized_filename, stream.read())
    s.seek(0)

    filename = "{0}-{1}-doc.zip.gpg".format(count, journalist_filename)
    file_loc = path(sid, filename)
    crypto_util.encrypt(s, config.JOURNALIST_KEY, file_loc)
    return filename


def save_message_submission(sid, count, journalist_filename, message):
    filename = "{0}-{1}-msg.gpg".format(count, journalist_filename)
    msg_loc = path(sid, filename)
    crypto_util.encrypt(message, config.JOURNALIST_KEY, msg_loc)<commit_msg>Use gzip instead of zipfile for uploaded files<commit_after># -*- coding: utf-8 -*-
import os
import re
import config
import zipfile
import crypto_util
import uuid
import tempfile
import subprocess
from cStringIO import StringIO
import gzip

import logging
log = logging.getLogger(__name__)

from werkzeug import secure_filename

from secure_tempfile import SecureTemporaryFile

VALIDATE_FILENAME = re.compile("^(?P<index>\d+)\-[a-z0-9-_]+?(?P<file_type>msg|doc\.(gz|zip))\.gpg$").match

class PathException(Exception):

    '''An exception raised by `store.verify` when it encounters a bad path. A path
    can be bad when it is not absolute, not normalized, not within
    `config.STORE_DIR`, or doesn't match the filename format.
    '''
    pass


def verify(p):
    '''Assert that the path is absolute, normalized, inside `config.STORE_DIR`, and
    matches the filename format.
    '''
    if not os.path.isabs(config.STORE_DIR):
        raise PathException("config.STORE_DIR(%s) is not absolute" % (
            config.STORE_DIR, ))

    # os.path.abspath makes the path absolute and normalizes '/foo/../bar' to
    # '/bar', etc. We have to check that the path is normalized before checking
    # that it starts with the `config.STORE_DIR` or else a malicious actor could
    # append a bunch of '../../..' to access files outside of the store.
    if not p == os.path.abspath(p):
        raise PathException("The path is not absolute and/or normalized")

    # Check that the path p is in config.STORE_DIR
    if os.path.relpath(p, config.STORE_DIR).startswith('..'):
        raise PathException("Invalid directory %s" % (p, ))

    if os.path.isfile(p):
        filename = os.path.basename(p)
        ext = os.path.splitext(filename)[-1]
        if filename == '_FLAG':
            return True
        if ext != '.gpg':
            # if there's an extension, verify it's a GPG
            raise PathException("Invalid file extension %s" % (ext, ))
        if not VALIDATE_FILENAME(filename):
            raise PathException("Invalid filename %s" % (filename, ))


def path(*s):
    '''Get the normalized, absolute file path, within `config.STORE_DIR`.'''
    joined = os.path.join(os.path.abspath(config.STORE_DIR), *s)
    absolute = os.path.abspath(joined)
    verify(absolute)
    return absolute


def get_bulk_archive(filenames, zip_directory=''):
    zip_file = tempfile.NamedTemporaryFile(prefix='tmp_securedrop_bulk_dl_',
                                           dir=config.TEMP_DIR,
                                           delete=False)
    with zipfile.ZipFile(zip_file, 'w') as zip:
        for filename in filenames:
            verify(filename)
            zip.write(filename, arcname=os.path.join(
                zip_directory,
                os.path.basename(filename)
            ))
    return zip_file


def save_file_submission(sid, count, journalist_filename, filename, stream):
    sanitized_filename = secure_filename(filename)

    # We store file submissions in a .gz file for two reasons:
    #
    # 1. Downloading large files over Tor is very slow. If we can
    # compress the file, we can speed up future downloads.
    #
    # 2. We want to record the original filename because it might be
    # useful, either for context about the content of the submission
    # or for figuring out which application should be used to open
    # it. However, we'd like to encrypt that info and have the
    # decrypted file automatically have the name of the original
    # file. Given various usability constraints in GPG and Tails, this
    # is the most user-friendly way we have found to do this.

    encrypted_file_name = "{0}-{1}-doc.gz.gpg".format(count, journalist_filename)
    encrypted_file_path = path(sid, encrypted_file_name)
    with SecureTemporaryFile("/tmp") as stf:
        with gzip.GzipFile(filename=sanitized_filename, mode='wb', fileobj=stf) as gzf:
            # Buffer the stream into the gzip file to avoid excessive
            # memory consumption
            while True:
                buf = stream.read(1024 * 8)
                if not buf:
                    break
                gzf.write(buf)

        crypto_util.encrypt(stf, config.JOURNALIST_KEY, encrypted_file_path)

    return encrypted_file_name


def save_message_submission(sid, count, journalist_filename, message):
    filename = "{0}-{1}-msg.gpg".format(count, journalist_filename)
    msg_loc = path(sid, filename)
    crypto_util.encrypt(message, config.JOURNALIST_KEY, msg_loc)
{'fraction_non_alphanumeric': 0.10401712630651051, 'fraction_numerical': 0.003022289384208538, 'mean_word_length': 2.812770043206913, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3150286', 'n_tokens_mistral': 2403, 'n_tokens_neox': 2335, 'n_words': 713}