<commit_before>import os
import cgi
import json
import shutil
import hashlib
import zipfile
import datetime
import collections

from . import util
from . import config
from . import tempdir as tempfile

log = config.log

DEFAULT_HASH_ALG='sha384'

def move_file(path, target_path):
    target_dir = os.path.dirname(target_path)
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    shutil.move(path, target_path)

def move_form_file_field_into_cas(file_field):
    """
    Given a file form field, move the (downloaded, tempdir-stored) file into the CAS.

    Requires an augmented file field; see upload.process_upload() for details.
    """

    if not file_field.hash or not file_field.path:
        raise Exception("Field is not a file field with hash and path")

    base   = config.get_item('persistent', 'data_path')
    cas    = util.path_from_hash(file_field.hash)
    move_file(file_field.path, os.path.join(base, cas))

def hash_file_formatted(path, hash_alg=None):
    """
    Return the scitran-formatted hash of a file, specified by path.

    REVIEW: if there's an intelligent io-copy in python stdlib, I missed it. This uses an arbitrary buffer size :/
    """

    hash_alg = hash_alg or DEFAULT_HASH_ALG
    hasher = hashlib.new(hash_alg)

    BUF_SIZE = 65536

    with open(path, 'rb') as f:
        while True:
            data = f.read(BUF_SIZE)
            if not data:
                break
            hasher.update(data)

    return util.format_hash(hash_alg, hasher.hexdigest())


class FileStoreException(Exception):
    pass

class HashingFile(file):
    def __init__(self, file_path, hash_alg):
        super(HashingFile, self).__init__(file_path, "wb")
        self.hash_alg = hashlib.new(hash_alg)
        self.hash_name = hash_alg

    def write(self, data):
        self.hash_alg.update(data)
        return file.write(self, data)

    def get_hash(self):
        return self.hash_alg.hexdigest()

    def get_formatted_hash(self):
        return util.format_hash(self.hash_name, self.get_hash())

ParsedFile = collections.namedtuple('ParsedFile', ['info', 'path'])

def process_form(request, hash_alg=None):
    """
    Some workarounds to make webapp2 process forms in an intelligent way, and hash files we process.
    Could subsume getHashingFieldStorage.

    This is a bit arcane, and deals with webapp2 / uwsgi / python complexities. Ask a team member, sorry!

    Returns the processed form, and the tempdir it was stored in.
    Keep tempdir in scope until you don't need it anymore; it will be deleted on GC.
    """

    hash_alg = hash_alg or DEFAULT_HASH_ALG

    # Store form file fields in a tempdir
    tempdir = tempfile.TemporaryDirectory(prefix='.tmp', dir=config.get_item('persistent', 'data_path'))
    tempdir_path = tempdir.name

    # Deep vodoo; docs?
    env = request.environ.copy()
    env.setdefault('CONTENT_LENGTH', '0')
    env['QUERY_STRING'] = ''

    # Wall-clock warning: despite its name, getHashingFieldStorage will actually process the entire form to disk. This involves recieving the entire upload stream and storing any files in the tempdir.
    form = getHashingFieldStorage(tempdir.name, DEFAULT_HASH_ALG)(
        fp=request.body_file, environ=env, keep_blank_values=True
    )

    return (form, tempdir)

def getHashingFieldStorage(upload_dir, hash_alg):
    class HashingFieldStorage(cgi.FieldStorage):
        bufsize = 2**20

        def make_file(self, binary=None):
            # Sanitize form's filename (read: prevent malicious escapes, bad characters, etc)
            self.filename = os.path.basename(self.filename)
            # self.filename = util.sanitize_string_to_filename(self.filename)

            self.open_file = HashingFile(os.path.join(upload_dir, self.filename), hash_alg)
            return self.open_file

        # override private method __write of superclass FieldStorage
        # _FieldStorage__file is the private variable __file of the same class
        def _FieldStorage__write(self, line):
            # pylint: disable=access-member-before-definition
            if self._FieldStorage__file is not None:
                # use the make_file method only if the form includes a filename
                # e.g. do not create a file and a hash for the form metadata.
                if self.filename:
                    self.file = self.make_file('')

                    self.file.write(self._FieldStorage__file.getvalue())
                self._FieldStorage__file = None
            self.file.write(line)

        def get_hash(self):
            return self.open_file.get_hash()

    return HashingFieldStorage


class FileStore(object):
    """This class provides and interface for file uploads.
    To perform an upload the client of the class should follow these steps:

    1) initialize the request
    2) save a temporary file
    3) check identical
    4) move the temporary file to its destination

    The operations could be safely interleaved with other actions like permission checks or database updates.
    """

    def __init__(self, request, dest_path, filename=None, hash_alg=DEFAULT_HASH_ALG):
        self.body = request.body_file
        self.environ = request.environ.copy()
        self.environ.setdefault('CONTENT_LENGTH', '0')
        self.environ['QUERY_STRING'] = ''
        self.hash_alg = hash_alg
        start_time = datetime.datetime.utcnow()
        if request.content_type == 'multipart/form-data':
            self._save_multipart_file(dest_path, hash_alg)
            self.payload = request.POST.mixed()
        else:
            self.payload = request.POST.mixed()
            self.filename = filename or self.payload.get('filename')
            self._save_body_file(dest_path, filename, hash_alg)
        self.mimetype = util.guess_mimetype(self.filename)
        self.path = os.path.join(dest_path, self.filename)
        self.duration = datetime.datetime.utcnow() - start_time
        # the hash format is:
        # <version>-<hashing algorithm>-<actual hash>
        # version will track changes on hash related methods like for example how we check for identical files.
        self.hash = util.format_hash(hash_alg, self.received_file.get_hash())
        self.size = os.path.getsize(self.path)

    def _save_multipart_file(self, dest_path, hash_alg):
        form = getHashingFieldStorage(dest_path, hash_alg)(fp=self.body, environ=self.environ, keep_blank_values=True)

        self.received_file = form['file'].file
        self.filename = os.path.basename(form['file'].filename)
        self.tags = json.loads(form['tags'].file.getvalue()) if 'tags' in form else None
        self.metadata = json.loads(form['metadata'].file.getvalue()) if 'metadata' in form else None

    def _save_body_file(self, dest_path, filename, hash_alg):
        if not filename:
            raise FileStoreException('filename is required for body uploads')
        self.filename = os.path.basename(filename)
        self.received_file = HashingFile(os.path.join(dest_path, filename), hash_alg)
        for chunk in iter(lambda: self.body.read(2**20), ''):
            self.received_file.write(chunk)
        self.tags = None
        self.metadata = None

    def move_file(self, target_path):
        move_file(self.path, target_path)
        self.path = target_path

def identical(hash_0, path_0, hash_1, path_1):
    if zipfile.is_zipfile(path_0) and zipfile.is_zipfile(path_1):
        with zipfile.ZipFile(path_0) as zf1, zipfile.ZipFile(path_1) as zf2:
            zf1_infolist = sorted(zf1.infolist(), key=lambda zi: zi.filename)
            zf2_infolist = sorted(zf2.infolist(), key=lambda zi: zi.filename)
            if zf1.comment != zf2.comment:
                return False
            if len(zf1_infolist) != len(zf2_infolist):
                return False
            for zii, zij in zip(zf1_infolist, zf2_infolist):
                if zii.CRC != zij.CRC:
                    return False
            else:
                return True
    else:
        return hash_0 == hash_1

# TODO: Hopefully deprecated by unification branch
class MultiFileStore(object):
    """This class provides and interface for file uploads.
    """

    def __init__(self, request, dest_path, filename=None, hash_alg=DEFAULT_HASH_ALG):
        self.body = request.body_file
        self.environ = request.environ.copy()
        self.environ.setdefault('CONTENT_LENGTH', '0')
        self.environ['QUERY_STRING'] = ''
        self.hash_alg = hash_alg
        self.files = {}
        self._save_multipart_files(dest_path, hash_alg)
        self.payload = request.POST.mixed()

    def _save_multipart_files(self, dest_path, hash_alg):
        form = getHashingFieldStorage(dest_path, hash_alg)(fp=self.body, environ=self.environ, keep_blank_values=True)
        self.metadata = json.loads(form['metadata'].file.getvalue()) if 'metadata' in form else None
        for field in form:
            if form[field].filename:
                filename = os.path.basename(form[field].filename)
                self.files[filename] = ParsedFile(
                    {
                        'hash': util.format_hash(hash_alg, form[field].file.get_hash()),
                        'size': os.path.getsize(os.path.join(dest_path, filename)),
                        'mimetype': util.guess_mimetype(filename)
                    }, os.path.join(dest_path, filename))


# File extension --> scitran file type detection hueristics.
# Listed in precendence order.
FILE_EXTENSIONS = [
    # Scientific file types
    {'bval':         [ '.bval', '.bvecs', '.bvals' ]},
    {'bvec':         [ '.bvec' ]},
    {'dicom':        [ '.dcm' ]},
    {'gephysio':     [ '.gephysio.zip' ]},
    {'nifti':        [ '.nii.gz', '.nii' ]},
    {'pfile':        [ '.7.gz', '.7' ]},
    {'qa':           [ '.qa.png', '.qa.json' ]},

    # Basic file types
    {'archive':      [ '.zip', '.tbz2', '.tar.gz', '.tbz', '.tar.bz2', '.tgz', '.tar', '.txz', '.tar.xz' ]},
    {'document':     [ '.docx', '.doc' ]},
    {'image':        [ '.jpg', '.tif', '.jpeg', '.gif', '.bmp', '.png', '.tiff' ]},
    {'pdf':          [ '.pdf' ]},
    {'presentation': [ '.ppt', '.pptx' ]},
    {'source code':  [ '.c', '.py', '.cpp', '.js', '.m', '.json', '.java', '.html', '.htm', '.php', '.css'  ]},
    {'spredsheet':   [ '.xls', '.xlsx' ]},
    {'tabular data': [ '.csv.gz', '.csv' ]},
    {'text':         [ '.txt' ]},
    {'video':        [ '.mpeg', '.mpg', '.mov', '.mp4', '.m4v', '.MTS' ]},
]

def guess_type_from_filename(filename):
    for x in FILE_EXTENSIONS:
        key = x.keys()[0]
        for extension in x[key]:
            if filename.endswith(extension):
                return key
    return None<commit_msg>Small corrections and a couple more types<commit_after>import os
import cgi
import json
import shutil
import hashlib
import zipfile
import datetime
import collections

from . import util
from . import config
from . import tempdir as tempfile

log = config.log

DEFAULT_HASH_ALG='sha384'

def move_file(path, target_path):
    target_dir = os.path.dirname(target_path)
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    shutil.move(path, target_path)

def move_form_file_field_into_cas(file_field):
    """
    Given a file form field, move the (downloaded, tempdir-stored) file into the CAS.

    Requires an augmented file field; see upload.process_upload() for details.
    """

    if not file_field.hash or not file_field.path:
        raise Exception("Field is not a file field with hash and path")

    base   = config.get_item('persistent', 'data_path')
    cas    = util.path_from_hash(file_field.hash)
    move_file(file_field.path, os.path.join(base, cas))

def hash_file_formatted(path, hash_alg=None):
    """
    Return the scitran-formatted hash of a file, specified by path.

    REVIEW: if there's an intelligent io-copy in python stdlib, I missed it. This uses an arbitrary buffer size :/
    """

    hash_alg = hash_alg or DEFAULT_HASH_ALG
    hasher = hashlib.new(hash_alg)

    BUF_SIZE = 65536

    with open(path, 'rb') as f:
        while True:
            data = f.read(BUF_SIZE)
            if not data:
                break
            hasher.update(data)

    return util.format_hash(hash_alg, hasher.hexdigest())


class FileStoreException(Exception):
    pass

class HashingFile(file):
    def __init__(self, file_path, hash_alg):
        super(HashingFile, self).__init__(file_path, "wb")
        self.hash_alg = hashlib.new(hash_alg)
        self.hash_name = hash_alg

    def write(self, data):
        self.hash_alg.update(data)
        return file.write(self, data)

    def get_hash(self):
        return self.hash_alg.hexdigest()

    def get_formatted_hash(self):
        return util.format_hash(self.hash_name, self.get_hash())

ParsedFile = collections.namedtuple('ParsedFile', ['info', 'path'])

def process_form(request, hash_alg=None):
    """
    Some workarounds to make webapp2 process forms in an intelligent way, and hash files we process.
    Could subsume getHashingFieldStorage.

    This is a bit arcane, and deals with webapp2 / uwsgi / python complexities. Ask a team member, sorry!

    Returns the processed form, and the tempdir it was stored in.
    Keep tempdir in scope until you don't need it anymore; it will be deleted on GC.
    """

    hash_alg = hash_alg or DEFAULT_HASH_ALG

    # Store form file fields in a tempdir
    tempdir = tempfile.TemporaryDirectory(prefix='.tmp', dir=config.get_item('persistent', 'data_path'))
    tempdir_path = tempdir.name

    # Deep vodoo; docs?
    env = request.environ.copy()
    env.setdefault('CONTENT_LENGTH', '0')
    env['QUERY_STRING'] = ''

    # Wall-clock warning: despite its name, getHashingFieldStorage will actually process the entire form to disk. This involves recieving the entire upload stream and storing any files in the tempdir.
    form = getHashingFieldStorage(tempdir.name, DEFAULT_HASH_ALG)(
        fp=request.body_file, environ=env, keep_blank_values=True
    )

    return (form, tempdir)

def getHashingFieldStorage(upload_dir, hash_alg):
    class HashingFieldStorage(cgi.FieldStorage):
        bufsize = 2**20

        def make_file(self, binary=None):
            # Sanitize form's filename (read: prevent malicious escapes, bad characters, etc)
            self.filename = os.path.basename(self.filename)
            # self.filename = util.sanitize_string_to_filename(self.filename)

            self.open_file = HashingFile(os.path.join(upload_dir, self.filename), hash_alg)
            return self.open_file

        # override private method __write of superclass FieldStorage
        # _FieldStorage__file is the private variable __file of the same class
        def _FieldStorage__write(self, line):
            # pylint: disable=access-member-before-definition
            if self._FieldStorage__file is not None:
                # use the make_file method only if the form includes a filename
                # e.g. do not create a file and a hash for the form metadata.
                if self.filename:
                    self.file = self.make_file('')

                    self.file.write(self._FieldStorage__file.getvalue())
                self._FieldStorage__file = None
            self.file.write(line)

        def get_hash(self):
            return self.open_file.get_hash()

    return HashingFieldStorage


class FileStore(object):
    """This class provides and interface for file uploads.
    To perform an upload the client of the class should follow these steps:

    1) initialize the request
    2) save a temporary file
    3) check identical
    4) move the temporary file to its destination

    The operations could be safely interleaved with other actions like permission checks or database updates.
    """

    def __init__(self, request, dest_path, filename=None, hash_alg=DEFAULT_HASH_ALG):
        self.body = request.body_file
        self.environ = request.environ.copy()
        self.environ.setdefault('CONTENT_LENGTH', '0')
        self.environ['QUERY_STRING'] = ''
        self.hash_alg = hash_alg
        start_time = datetime.datetime.utcnow()
        if request.content_type == 'multipart/form-data':
            self._save_multipart_file(dest_path, hash_alg)
            self.payload = request.POST.mixed()
        else:
            self.payload = request.POST.mixed()
            self.filename = filename or self.payload.get('filename')
            self._save_body_file(dest_path, filename, hash_alg)
        self.mimetype = util.guess_mimetype(self.filename)
        self.path = os.path.join(dest_path, self.filename)
        self.duration = datetime.datetime.utcnow() - start_time
        # the hash format is:
        # <version>-<hashing algorithm>-<actual hash>
        # version will track changes on hash related methods like for example how we check for identical files.
        self.hash = util.format_hash(hash_alg, self.received_file.get_hash())
        self.size = os.path.getsize(self.path)

    def _save_multipart_file(self, dest_path, hash_alg):
        form = getHashingFieldStorage(dest_path, hash_alg)(fp=self.body, environ=self.environ, keep_blank_values=True)

        self.received_file = form['file'].file
        self.filename = os.path.basename(form['file'].filename)
        self.tags = json.loads(form['tags'].file.getvalue()) if 'tags' in form else None
        self.metadata = json.loads(form['metadata'].file.getvalue()) if 'metadata' in form else None

    def _save_body_file(self, dest_path, filename, hash_alg):
        if not filename:
            raise FileStoreException('filename is required for body uploads')
        self.filename = os.path.basename(filename)
        self.received_file = HashingFile(os.path.join(dest_path, filename), hash_alg)
        for chunk in iter(lambda: self.body.read(2**20), ''):
            self.received_file.write(chunk)
        self.tags = None
        self.metadata = None

    def move_file(self, target_path):
        move_file(self.path, target_path)
        self.path = target_path

def identical(hash_0, path_0, hash_1, path_1):
    if zipfile.is_zipfile(path_0) and zipfile.is_zipfile(path_1):
        with zipfile.ZipFile(path_0) as zf1, zipfile.ZipFile(path_1) as zf2:
            zf1_infolist = sorted(zf1.infolist(), key=lambda zi: zi.filename)
            zf2_infolist = sorted(zf2.infolist(), key=lambda zi: zi.filename)
            if zf1.comment != zf2.comment:
                return False
            if len(zf1_infolist) != len(zf2_infolist):
                return False
            for zii, zij in zip(zf1_infolist, zf2_infolist):
                if zii.CRC != zij.CRC:
                    return False
            else:
                return True
    else:
        return hash_0 == hash_1

# TODO: Hopefully deprecated by unification branch
class MultiFileStore(object):
    """This class provides and interface for file uploads.
    """

    def __init__(self, request, dest_path, filename=None, hash_alg=DEFAULT_HASH_ALG):
        self.body = request.body_file
        self.environ = request.environ.copy()
        self.environ.setdefault('CONTENT_LENGTH', '0')
        self.environ['QUERY_STRING'] = ''
        self.hash_alg = hash_alg
        self.files = {}
        self._save_multipart_files(dest_path, hash_alg)
        self.payload = request.POST.mixed()

    def _save_multipart_files(self, dest_path, hash_alg):
        form = getHashingFieldStorage(dest_path, hash_alg)(fp=self.body, environ=self.environ, keep_blank_values=True)
        self.metadata = json.loads(form['metadata'].file.getvalue()) if 'metadata' in form else None
        for field in form:
            if form[field].filename:
                filename = os.path.basename(form[field].filename)
                self.files[filename] = ParsedFile(
                    {
                        'hash': util.format_hash(hash_alg, form[field].file.get_hash()),
                        'size': os.path.getsize(os.path.join(dest_path, filename)),
                        'mimetype': util.guess_mimetype(filename)
                    }, os.path.join(dest_path, filename))


# File extension --> scitran file type detection hueristics.
# Listed in precendence order.
FILE_EXTENSIONS = [
    # Scientific file types
    {'bval':         [ '.bval', '.bvals' ]},
    {'bvec':         [ '.bvec', '.bvecs' ]},
    {'dicom':        [ '.dcm', '.dcm.zip', '.dicom.zip' ]},
    {'gephysio':     [ '.gephysio.zip' ]},
    {'nifti':        [ '.nii.gz', '.nii' ]},
    {'pfile':        [ '.7.gz', '.7' ]},
    {'qa':           [ '.qa.png', '.qa.json' ]},

    # Basic file types
    {'archive':      [ '.zip', '.tbz2', '.tar.gz', '.tbz', '.tar.bz2', '.tgz', '.tar', '.txz', '.tar.xz' ]},
    {'document':     [ '.docx', '.doc' ]},
    {'image':        [ '.jpg', '.tif', '.jpeg', '.gif', '.bmp', '.png', '.tiff' ]},
    {'markup':       [ '.html', '.htm' ]},
    {'pdf':          [ '.pdf' ]},
    {'presentation': [ '.ppt', '.pptx' ]},
    {'source code':  [ '.c', '.py', '.cpp', '.js', '.m', '.json', '.java', '.php', '.css' ]},
    {'spreadsheet':  [ '.xls', '.xlsx' ]},
    {'tabular data': [ '.csv.gz', '.csv' ]},
    {'text':         [ '.txt' ]},
    {'video':        [ '.mpeg', '.mpg', '.mov', '.mp4', '.m4v', '.MTS' ]},
]

def guess_type_from_filename(filename):
    for x in FILE_EXTENSIONS:
        key = x.keys()[0]
        for extension in x[key]:
            if filename.endswith(extension):
                return key
    return None
{'fraction_non_alphanumeric': 0.11811974545961261, 'fraction_numerical': 0.004830693483208695, 'mean_word_length': 2.644211238997969, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6663910', 'n_tokens_mistral': 6741, 'n_tokens_neox': 6559, 'n_words': 1684}