<commit_before>#
# Copyright (C) 2017 <NAME>
#           (C) 2015 <NAME> (MIT)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Imported from django-resumable by jean-philippe serafin 0.2.0-dev, (c) 2015
# MIT License (no file header or project LICENSE file, see setup.py)
#
"""
Control the resumable file API (javascript client side)
"""

import os
from datetime import datetime

import pytz

from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import FileSystemStorage
from django.conf import settings

class ResumableFile(object):
    """A resumable file controls getting pieces of a file"""
    upload_root = getattr(settings, 'UPLOAD_ROOT', None)
    name_template = "part_%(resumableChunkNumber)04d.tmp"

    def __init__(self, user, kwargs):
        self.kwargs = kwargs
        self.user = user

        if not self.upload_root:
            raise ImproperlyConfigured('You must set UPLOAD_ROOT in settings')

        self.storage = FileSystemStorage(location=self.upload_dir)

    @property
    def chunk_exists(self):
        """Checks if the requested chunk exists."""
        name = self.name_template % self.kwargs
        if not self.storage.exists(name):
            return False
        chunk_size = int(self.kwargs.get('resumableCurrentChunkSize'))
        return self.storage.size(name) == chunk_size

    def chunk_names(self):
        """Iterates over all stored chunks and yields their names."""
        try:
            return sorted(self.storage.listdir('')[1])
        except OSError:
            return []

    def chunks(self):
        """Yield the contents of every chunk, FileSystemStorage.save compatible"""
        for name in self.chunk_names():
            try:
                yield self.storage.open(name).read()
            except AttributeError:
                raise IOError("Couldn't read {}".format(name))

    def delete_chunks(self):
        """Remove every chunk (once complete)"""
        return [self.storage.delete(chunk) for chunk in self.chunk_names()]

    @property
    def filename(self):
        """Gets the filename."""
        filename = self.kwargs.get('resumableFilename')
        if '/' in filename:
            raise Exception('Invalid filename')
        return filename

    @property
    def upload_dir(self):
        """Gets the directory to save chunks to"""
        directory = os.path.join(self.upload_root, str(self.user.pk), self.filename)
        # When orginal uploads have been deleted, re-upload them.
        if os.path.islink(directory) and not os.path.exists(os.readlink(directory)):
            try:
                os.unlink(directory)
            except FileNotFoundError:
                # Catch when a different thread cleaned up just as we were about to.
                pass
        # This might be a file, or a directory, or a symbolic link at this point.
        return directory

    @property
    def is_complete(self):
        """Checks if all chunks are allready stored."""
        return os.path.isfile(self.upload_dir) or \
            self.kwargs['resumableTotalSize'] == self.size

    def process_chunk(self, _file):
        """Process the chunks of the given file"""
        if not self.chunk_exists:
            try:
                self.storage.save(self.name_template % self.kwargs, _file)
            except AttributeError:
                pass # Error saving file
            except (IOError, OSError) as err:

                raise IOError("Tried to save: {}, {}, {}".format(self.name_template, self.kwargs, self.name_template % self.kwargs))
                #pass # Existing file in the way

    def save_to(self, new_dir):
        """When saving all the chunks to a new directory"""
        filename = os.path.join(new_dir, self.filename)

        if os.path.islink(self.upload_dir):
            # This was previously uploaded and we can relink it.
            if not os.path.exists(filename):
                linkto = os.readlink(self.upload_dir)
                os.symlink(linkto, filename)
            return

        # Actually save the file using storage
        storage = FileSystemStorage(location=new_dir)
        storage.save(self.filename, self)

        # Delete all the chunks after use
        self.delete_chunks()

        # Sometimes there's arace condition if people double-click
        if os.path.isdir(self.upload_dir) and not os.listdir(self.upload_dir):
            os.rmdir(self.upload_dir)

        # Create a symlink for tracking and re-user
        if os.path.isfile(filename):
            os.symlink(filename, self.upload_dir)

    @property
    def size(self):
        """Gets chunks size."""
        size = 0
        for chunk in self.chunk_names():
            size += self.storage.size(chunk)
        return size

    @property
    def started(self):
        """Return the first modified datetime"""
        return self.get_times()[0]
<commit_msg>Lock uploads during save to prevent double-saving errors<commit_after>#
# Copyright (C) 2017 <NAME>
#           (C) 2015 <NAME> (MIT)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Imported from django-resumable by <NAME> 0.2.0-dev, (c) 2015
# MIT License (no file header or project LICENSE file, see setup.py)
#
"""
Control the resumable file API (javascript client side)
"""

import os
from datetime import datetime

import pytz

from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import FileSystemStorage
from django.conf import settings

class ResumableFile(object):
    """A resumable file controls getting pieces of a file"""
    upload_root = getattr(settings, 'UPLOAD_ROOT', None)
    name_template = "part_%(resumableChunkNumber)04d.tmp"

    def __init__(self, user, kwargs):
        self.kwargs = kwargs
        self.user = user

        if not self.upload_root:
            raise ImproperlyConfigured('You must set UPLOAD_ROOT in settings')

        self.storage = FileSystemStorage(location=self.upload_dir)

    @property
    def chunk_exists(self):
        """Checks if the requested chunk exists."""
        name = self.name_template % self.kwargs
        if not self.storage.exists(name):
            return False
        chunk_size = int(self.kwargs.get('resumableCurrentChunkSize'))
        return self.storage.size(name) == chunk_size

    def chunk_names(self):
        """Iterates over all stored chunks and yields their names."""
        try:
            return sorted(self.storage.listdir('')[1])
        except OSError:
            return []

    def chunks(self):
        """Yield the contents of every chunk, FileSystemStorage.save compatible"""
        for name in self.chunk_names():
            try:
                yield self.storage.open(name).read()
            except AttributeError:
                raise IOError("Couldn't read {}".format(name))

    def delete_chunks(self):
        """Remove every chunk (once complete)"""
        return [self.storage.delete(chunk) for chunk in self.chunk_names()]

    @property
    def filename(self):
        """Gets the filename."""
        filename = self.kwargs.get('resumableFilename')
        if '/' in filename:
            raise Exception('Invalid filename')
        return filename

    @property
    def upload_dir(self):
        """Gets the directory to save chunks to"""
        directory = os.path.join(self.upload_root, str(self.user.pk), self.filename)
        # When orginal uploads have been deleted, re-upload them.
        if os.path.islink(directory) and not os.path.exists(os.readlink(directory)):
            try:
                os.unlink(directory)
            except FileNotFoundError:
                # Catch when a different thread cleaned up just as we were about to.
                pass
        # This might be a file, or a directory, or a symbolic link at this point.
        return directory

    @property
    def is_complete(self):
        """Checks if all chunks are allready stored."""
        return os.path.isfile(self.upload_dir) or \
            self.kwargs['resumableTotalSize'] == self.size

    def process_chunk(self, _file):
        """Process the chunks of the given file"""
        if not self.chunk_exists:
            try:
                self.storage.save(self.name_template % self.kwargs, _file)
            except AttributeError:
                pass # Error saving file
            except (IOError, OSError) as err:

                raise IOError("Tried to save: {}, {}, {}".format(self.name_template, self.kwargs, self.name_template % self.kwargs))
                #pass # Existing file in the way

    def save_to(self, new_dir):
        """When saving all the chunks to a new directory"""
        filename = os.path.join(new_dir, self.filename)
        lock_file = filename + '.lock'

        if os.path.islink(self.upload_dir):
            # This was previously uploaded and we can relink it.
            if not os.path.exists(filename):
                linkto = os.readlink(self.upload_dir)
                os.symlink(linkto, filename)
            return

        if os.path.isfile(lock_file):
            return
        with open(lock_file, "w") as fhl:
            fhl.write("1")

        # Actually save the file using storage
        storage = FileSystemStorage(location=new_dir)
        storage.save(self.filename, self)

        # Delete all the chunks after use
        self.delete_chunks()

        # Sometimes there's arace condition if people double-click
        if os.path.isdir(self.upload_dir) and not os.listdir(self.upload_dir):
            os.rmdir(self.upload_dir)

        # Create a symlink for tracking and re-user
        if os.path.isfile(filename):
            os.symlink(filename, self.upload_dir)
        if os.path.isfile(lock_file):
            os.unlink(lock_file)

    @property
    def size(self):
        """Gets chunks size."""
        size = 0
        for chunk in self.chunk_names():
            size += self.storage.size(chunk)
        return size

    @property
    def started(self):
        """Return the first modified datetime"""
        return self.get_times()[0]

{'fraction_non_alphanumeric': 0.09322709163346614, 'fraction_numerical': 0.003806994245241257, 'mean_word_length': 2.44390243902439, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5133898', 'n_tokens_mistral': 3126, 'n_tokens_neox': 3029, 'n_words': 1033}