<commit_before>try:
    from staticfiles import finders
    from staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage
except ImportError:
    from django.contrib.staticfiles import finders
    from django.contrib.staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage
from django.core.exceptions import ImproperlyConfigured
from django.core.files.storage import get_storage_class
from django.utils.functional import LazyObject

from pipeline.conf import settings


class BaseFinderStorage(StaticFilesStorage):
    finders = None

    def __init__(self, finders=None, *args, **kwargs):
        if finders is not None:
            self.finders = finders
        if self.finders is None:
            raise ImproperlyConfigured("The storage %r doesn't have a finders class assigned." % self.__class__)
        super(BaseFinderStorage, self).__init__(*args, **kwargs)

    def path(self, name):
        path = self.finders.find(name)
        if not path:
            path = super(BaseFinderStorage, self).path(name)
        return path

    def exists(self, name):
        exists = self.finders.find(name) != None
        if not exists:
            exists = super(BaseFinderStorage, self).exists(name)
        return exists


class PipelineFinderStorage(BaseFinderStorage):
    finders = finders


class PipelineStorage(PipelineFinderStorage):
    def post_process(self, paths, dry_run=False, **options):
        from pipeline.packager import Packager
        if dry_run:
            return []
        
        packager = Packager()
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            output_file = packager.pack_stylesheets(package)
            paths.append(output_file)
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            output_file = packager.pack_javascripts(package)
            paths.append(output_file)
        return super(PipelineStorage, self).post_process(paths, dry_run, **options)


class PipelineCachedStorage(PipelineStorage, CachedStaticFilesStorage):
    pass


class DefaultStorage(LazyObject):
    def _setup(self):
        self._wrapped = get_storage_class(settings.PIPELINE_STORAGE)()


storage = DefaultStorage()<commit_msg>remove packages files from processing<commit_after>try:
    from staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage
except ImportError:
    from django.contrib.staticfiles.storage import CachedStaticFilesStorage, StaticFilesStorage

from django.core.files.storage import get_storage_class
from django.utils.functional import LazyObject

from pipeline.conf import settings


class PipelineStorage(StaticFilesStorage):
    def post_process(self, paths, dry_run=False, **options):
        from pipeline.packager import Packager
        if dry_run:
            return []

        packager = Packager()
        for package_name in packager.packages['css']:
            package = packager.package_for('css', package_name)
            for path in package['paths']:
                if path in paths:
                    paths.remove(path)
            output_file = packager.pack_stylesheets(package)
            paths.append(output_file)
        for package_name in packager.packages['js']:
            package = packager.package_for('js', package_name)
            for path in package['paths']:
                if path in paths:
                    paths.remove(path)
            output_file = packager.pack_javascripts(package)
            paths.append(output_file)
        return super(PipelineStorage, self).post_process(paths, dry_run, **options)


class PipelineCachedStorage(PipelineStorage, CachedStaticFilesStorage):
    pass


class DefaultStorage(LazyObject):
    def _setup(self):
        self._wrapped = get_storage_class(settings.PIPELINE_STORAGE)()


storage = DefaultStorage()
{'fraction_non_alphanumeric': 0.07924815849631699, 'fraction_numerical': 0.0, 'mean_word_length': 3.1893617021276595, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '615147', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1039, 'n_words': 254}