<commit_before>import os
from optparse import make_option

from django.core.management.base import BaseCommand, NoArgsCommand
from django.core.files import File as DjangoFile

from filer.models.foldermodels import Folder
from filer.models.filemodels import File
from filer.models.imagemodels import Image

from filer.settings import FILER_IS_PUBLIC_DEFAULT

class FileImporter(object):
    def __init__(self, * args, **kwargs):
        self.path = kwargs.get('path')
        self.verbosity = int(kwargs.get('verbosity', 1))
        self.file_created = 0
        self.image_created = 0
        self.folder_created = 0
    
    def import_file(self, file, folder):
        """
        Create a File or an Image into the given folder
        
        """
        try:
            iext = os.path.splitext(file.name)[1].lower()
        except:
            iext = ''
        if iext in ['.jpg', '.jpeg', '.png', '.gif']:
            obj, created = Image.objects.get_or_create(
                                original_filename=file.name,
                                file=file,
                                folder=folder,
                                is_public=FILER_IS_PUBLIC_DEFAULT)
            if created:
                self.image_created += 1
        else:
            obj, created = File.objects.get_or_create(
                                original_filename=file.name,
                                file=file,
                                folder=folder,
                                is_public=FILER_IS_PUBLIC_DEFAULT)
            if created:
                self.file_created += 1
        if self.verbosity >= 2:
            print u"file_created #%s / image_created #%s -- file : %s -- created : %s" % (self.file_created,
                                                        self.image_created,
                                                        obj, created)  
        return obj
    
    def get_or_create_folder(self, folder_names):
        """
        Gets or creates a Folder based the list of folder names in hierarchical 
        order (like breadcrumbs).
        
        get_or_create_folder(['root', 'subfolder', 'subsub folder'])
        
        creates the folders with correct parent relations and returns the 
        'subsub folder' instance.
        """
        if not len(folder_names):
            return None
        current_parent = None
        for folder_name in folder_names:
            current_parent, created = Folder.objects.get_or_create(name=folder_name, parent=current_parent)
            if created:
                self.folder_created += 1
                if self.verbosity >= 2:
                    print u"folder_created #%s folder : %s -- created : %s" % (self.folder_created,
                                                                               current_parent, created) 
        return current_parent
    
    def walker(self, path=None):
        """
        This method walk a directory structure and create the
        Folders and Files as they appear.
        """
        path = path or self.path
        # prevent trailing slashes and other inconsistencies on path.
        # cast to unicode so that os.walk returns path names in unicode
        # (prevents encoding/decoding errors)
        path = unicode(os.path.normpath(path))
        if self.verbosity >= 1:
            print u"Import the folders and files in %s" % path
        root_folder_name = os.path.basename(path)
        for root, dirs, files in os.walk(path):
            rel_folders = root.partition(path)[2].strip(os.path.sep).split(os.path.sep)
            while '' in rel_folders:
                rel_folders.remove('')
            folder_names = [root_folder_name] + rel_folders
            folder = self.get_or_create_folder(folder_names)
            for file in files:
                dj_file = DjangoFile(open(os.path.join(root, file)),
                                     name=file)
                self.import_file(file=dj_file, folder=folder)
        if self.verbosity >= 1:
            print u"folder_created #%s / file_created #%s / image_created #%s "% (self.folder_created,
                                                                                 self.file_created,
                                                                                 self.image_created)

class Command(NoArgsCommand):
    option_list = BaseCommand.option_list + (
        make_option('--path',
            action='store',
            dest='path',
            default=False,
            help='Import files located in the path into django-filer'),
        )

    def handle_noargs(self, **options):
        file_importer = FileImporter(**options)
        file_importer.walker()<commit_msg>Add an option to the management command to enable the user to import the directory structure into an existing folder<commit_after>import os
from optparse import make_option

from django.core.management.base import BaseCommand, NoArgsCommand
from django.core.files import File as DjangoFile

from filer.models.foldermodels import Folder
from filer.models.filemodels import File
from filer.models.imagemodels import Image

from filer.settings import FILER_IS_PUBLIC_DEFAULT

class FileImporter(object):
    def __init__(self, * args, **kwargs):
        self.path = kwargs.get('path')
        self.base_folder = kwargs.get('base_folder')
        self.verbosity = int(kwargs.get('verbosity', 1))
        self.file_created = 0
        self.image_created = 0
        self.folder_created = 0
    
    def import_file(self, file, folder):
        """
        Create a File or an Image into the given folder
        
        """
        try:
            iext = os.path.splitext(file.name)[1].lower()
        except:
            iext = ''
        if iext in ['.jpg', '.jpeg', '.png', '.gif']:
            obj, created = Image.objects.get_or_create(
                                original_filename=file.name,
                                file=file,
                                folder=folder,
                                is_public=FILER_IS_PUBLIC_DEFAULT)
            if created:
                self.image_created += 1
        else:
            obj, created = File.objects.get_or_create(
                                original_filename=file.name,
                                file=file,
                                folder=folder,
                                is_public=FILER_IS_PUBLIC_DEFAULT)
            if created:
                self.file_created += 1
        if self.verbosity >= 2:
            print u"file_created #%s / image_created #%s -- file : %s -- created : %s" % (self.file_created,
                                                        self.image_created,
                                                        obj, created)  
        return obj
    
    def get_or_create_folder(self, folder_names):
        """
        Gets or creates a Folder based the list of folder names in hierarchical 
        order (like breadcrumbs).
        
        get_or_create_folder(['root', 'subfolder', 'subsub folder'])
        
        creates the folders with correct parent relations and returns the 
        'subsub folder' instance.
        """
        if not len(folder_names):
            return None
        current_parent = None
        for folder_name in folder_names:
            current_parent, created = Folder.objects.get_or_create(name=folder_name, parent=current_parent)
            if created:
                self.folder_created += 1
                if self.verbosity >= 2:
                    print u"folder_created #%s folder : %s -- created : %s" % (self.folder_created,
                                                                               current_parent, created) 
        return current_parent
    
    def walker(self, path=None, base_folder=None):
        """
        This method walk a directory structure and create the
        Folders and Files as they appear.
        """
        path = path or self.path
        base_folder = base_folder or self.base_folder
        # prevent trailing slashes and other inconsistencies on path.
        # cast to unicode so that os.walk returns path names in unicode
        # (prevents encoding/decoding errors)
        path = unicode(os.path.normpath(path))
        if base_folder:
            base_folder = unicode(os.path.normpath(base_folder))
            print u"The directory structure will be imported in %s" % (base_folder,)
        if self.verbosity >= 1:
            print u"Import the folders and files in %s" % (path,)
        root_folder_name = os.path.basename(path)
        for root, dirs, files in os.walk(path):
            rel_folders = root.partition(path)[2].strip(os.path.sep).split(os.path.sep)
            while '' in rel_folders:
                rel_folders.remove('')
            if base_folder:
                folder_names = base_folder.split('/') + [root_folder_name] + rel_folders
            else:
                folder_names = [root_folder_name] + rel_folders
            folder = self.get_or_create_folder(folder_names)
            for file in files:
                dj_file = DjangoFile(open(os.path.join(root, file)),
                                     name=file)
                self.import_file(file=dj_file, folder=folder)
        if self.verbosity >= 1:
            print u"folder_created #%s / file_created #%s / image_created #%s "% (self.folder_created,
                                                                                 self.file_created,
                                                                                 self.image_created)

class Command(NoArgsCommand):
    """
    Import directory structure into the filer ::
    
        manage.py --path=/tmp/assets/images
        manage.py --path=/tmp/assets/news --folder=images
        
    """
    
    option_list = BaseCommand.option_list + (
        make_option('--path',
            action='store',
            dest='path',
            default=False,
            help='Import files located in the path into django-filer'),
        make_option('--folder',
            action='store',
            dest='base_folder',
            default=False,
            help='Specify the destination folder in which the directory structure should be imported'),
        )

    def handle_noargs(self, **options):
        file_importer = FileImporter(**options)
        file_importer.walker()
{'fraction_non_alphanumeric': 0.08326139088729016, 'fraction_numerical': 0.0024940047961630697, 'mean_word_length': 1.5485211439745783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5204616', 'n_tokens_mistral': 2705, 'n_tokens_neox': 2600, 'n_words': 691}