<commit_before>'''
Datastore via remote webdav connection
'''

import os
import mimetypes
import tarfile
import logging
import hashlib
import shutil
from fs.contrib.davfs import DAVFS
from urlparse import urlparse
from contextlib import closing  # needed for Python 2.6

from ..compatibility import urlopen
from .base import DataItem, DataKey
from .archivingfs import ArchivingFileSystemDataStore, ArchivedDataFile, TIMESTAMP_FORMAT
from .filesystem import FileSystemDataStore, DataFile


class DavFsDataItem(ArchivedDataFile):
    """Base class for data item classes, that may represent files or database records."""

    def __init__(self, path, store):
        # needs to be first cause _get_info is called in Base __init__
        self.store = store
        super(DavFsDataItem, self).__init__(path, store)

    def get_content(self, max_length=None):
        obj = self.store.dav_fs.open(self.tarfile_path, 'rb')
        with closing(tarfile.open(fileobj=obj)) as data_archive:
            f = data_archive.extractfile(self.path)
            if max_length:
                content = f.read(max_length)
            else:
                content = f.read()
            f.close()
            return content

    # mandatory repeat
    content = property(fget=get_content)

    def _get_info(self):
        obj = self.store.dav_fs.open(self.tarfile_path, 'rb')
        with closing(tarfile.open(fileobj=obj)) as data_archive:
            return data_archive.getmember(self.path)
        return tarfile.TarInfo()


class DavFsDataStore(ArchivingFileSystemDataStore):
    """ArchivingFileSystemDataStore that archives to webdav storage"""

    data_item_class = DavFsDataItem

    def __init__(self, root, dav_url, dav_user=None, dav_pw=None):
        super(DavFsDataStore, self).__init__(root)
        parsed = urlparse(dav_url)
        self.dav_user = dav_user or parsed.username
        self.dav_pw = dav_pw or parsed.password
        self.dav_url = parsed.geturl()
        self.dav_fs = DAVFS(url=self.dav_url, credentials={'username': self.dav_user, 'password': self.dav_pw})

    def __getstate__(self):
        return {'root': self.root, 'dav_url': self.dav_url, 'dav_user': self.dav_user, 'dav_pw': self.dav_pw}

    def find_new_data(self, timestamp):
        """Finds newly created/changed data items"""
        new_files = self._find_new_data_files(timestamp)
        label = timestamp.strftime(TIMESTAMP_FORMAT)
        archive_paths = self._archive(label, new_files)
        return [DavFsDataItem(path, self).generate_key()
                for path in archive_paths]

    def _archive(self, label, files, delete_originals=True):
        """
        Archives files and, by default, deletes the originals.
        """
        fs = self.dav_fs
        if not fs.isdir(self.archive_store):
            fs.makedir(self.archive_store, recursive=True)
        tf_obj = fs.open(os.path.join(self.archive_store, label + ".tar.gz"), mode='wb')
        tf = tarfile.open(fileobj=tf_obj, mode='w:gz')
        logging.info("Archiving data to file %s" % tf.name)
        # Add data files
        archive_paths = []
        for file_path in files:
            archive_path = os.path.join(label, file_path)
            tf.add(os.path.join(self.root, file_path), archive_path)
            archive_paths.append(archive_path)
        tf.close()
        tf_obj.close()

        # Delete original files.
        if delete_originals:
            for file_path in files:
                os.remove(os.path.join(self.root, file_path))
        self._last_label = label # useful for testing
        return archive_paths

<commit_msg>code cleanup in new davfs.py datastore<commit_after>'''
Datastore via remote webdav connection
'''

import os
import tarfile
import logging
from fs.contrib.davfs import DAVFS
from urlparse import urlparse
from contextlib import closing  # needed for Python 2.6

from .archivingfs import ArchivingFileSystemDataStore, ArchivedDataFile, TIMESTAMP_FORMAT


class DavFsDataItem(ArchivedDataFile):
    """Base class for data item classes, that may represent files or database records."""

    def __init__(self, path, store):
        # needs to be first cause _get_info is called in Base __init__
        self.store = store
        super(DavFsDataItem, self).__init__(path, store)

    def get_content(self, max_length=None):
        obj = self.store.dav_fs.open(self.tarfile_path, 'rb')
        with closing(tarfile.open(fileobj=obj)) as data_archive:
            f = data_archive.extractfile(self.path)
            if max_length:
                content = f.read(max_length)
            else:
                content = f.read()
            f.close()
            return content

    # mandatory repeat
    content = property(fget=get_content)

    def _get_info(self):
        obj = self.store.dav_fs.open(self.tarfile_path, 'rb')
        with closing(tarfile.open(fileobj=obj)) as data_archive:
            return data_archive.getmember(self.path)
        return tarfile.TarInfo()


class DavFsDataStore(ArchivingFileSystemDataStore):
    """ArchivingFileSystemDataStore that archives to webdav storage"""

    data_item_class = DavFsDataItem

    def __init__(self, root, dav_url, dav_user=None, dav_pw=None):
        super(DavFsDataStore, self).__init__(root)
        parsed = urlparse(dav_url)
        self.dav_user = dav_user or parsed.username
        self.dav_pw = dav_pw or parsed.password
        self.dav_url = parsed.geturl()
        self.dav_fs = DAVFS(url=self.dav_url, credentials={'username': self.dav_user, 'password': self.dav_pw})

    def __getstate__(self):
        return {'root': self.root, 'dav_url': self.dav_url, 'dav_user': self.dav_user, 'dav_pw': self.dav_pw}

    def find_new_data(self, timestamp):
        """Finds newly created/changed data items"""
        new_files = self._find_new_data_files(timestamp)
        label = timestamp.strftime(TIMESTAMP_FORMAT)
        archive_paths = self._archive(label, new_files)
        return [DavFsDataItem(path, self).generate_key()
                for path in archive_paths]

    def _archive(self, label, files, delete_originals=True):
        """
        Archives files and, by default, deletes the originals.
        """
        fs = self.dav_fs
        if not fs.isdir(self.archive_store):
            fs.makedir(self.archive_store, recursive=True)
        tf_obj = fs.open(os.path.join(self.archive_store, label + ".tar.gz"), mode='wb')
        with tarfile.open(fileobj=tf_obj, mode='w:gz') as tf:
            logging.info("Archiving data to file %s" % tf.name)
            # Add data files
            archive_paths = []
            for file_path in files:
                archive_path = os.path.join(label, file_path)
                tf.add(os.path.join(self.root, file_path), archive_path)
                archive_paths.append(archive_path)
            tf.close()
        tf_obj.close()

        # Delete original files.
        if delete_originals:
            for file_path in files:
                os.remove(os.path.join(self.root, file_path))
        self._last_label = label # useful for testing
        return archive_paths


{'fraction_non_alphanumeric': 0.09638047138047139, 'fraction_numerical': 0.0005611672278338945, 'mean_word_length': 2.771957671957672, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6170954', 'n_tokens_mistral': 2214, 'n_tokens_neox': 2113, 'n_words': 519}