<commit_before># Copyright (c) 2013 <NAME> <<EMAIL>>
# See the file license.txt for copying permission.

from metadata import MountLoadMetaData
from os import getgid, getuid
import os.path
from source import MountLoadSource
import stat
from target import MountLoadTarget
from threading import Semaphore

class Controller:
    def __init__(self, sourceURI, targetDirectory, password):
        self.gid = getgid()
        self.uid = getuid()

        # Initialize target and metadata
        self.target = MountLoadTarget(targetDirectory)
        self.metadata = MountLoadMetaData(self.target.getDBPath())

        # Store and check source URI
        knownSourceURI = self.metadata.getConfigString('sourceURI')
        if sourceURI is None:
            sourceURI = knownSourceURI
        elif knownSourceURI is None:
            self.metadata.setConfig('sourceURI', sourceURI)
        elif knownSourceURI != sourceURI:
            raise RuntimeError('Given source URI differs from known source URI')

        # Initialize source
        self.source = MountLoadSource(sourceURI, password)

        # Bootstrap the remote root
        self.metadata.begin()
        if self._getPath('/') is None:
            rootEntry = self.source.getEntry('/')
            if rootEntry is None:
                raise RuntimeError('Failed to retrieve the remote root directory %s' % self.source.getRemoteDirectory())
            self._registerPath('/', rootEntry)
        self.metadata.commit()

    def close(self):
        self.source.close()
        self.metadata.close()
        self.target.close()

    def _downloadFileData(self, pathInfo, offset, size):
        # Read data from source
        path = pathInfo['dirname'] + pathInfo['basename']
        data = self.source.readData(path, offset, size)

        # Write data to target
        self.target.writeData(path, offset, data)

        # Remove the remote segments we've overwritten
        pathId = pathInfo['pathId']
        self.metadata.begin()
        self.metadata.removeRemoteSegments(pathId, offset, offset + size - 1)

        # If all remote segments have been downloaded, we mark the file as synced
        if len(self.metadata.getRemoteSegments(pathId)) == 0:
            self.metadata.setPathSynced(pathId)
        self.metadata.commit()

        # Return the data<commit_msg>Store the source URI in metadata as late as possible in the Controller constructor, so we don't need to clean up the target directory with every typo.<commit_after># Copyright (c) 2013 <NAME> <<EMAIL>>
# See the file license.txt for copying permission.

from metadata import MountLoadMetaData
from os import getgid, getuid
import os.path
from source import MountLoadSource
import stat
from target import MountLoadTarget
from threading import Semaphore

class Controller:
    def __init__(self, sourceURI, targetDirectory, password):
        self.gid = getgid()
        self.uid = getuid()

        # Initialize target and metadata
        self.target = MountLoadTarget(targetDirectory)
        self.metadata = MountLoadMetaData(self.target.getDBPath())

        # Store and check source URI
        knownSourceURI = self.metadata.getConfigString('sourceURI')
        if sourceURI is None:
            sourceURI = knownSourceURI
        elif knownSourceURI is not None and knownSourceURI != sourceURI:
            raise RuntimeError('Given source URI differs from known source URI')

        # Initialize SFTP source
        self.source = MountLoadSource(sourceURI, password)

        # Bootstrap the remote root
        self.metadata.begin()
        if self._getPath('/') is None:
            rootEntry = self.source.getEntry('/')
            if rootEntry is None:
                raise RuntimeError('Failed to retrieve the remote root directory %s' % self.source.getRemoteDirectory())
            self._registerPath('/', rootEntry)
        self.metadata.commit()

        # Store the source URI in metadata
        if knownSourceURI is None:
            self.metadata.setConfig('sourceURI', sourceURI)

    def close(self):
        self.source.close()
        self.metadata.close()
        self.target.close()

    def _downloadFileData(self, pathInfo, offset, size):
        # Read data from source
        path = pathInfo['dirname'] + pathInfo['basename']
        data = self.source.readData(path, offset, size)

        # Write data to target
        self.target.writeData(path, offset, data)

        # Remove the remote segments we've overwritten
        pathId = pathInfo['pathId']
        self.metadata.begin()
        self.metadata.removeRemoteSegments(pathId, offset, offset + size - 1)

        # If all remote segments have been downloaded, we mark the file as synced
        if len(self.metadata.getRemoteSegments(pathId)) == 0:
            self.metadata.setPathSynced(pathId)
        self.metadata.commit()

        # Return the data
{'fraction_non_alphanumeric': 0.08403361344537816, 'fraction_numerical': 0.0024595203935232628, 'mean_word_length': 2.6802413273001506, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1323461', 'n_tokens_mistral': 1278, 'n_tokens_neox': 1226, 'n_words': 405}