<commit_before>#!/usr/bin/env python
#
# Created as part of the StratusLab project (http://stratuslab.eu),
# co-funded by the European Commission under the Grant Agreement
# INFSO-RI-261552."
#
# Copyright (c) 2011, <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys

sys.path.append('/var/lib/stratuslab/python')

from time import time
from signal import signal, SIGINT
from os.path import basename, dirname
from getpass import getuser
from urlparse import urlparse
from stratuslab.Util import sshCmdWithOutput, defaultConfigFile
from stratuslab.Authn import LocalhostCredentialsConnector
from stratuslab.Defaults import sshPublicKeyLocation
from stratuslab.ConfigHolder import ConfigHolder
from stratuslab.Configurator import Configurator
from stratuslab.PersistentDisk import PersistentDisk
from stratuslab.marketplace.Policy import Policy
from stratuslab.CloudConnectorFactory import CloudConnectorFactory
from stratuslab.commandbase.StorageCommand import PDiskEndpoint
from stratuslab.marketplace.ManifestDownloader import ManifestDownloader
import stratuslab.ManifestInfo as ManifestInfo

class TMCloneCache(object):
    ''' Clone or retrieve from cache disk image
    '''

    # Debug option
    PRINT_TRACE_ON_ERROR = True

    # Position of the provided args
    _ARG_SRC_POS = 1
    _ARG_DST_POS = 2

    _PDISK_PORT = 8445
    
    _UNCOMPRESS_TOOL = {'gz': '/bin/gunzip',
                       'bz2': '/bin/bunzip2'}
    
    _CHECKSUM = 'sha1'
    _CHECKSUM_CMD = '%ssum' % _CHECKSUM
    
    _ACCEPTED_EXTRA_DISK_TYPE = ('DATA_IMAGE_RAW_READONLY', 'DATA_IMAGE_RAW_READ_WRITE')
    _ACCEPTED_ROOT_DISK_TYPE = ('MACHINE_IMAGE_LIVE')
        
    _IDENTIFIER_KEY = 'identifier'

    def __init__(self, args, **kwargs):
        self.args = args
        
        self.diskSrc = None
        self.diskDstPath = None
        self.diskDstHost = None
        self.marketplaceEndpoint = None
        self.marketplaceImageId = None
        self.pdiskImageId = None
        self.pdiskSnapshotId = None
        self.downloadedLocalImageLocation = None
        self.downloadedLocalImageSize = 0
        
        configFile = kwargs.get('config', defaultConfigFile)
        config = ConfigHolder.configFileToDict(configFile)
        options = PDiskEndpoint.options()
        options.update({'verboseLevel': 0, 'configFile': configFile})
        self.configHolder = ConfigHolder(options, config)
        self.config = Configurator(self.configHolder)
        self.pdiskEndpoint = self.config.getValue('persistent_disk_ip')    
        self.pdiskLVMDevice = self.config.getValue('persistent_disk_lvm_device')
        self.configHolder.set('pdiskEndpoint', self.pdiskEndpoint)
        
        self.pdisk = PersistentDisk(self.configHolder)
        self.manifestDownloader = ManifestDownloader(self.configHolder)
        
        self.defaultSignalHandler = None
    
    def run(self):
        self._checkArgs()
        self._parseArgs()
        self._retrieveDisk()
    
    def _checkArgs(self):
        self._assertLength(self.args, 3, 'Invalid number of arguments')
    
    def _parseArgs(self):
        dst = self.args[self._ARG_DST_POS]
        self.diskDstHost = self._getDiskHost(dst)
        self.diskDstPath = self._getDiskPath(dst)
        self.diskSrc = self.args[self._ARG_SRC_POS]

    def _retrieveDisk(self):
        if self.diskSrc.startswith('pdisk:'):
            self._startFromPersisted()
        else:
            self._startFromCowSnapshot()
    
    def _startFromPersisted(self):
        diskId = self._getStringPart(self.diskSrc, -1, 4)
        diskType = self.pdisk.getValue('type', diskId)
        
        is_root_disk = self._getDiskIndex(self.diskDstPath) is 0
        if is_root_disk:
            self._checkBootDisk(diskId, diskType)
        elif diskType not in self._ACCEPTED_EXTRA_DISK_TYPE:
            raise ValueError('Only %s type disks can be attached as extra disks'
                             % self._ACCEPTED_EXTRA_DISK_TYPE.join(', '))
        
        self._createDestinationDir()
        self._attachPDisk(self.diskSrc)
            
    def _checkBootDisk(self, diskId, diskType):
        is_live_machine_disk = diskType in self._ACCEPTED_ROOT_DISK_TYPE
        user_count = self.pdisk.getVolumeUserCount(diskId)

        if not is_live_machine_disk:
            raise Exception('Only a live persistent disk can be booted from.')
        if user_count != 0:
            raise Exception('User count must be zero on the live disk to boot from.')

    def _startFromCowSnapshot(self):
        self._retrieveMarketplaceInfos()
        
        if self._cacheMiss():
            self._retrieveAndCachePDiskImage()<commit_msg>increment volume user count when operating with persisted disk<commit_after>#!/usr/bin/env python
#
# Created as part of the StratusLab project (http://stratuslab.eu),
# co-funded by the European Commission under the Grant Agreement
# INFSO-RI-261552."
#
# Copyright (c) 2011, SixSq Sarl
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import sys

sys.path.append('/var/lib/stratuslab/python')

from time import time
from signal import signal, SIGINT
from os.path import basename, dirname
from getpass import getuser
from urlparse import urlparse
from stratuslab.Util import sshCmdWithOutput, defaultConfigFile
from stratuslab.Authn import LocalhostCredentialsConnector
from stratuslab.Defaults import sshPublicKeyLocation
from stratuslab.ConfigHolder import ConfigHolder
from stratuslab.Configurator import Configurator
from stratuslab.PersistentDisk import PersistentDisk
from stratuslab.marketplace.Policy import Policy
from stratuslab.CloudConnectorFactory import CloudConnectorFactory
from stratuslab.commandbase.StorageCommand import PDiskEndpoint
from stratuslab.marketplace.ManifestDownloader import ManifestDownloader
import stratuslab.ManifestInfo as ManifestInfo

class TMCloneCache(object):
    ''' Clone or retrieve from cache disk image
    '''

    # Debug option
    PRINT_TRACE_ON_ERROR = True

    # Position of the provided args
    _ARG_SRC_POS = 1
    _ARG_DST_POS = 2

    _PDISK_PORT = 8445
    
    _UNCOMPRESS_TOOL = {'gz': '/bin/gunzip',
                       'bz2': '/bin/bunzip2'}
    
    _CHECKSUM = 'sha1'
    _CHECKSUM_CMD = '%ssum' % _CHECKSUM
    
    _ACCEPTED_EXTRA_DISK_TYPE = ('DATA_IMAGE_RAW_READONLY', 'DATA_IMAGE_RAW_READ_WRITE')
    _ACCEPTED_ROOT_DISK_TYPE = ('MACHINE_IMAGE_LIVE')
        
    _IDENTIFIER_KEY = 'identifier'
    _COUNT_KEY = 'count'

    def __init__(self, args, **kwargs):
        self.args = args
        
        self.diskSrc = None
        self.diskDstPath = None
        self.diskDstHost = None
        self.marketplaceEndpoint = None
        self.marketplaceImageId = None
        self.pdiskImageId = None
        self.pdiskSnapshotId = None
        self.downloadedLocalImageLocation = None
        self.downloadedLocalImageSize = 0
        
        configFile = kwargs.get('config', defaultConfigFile)
        config = ConfigHolder.configFileToDict(configFile)
        options = PDiskEndpoint.options()
        options.update({'verboseLevel': 0, 'configFile': configFile})
        self.configHolder = ConfigHolder(options, config)
        self.config = Configurator(self.configHolder)
        self.pdiskEndpoint = self.config.getValue('persistent_disk_ip')    
        self.pdiskLVMDevice = self.config.getValue('persistent_disk_lvm_device')
        self.configHolder.set('pdiskEndpoint', self.pdiskEndpoint)
        
        self.pdisk = PersistentDisk(self.configHolder)
        self.manifestDownloader = ManifestDownloader(self.configHolder)
        
        self.defaultSignalHandler = None
    
    def run(self):
        self._checkArgs()
        self._parseArgs()
        self._retrieveDisk()
    
    def _checkArgs(self):
        self._assertLength(self.args, 3, 'Invalid number of arguments')
    
    def _parseArgs(self):
        dst = self.args[self._ARG_DST_POS]
        self.diskDstHost = self._getDiskHost(dst)
        self.diskDstPath = self._getDiskPath(dst)
        self.diskSrc = self.args[self._ARG_SRC_POS]

    def _retrieveDisk(self):
        if self.diskSrc.startswith('pdisk:'):
            self._startFromPersisted()
        else:
            self._startFromCowSnapshot()
    
    def _startFromPersisted(self):
        diskId = self._getStringPart(self.diskSrc, -1, 4)
        diskType = self.pdisk.getValue('type', diskId)
        
        is_root_disk = self._getDiskIndex(self.diskDstPath) is 0
        if is_root_disk:
            self._checkBootDisk(diskId, diskType)
        elif diskType not in self._ACCEPTED_EXTRA_DISK_TYPE:
            raise ValueError('Only %s type disks can be attached as extra disks'
                             % self._ACCEPTED_EXTRA_DISK_TYPE.join(', '))
        
        self._createDestinationDir()
        self._attachPDisk(self.diskSrc)
        self._incrementVolumeUserCount(diskId)

    def _incrementVolumeUserCount(self, diskId):
        user_count = self.pdisk.getVolumeUserCount(diskId)
        self.pdisk.updateVolume({self._COUNT_KEY: str(user_count + 1)}, diskId)

    def _checkBootDisk(self, diskId, diskType):
        is_live_machine_disk = diskType in self._ACCEPTED_ROOT_DISK_TYPE
        user_count = self.pdisk.getVolumeUserCount(diskId)

        if not is_live_machine_disk:
            raise Exception('Only a live persistent disk can be booted from.')
        if user_count != 0:
            raise Exception('User count must be zero on the live disk to boot from.')

    def _startFromCowSnapshot(self):
        self._retrieveMarketplaceInfos()
        
        if self._cacheMiss():
            self._retrieveAndCachePDiskImage()
{'fraction_non_alphanumeric': 0.0788782592384168, 'fraction_numerical': 0.0057212530482085916, 'mean_word_length': 3.0854406130268197, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377227', 'n_tokens_mistral': 3164, 'n_tokens_neox': 3085, 'n_words': 772}