<commit_before>import os, re, sys
import commands
from time import time, sleep
import urlparse
import traceback

from TimerCommand import TimerCommand
import SiteMover
from futil import *
from PilotErrors import PilotErrors
from pUtil import tolog, readpar, verifySetupCommand, getSiteInformation, extractFilePaths
from FileStateClient import updateFileState
from SiteInformation import SiteInformation
from config import config_sm

CMD_CHECKSUM = config_sm.COMMAND_MD5

class S3ObjectstoreSiteMover(SiteMover.SiteMover):
    """ SiteMover that uses boto S3 client for both get and put """
    # no registration is done
    copyCommand = "S3Objectstore"
    checksum_command = "adler32"
    timeout = 3600
    _instance = None

    def __init__(self, setup_path, useTimerCommand=True, *args, **kwrds):
        self._setup = setup_path.strip()
        self._defaultSetup = "source /cvmfs/atlas.cern.ch/repo/sw/external/boto/setup.sh; unset http_proxy; unset https_proxy"
        self.s3Objectstore = None
        self.__isBotoLoaded = False
        self._useTimerCommand = useTimerCommand

    def __new__(cls, *args, **kwargs):
        if not cls._instance:
            cls._instance = super(S3ObjectstoreSiteMover, cls).__new__(cls, *args, **kwargs)
        return cls._instance

    def get_timeout(self):
        return self.timeout

    def log(self, errorLog):
        tolog(errorLog)

    def setup(self, experiment=None, surl=None):
        """ setup env """
        if not self.__isBotoLoaded:
            try:
                import boto
                import boto.s3.connection
                from boto.s3.key import Key
                self.__isBotoLoaded = True
            except ImportError:
                tolog("Failed to import boto, add /cvmfs/atlas.cern.ch/repo/sw/external/boto/lib/python2.6/site-packages/ to sys.path")
                sys.path.append('/cvmfs/atlas.cern.ch/repo/sw/external/boto/lib/python2.6/site-packages/')
                try:
                    import boto
                    import boto.s3.connection
                    from boto.s3.key import Key
                    self.__isBotoLoaded = True
                except ImportError:
                    tolog("Failed to import boto again. exit")<commit_msg>fix s3sitemover creation problem<commit_after>import os, re, sys
import commands
from time import time, sleep
import urlparse
import traceback

from TimerCommand import TimerCommand
import SiteMover
from futil import *
from PilotErrors import PilotErrors
from pUtil import tolog, readpar, verifySetupCommand, getSiteInformation, extractFilePaths
from FileStateClient import updateFileState
from SiteInformation import SiteInformation
from config import config_sm

CMD_CHECKSUM = config_sm.COMMAND_MD5

class S3ObjectstoreSiteMover(SiteMover.SiteMover):
    """ SiteMover that uses boto S3 client for both get and put """
    # no registration is done
    copyCommand = "S3Objectstore"
    checksum_command = "adler32"
    timeout = 3600
    _instance = None

    def __init__(self, setup_path, useTimerCommand=True, *args, **kwrds):
        self._setup = setup_path.strip()
        self._defaultSetup = "source /cvmfs/atlas.cern.ch/repo/sw/external/boto/setup.sh; unset http_proxy; unset https_proxy"
        self.s3Objectstore = None
        self.__isBotoLoaded = False
        self._useTimerCommand = useTimerCommand

    def get_timeout(self):
        return self.timeout

    def log(self, errorLog):
        tolog(errorLog)

    def setup(self, experiment=None, surl=None):
        """ setup env """
        if not self.__isBotoLoaded:
            try:
                import boto
                import boto.s3.connection
                from boto.s3.key import Key
                self.__isBotoLoaded = True
            except ImportError:
                tolog("Failed to import boto, add /cvmfs/atlas.cern.ch/repo/sw/external/boto/lib/python2.6/site-packages/ to sys.path")
                sys.path.append('/cvmfs/atlas.cern.ch/repo/sw/external/boto/lib/python2.6/site-packages/')
                try:
                    import boto
                    import boto.s3.connection
                    from boto.s3.key import Key
                    self.__isBotoLoaded = True
                except ImportError:
                    tolog("Failed to import boto again. exit")
{'fraction_non_alphanumeric': 0.07747290753977404, 'fraction_numerical': 0.009222965183306432, 'mean_word_length': 2.4319620253164556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4899565', 'n_tokens_mistral': 1272, 'n_tokens_neox': 1221, 'n_words': 324}