parameters: (self, name_expansion_result, thread_state=None)
docstring: Worker function for performing the actual copy (and rm, for mv).
docstring_summary: Worker function for performing the actual copy (and rm, for mv).
docstring_tokens: Worker function for performing the actual copy ( and rm for mv ) .
function: def CopyFunc(self, name_expansion_result, thread_state=None):
      """Worker function for performing the actual copy (and rm, for mv)."""
      gsutil_api = GetCloudApiInstance(self, thread_state=thread_state)
  
      copy_helper_opts = copy_helper.GetCopyHelperOpts()
      if copy_helper_opts.perform_mv:
        cmd_name = 'mv'
      else:
        cmd_name = self.command_name
      src_url = name_expansion_result.source_storage_url
      exp_src_url = name_expansion_result.expanded_storage_url
      src_url_names_container = name_expansion_result.names_container
      have_multiple_srcs = name_expansion_result.is_multi_source_request
  
      if src_url.IsCloudUrl() and src_url.IsProvider():
        raise CommandException(
            'The %s command does not allow provider-only source URLs (%s)' %
            (cmd_name, src_url))
      if have_multiple_srcs:
        copy_helper.InsistDstUrlNamesContainer(
            self.exp_dst_url, self.have_existing_dst_container, cmd_name)
  
      # Various GUI tools (like the GCS web console) create placeholder objects
      # ending with '/' when the user creates an empty directory. Normally these
      # tools should delete those placeholders once objects have been written
      # "under" the directory, but sometimes the placeholders are left around. We
      # need to filter them out here, otherwise if the user tries to rsync from
      # GCS to a local directory it will result in a directory/file conflict
      # (e.g., trying to download an object called "mydata/" where the local
      # directory "mydata" exists).
      if IsCloudSubdirPlaceholder(exp_src_url):
        # We used to output the message 'Skipping cloud sub-directory placeholder
        # object...' but we no longer do so because it caused customer confusion.
        return
  
      if copy_helper_opts.use_manifest and self.manifest.WasSuccessful(
          exp_src_url.url_string):
        return
  
      if copy_helper_opts.perform_mv:
        if name_expansion_result.names_container:
          # Use recursion_requested when performing name expansion for the
          # directory mv case so we can determine if any of the source URLs are
          # directories (and then use cp -r and rm -r to perform the move, to
          # match the behavior of Linux mv (which when moving a directory moves
          # all the contained files).
          self.recursion_requested = True
          # Disallow wildcard src URLs when moving directories, as supporting it
          # would make the name transformation too complex and would also be
          # dangerous (e.g., someone could accidentally move many objects to the
          # wrong name, or accidentally overwrite many objects).
          if ContainsWildcard(src_url.url_string):
            raise CommandException('The mv command disallows naming source '
                                   'directories using wildcards')
  
      if (self.exp_dst_url.IsFileUrl()
          and not os.path.exists(self.exp_dst_url.object_name)
          and have_multiple_srcs):
        os.makedirs(self.exp_dst_url.object_name)
  
      dst_url = copy_helper.ConstructDstUrl(
          src_url, exp_src_url, src_url_names_container, have_multiple_srcs,
          self.exp_dst_url, self.have_existing_dst_container,
          self.recursion_requested)
      dst_url = copy_helper.FixWindowsNaming(src_url, dst_url)
  
      copy_helper.CheckForDirFileConflict(exp_src_url, dst_url)
      if copy_helper.SrcDstSame(exp_src_url, dst_url):
        raise CommandException('%s: "%s" and "%s" are the same file - '
                               'abort.' % (cmd_name, exp_src_url, dst_url))
  
      if dst_url.IsCloudUrl() and dst_url.HasGeneration():
        raise CommandException('%s: a version-specific URL\n(%s)\ncannot be '
                               'the destination for gsutil cp - abort.'
                               % (cmd_name, dst_url))
  
      elapsed_time = bytes_transferred = 0
      try:
        if copy_helper_opts.use_manifest:
          self.manifest.Initialize(
              exp_src_url.url_string, dst_url.url_string)
        (elapsed_time, bytes_transferred, result_url, md5) = (
            copy_helper.PerformCopy(
                self.logger, exp_src_url, dst_url, gsutil_api,
                self, _CopyExceptionHandler, allow_splitting=True,
                headers=self.headers, manifest=self.manifest,
                gzip_exts=self.gzip_exts))
        if copy_helper_opts.use_manifest:
          if md5:
            self.manifest.Set(exp_src_url.url_string, 'md5', md5)
          self.manifest.SetResult(
              exp_src_url.url_string, bytes_transferred, 'OK')
        if copy_helper_opts.print_ver:
          # Some cases don't return a version-specific URL (e.g., if destination
          # is a file).
          self.logger.info('Created: %s', result_url)
      except ItemExistsError:
        message = 'Skipping existing item: %s' % dst_url
        self.logger.info(message)
        if copy_helper_opts.use_manifest:
          self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message)
      except SkipUnsupportedObjectError, e:
        message = ('Skipping item %s with unsupported object type %s' %
                   (exp_src_url.url_string, e.unsupported_type))
        self.logger.info(message)
        if copy_helper_opts.use_manifest:
          self.manifest.SetResult(exp_src_url.url_string, 0, 'skip', message)
      except copy_helper.FileConcurrencySkipError, e:
        self.logger.warn('Skipping copy of source URL %s because destination URL '
                         '%s is already being copied by another gsutil process '
                         'or thread (did you specify the same source URL twice?) '
                         % (src_url, dst_url))
      except Exception, e:
        if (copy_helper_opts.no_clobber and
            copy_helper.IsNoClobberServerException(e)):
          message = 'Rejected (noclobber): %s' % dst_url
          self.logger.info(message)
          if copy_helper_opts.use_manifest:
            self.manifest.SetResult(
                exp_src_url.url_string, 0, 'skip', message)
        elif self.continue_on_error:
          message = 'Error copying %s: %s' % (src_url, str(e))
          self.op_failure_count += 1
          self.logger.error(message)
          if copy_helper_opts.use_manifest:
            self.manifest.SetResult(
                exp_src_url.url_string, 0, 'error',
                RemoveCRLFFromString(message))
        else:
          if copy_helper_opts.use_manifest:
            self.manifest.SetResult(
                exp_src_url.url_string, 0, 'error', str(e))
          raise
      else:
        if copy_helper_opts.perform_mv:
          self.logger.info('Removing %s...', exp_src_url)
          if exp_src_url.IsCloudUrl():
            gsutil_api.DeleteObject(exp_src_url.bucket_name,
                                    exp_src_url.object_name,
                                    generation=exp_src_url.generation,
                                    provider=exp_src_url.scheme)
          else:
            os.unlink(exp_src_url.object_name)
  
      with self.stats_lock:
        self.total_elapsed_time += elapsed_time
        self.total_bytes_transferred += bytes_transferred
function_tokens: def CopyFunc ( self , name_expansion_result , thread_state = None ) : gsutil_api = GetCloudApiInstance ( self , thread_state = thread_state ) copy_helper_opts = copy_helper . GetCopyHelperOpts ( ) if copy_helper_opts . perform_mv : cmd_name = 'mv' else : cmd_name = self . command_name src_url = name_expansion_result . source_storage_url exp_src_url = name_expansion_result . expanded_storage_url src_url_names_container = name_expansion_result . names_container have_multiple_srcs = name_expansion_result . is_multi_source_request if src_url . IsCloudUrl ( ) and src_url . IsProvider ( ) : raise CommandException ( 'The %s command does not allow provider-only source URLs (%s)' % ( cmd_name , src_url ) ) if have_multiple_srcs : copy_helper . InsistDstUrlNamesContainer ( self . exp_dst_url , self . have_existing_dst_container , cmd_name ) # Various GUI tools (like the GCS web console) create placeholder objects # ending with '/' when the user creates an empty directory. Normally these # tools should delete those placeholders once objects have been written # "under" the directory, but sometimes the placeholders are left around. We # need to filter them out here, otherwise if the user tries to rsync from # GCS to a local directory it will result in a directory/file conflict # (e.g., trying to download an object called "mydata/" where the local # directory "mydata" exists). if IsCloudSubdirPlaceholder ( exp_src_url ) : # We used to output the message 'Skipping cloud sub-directory placeholder # object...' but we no longer do so because it caused customer confusion. return if copy_helper_opts . use_manifest and self . manifest . WasSuccessful ( exp_src_url . url_string ) : return if copy_helper_opts . perform_mv : if name_expansion_result . names_container : # Use recursion_requested when performing name expansion for the # directory mv case so we can determine if any of the source URLs are # directories (and then use cp -r and rm -r to perform the move, to # match the behavior of Linux mv (which when moving a directory moves # all the contained files). self . recursion_requested = True # Disallow wildcard src URLs when moving directories, as supporting it # would make the name transformation too complex and would also be # dangerous (e.g., someone could accidentally move many objects to the # wrong name, or accidentally overwrite many objects). if ContainsWildcard ( src_url . url_string ) : raise CommandException ( 'The mv command disallows naming source ' 'directories using wildcards' ) if ( self . exp_dst_url . IsFileUrl ( ) and not os . path . exists ( self . exp_dst_url . object_name ) and have_multiple_srcs ) : os . makedirs ( self . exp_dst_url . object_name ) dst_url = copy_helper . ConstructDstUrl ( src_url , exp_src_url , src_url_names_container , have_multiple_srcs , self . exp_dst_url , self . have_existing_dst_container , self . recursion_requested ) dst_url = copy_helper . FixWindowsNaming ( src_url , dst_url ) copy_helper . CheckForDirFileConflict ( exp_src_url , dst_url ) if copy_helper . SrcDstSame ( exp_src_url , dst_url ) : raise CommandException ( '%s: "%s" and "%s" are the same file - ' 'abort.' % ( cmd_name , exp_src_url , dst_url ) ) if dst_url . IsCloudUrl ( ) and dst_url . HasGeneration ( ) : raise CommandException ( '%s: a version-specific URL\n(%s)\ncannot be ' 'the destination for gsutil cp - abort.' % ( cmd_name , dst_url ) ) elapsed_time = bytes_transferred = 0 try : if copy_helper_opts . use_manifest : self . manifest . Initialize ( exp_src_url . url_string , dst_url . url_string ) ( elapsed_time , bytes_transferred , result_url , md5 ) = ( copy_helper . PerformCopy ( self . logger , exp_src_url , dst_url , gsutil_api , self , _CopyExceptionHandler , allow_splitting = True , headers = self . headers , manifest = self . manifest , gzip_exts = self . gzip_exts ) ) if copy_helper_opts . use_manifest : if md5 : self . manifest . Set ( exp_src_url . url_string , 'md5' , md5 ) self . manifest . SetResult ( exp_src_url . url_string , bytes_transferred , 'OK' ) if copy_helper_opts . print_ver : # Some cases don't return a version-specific URL (e.g., if destination # is a file). self . logger . info ( 'Created: %s' , result_url ) except ItemExistsError : message = 'Skipping existing item: %s' % dst_url self . logger . info ( message ) if copy_helper_opts . use_manifest : self . manifest . SetResult ( exp_src_url . url_string , 0 , 'skip' , message ) except SkipUnsupportedObjectError , e : message = ( 'Skipping item %s with unsupported object type %s' % ( exp_src_url . url_string , e . unsupported_type ) ) self . logger . info ( message ) if copy_helper_opts . use_manifest : self . manifest . SetResult ( exp_src_url . url_string , 0 , 'skip' , message ) except copy_helper . FileConcurrencySkipError , e : self . logger . warn ( 'Skipping copy of source URL %s because destination URL ' '%s is already being copied by another gsutil process ' 'or thread (did you specify the same source URL twice?) ' % ( src_url , dst_url ) ) except Exception , e : if ( copy_helper_opts . no_clobber and copy_helper . IsNoClobberServerException ( e ) ) : message = 'Rejected (noclobber): %s' % dst_url self . logger . info ( message ) if copy_helper_opts . use_manifest : self . manifest . SetResult ( exp_src_url . url_string , 0 , 'skip' , message ) elif self . continue_on_error : message = 'Error copying %s: %s' % ( src_url , str ( e ) ) self . op_failure_count += 1 self . logger . error ( message ) if copy_helper_opts . use_manifest : self . manifest . SetResult ( exp_src_url . url_string , 0 , 'error' , RemoveCRLFFromString ( message ) ) else : if copy_helper_opts . use_manifest : self . manifest . SetResult ( exp_src_url . url_string , 0 , 'error' , str ( e ) ) raise else : if copy_helper_opts . perform_mv : self . logger . info ( 'Removing %s...' , exp_src_url ) if exp_src_url . IsCloudUrl ( ) : gsutil_api . DeleteObject ( exp_src_url . bucket_name , exp_src_url . object_name , generation = exp_src_url . generation , provider = exp_src_url . scheme ) else : os . unlink ( exp_src_url . object_name ) with self . stats_lock : self . total_elapsed_time += elapsed_time self . total_bytes_transferred += bytes_transferred