parameters: (in_iter, out_file)
docstring: Sorts input lines from in_iter and outputs to out_file.
  
    Sorts in batches as input arrives, so input file does not need to be loaded
    into memory all at once. Derived from Python Recipe 466302: Sorting big
    files the Python 2.4 way by Nicolas Lehuen.
  
    Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line
    when we could just sort on the first record (URL); but the sort order is
    identical either way.
  
    Args:
      in_iter: Input iterator.
      out_file: Output file.
docstring_summary: Sorts input lines from in_iter and outputs to out_file.
docstring_tokens: Sorts input lines from in_iter and outputs to out_file .
function: def _BatchSort(in_iter, out_file):
    """Sorts input lines from in_iter and outputs to out_file.
  
    Sorts in batches as input arrives, so input file does not need to be loaded
    into memory all at once. Derived from Python Recipe 466302: Sorting big
    files the Python 2.4 way by Nicolas Lehuen.
  
    Sorted format is per _BuildTmpOutputLine. We're sorting on the entire line
    when we could just sort on the first record (URL); but the sort order is
    identical either way.
  
    Args:
      in_iter: Input iterator.
      out_file: Output file.
    """
    # Note: If chunk_files gets very large we can run out of open FDs. See .boto
    # file comments about rsync_buffer_lines. If increasing rsync_buffer_lines
    # doesn't suffice (e.g., for someone synchronizing with a really large
    # bucket), an option would be to make gsutil merge in passes, never
    # opening all chunk files simultaneously.
    buffer_size = config.getint('GSUtil', 'rsync_buffer_lines', 32000)
    chunk_files = []
    try:
      while True:
        current_chunk = sorted(islice(in_iter, buffer_size))
        if not current_chunk:
          break
        output_chunk = io.open('%s-%06i' % (out_file.name, len(chunk_files)),
                               mode='w+', encoding=UTF8)
        chunk_files.append(output_chunk)
        output_chunk.writelines(unicode(''.join(current_chunk)))
        output_chunk.flush()
        output_chunk.seek(0)
      out_file.writelines(heapq.merge(*chunk_files))
    except IOError as e:
      if e.errno == errno.EMFILE:
        raise CommandException('\n'.join(textwrap.wrap(
            'Synchronization failed because too many open file handles were '
            'needed while building synchronization state. Please see the '
            'comments about rsync_buffer_lines in your .boto config file for a '
            'possible way to address this problem.')))
      raise
    finally:
      for chunk_file in chunk_files:
        try:
          chunk_file.close()
          os.remove(chunk_file.name)
        except:
          pass
function_tokens: def _BatchSort ( in_iter , out_file ) : # Note: If chunk_files gets very large we can run out of open FDs. See .boto # file comments about rsync_buffer_lines. If increasing rsync_buffer_lines # doesn't suffice (e.g., for someone synchronizing with a really large # bucket), an option would be to make gsutil merge in passes, never # opening all chunk files simultaneously. buffer_size = config . getint ( 'GSUtil' , 'rsync_buffer_lines' , 32000 ) chunk_files = [ ] try : while True : current_chunk = sorted ( islice ( in_iter , buffer_size ) ) if not current_chunk : break output_chunk = io . open ( '%s-%06i' % ( out_file . name , len ( chunk_files ) ) , mode = 'w+' , encoding = UTF8 ) chunk_files . append ( output_chunk ) output_chunk . writelines ( unicode ( '' . join ( current_chunk ) ) ) output_chunk . flush ( ) output_chunk . seek ( 0 ) out_file . writelines ( heapq . merge ( * chunk_files ) ) except IOError as e : if e . errno == errno . EMFILE : raise CommandException ( '\n' . join ( textwrap . wrap ( 'Synchronization failed because too many open file handles were ' 'needed while building synchronization state. Please see the ' 'comments about rsync_buffer_lines in your .boto config file for a ' 'possible way to address this problem.' ) ) ) raise finally : for chunk_file in chunk_files : try : chunk_file . close ( ) os . remove ( chunk_file . name ) except : pass