parameters: (self,
                               output_path,
                               day,
                               pass_id,
                               key_num,
                               hadoop_fs_name,
                               hadoop_fs_ugi,
                               hadoop_home="$HADOOP_HOME",
                               donefile_name="sparse_cache.meta",
                               **kwargs)
docstring: write cache donefile
  
          Args:
              output_path(str): output path
              day(str|int): training day of model
              pass_id(str|int): training pass id of model
              key_num(str|int): save cache return value
              hadoop_fs_name(str): hdfs/afs fs name
              hadoop_fs_ugi(str): hdfs/afs fs ugi
              hadoop_home(str): hadoop home, default is "$HADOOP_HOME"
              donefile_name(str): donefile name, default is "sparse_cache.meta"
              kwargs(dict): user defined properties
                            file_num(int): cache file num
                            table_id(int): cache table id
  
          Examples:
              .. code-block:: python
  
                from paddle.fluid.incubate.fleet.utils.fleet_util import FleetUtil
                fleet_util = FleetUtil()
                fleet_util.write_cache_donefile(
                    output_path="hdfs:/my/output/",
                    day=20190722,
                    pass_id=1,
                    key_num=123456,
                    hadoop_fs_name="hdfs://xxx",
                    hadoop_fs_ugi="user,passwd",
                    )
docstring_summary: write cache donefile
docstring_tokens: write cache donefile
function: def write_cache_donefile(self,
                               output_path,
                               day,
                               pass_id,
                               key_num,
                               hadoop_fs_name,
                               hadoop_fs_ugi,
                               hadoop_home="$HADOOP_HOME",
                               donefile_name="sparse_cache.meta",
                               **kwargs):
          """
          write cache donefile
  
          Args:
              output_path(str): output path
              day(str|int): training day of model
              pass_id(str|int): training pass id of model
              key_num(str|int): save cache return value
              hadoop_fs_name(str): hdfs/afs fs name
              hadoop_fs_ugi(str): hdfs/afs fs ugi
              hadoop_home(str): hadoop home, default is "$HADOOP_HOME"
              donefile_name(str): donefile name, default is "sparse_cache.meta"
              kwargs(dict): user defined properties
                            file_num(int): cache file num
                            table_id(int): cache table id
  
          Examples:
              .. code-block:: python
  
                from paddle.fluid.incubate.fleet.utils.fleet_util import FleetUtil
                fleet_util = FleetUtil()
                fleet_util.write_cache_donefile(
                    output_path="hdfs:/my/output/",
                    day=20190722,
                    pass_id=1,
                    key_num=123456,
                    hadoop_fs_name="hdfs://xxx",
                    hadoop_fs_ugi="user,passwd",
                    )
  
          """
          day = str(day)
          pass_id = str(pass_id)
          key_num = int(key_num)
          file_num = kwargs.get("file_num", 16)
          table_id = kwargs.get("table_id", 0)
  
          if pass_id != "-1":
              suffix_name = "/%s/delta-%s/%03d_cache" % (day, pass_id, table_id)
              model_path = output_path.rstrip("/") + suffix_name
          else:
              suffix_name = "/%s/base/%03d_cache" % (day, table_id)
              model_path = output_path.rstrip("/") + suffix_name
  
          if fleet.worker_index() == 0:
              donefile_path = model_path + "/" + donefile_name
              configs = {
                  "fs.default.name": hadoop_fs_name,
                  "hadoop.job.ugi": hadoop_fs_ugi
              }
              client = HDFSClient(hadoop_home, configs)
              if client.is_file(donefile_path):
                  self.rank0_error( \
                      "not write because %s already exists" % donefile_path)
              else:
                  meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" \
                             % (file_num, key_num)
                  with open(donefile_name, "w") as f:
                      f.write(meta_str)
                  client.upload(donefile_name, model_path)
                  self.rank0_error("write %s succeed" % donefile_path)
          fleet._role_maker._barrier_worker()
function_tokens: def write_cache_donefile ( self , output_path , day , pass_id , key_num , hadoop_fs_name , hadoop_fs_ugi , hadoop_home = "$HADOOP_HOME" , donefile_name = "sparse_cache.meta" , * * kwargs ) : day = str ( day ) pass_id = str ( pass_id ) key_num = int ( key_num ) file_num = kwargs . get ( "file_num" , 16 ) table_id = kwargs . get ( "table_id" , 0 ) if pass_id != "-1" : suffix_name = "/%s/delta-%s/%03d_cache" % ( day , pass_id , table_id ) model_path = output_path . rstrip ( "/" ) + suffix_name else : suffix_name = "/%s/base/%03d_cache" % ( day , table_id ) model_path = output_path . rstrip ( "/" ) + suffix_name if fleet . worker_index ( ) == 0 : donefile_path = model_path + "/" + donefile_name configs = { "fs.default.name" : hadoop_fs_name , "hadoop.job.ugi" : hadoop_fs_ugi } client = HDFSClient ( hadoop_home , configs ) if client . is_file ( donefile_path ) : self . rank0_error ( "not write because %s already exists" % donefile_path ) else : meta_str = "file_prefix:part\npart_num:%s\nkey_num:%d\n" % ( file_num , key_num ) with open ( donefile_name , "w" ) as f : f . write ( meta_str ) client . upload ( donefile_name , model_path ) self . rank0_error ( "write %s succeed" % donefile_path ) fleet . _role_maker . _barrier_worker ( )