parameters: (self)
docstring: Obtain all information on the datasets that we need to run.
  
          Use DBS to figure out all required information on our
          datasets, like the run numbers and the GlobalTag. All
          information is stored in the datasets_information member
          variable.
docstring_summary: Obtain all information on the datasets that we need to run.
docstring_tokens: Obtain all information on the datasets that we need to run .
function: def build_datasets_information(self):
          """Obtain all information on the datasets that we need to run.
  
          Use DBS to figure out all required information on our
          datasets, like the run numbers and the GlobalTag. All
          information is stored in the datasets_information member
          variable.
  
          """
  
          # Get a list of runs in the dataset.
          # NOTE: The harvesting has to be done run-by-run, so we
          # split up datasets based on the run numbers. Strictly
          # speaking this is not (yet?) necessary for Monte Carlo
          # since all those samples use run number 1. Still, this
          # general approach should work for all samples.
  
          # Now loop over all datasets in the list and process them.
          # NOTE: This processing has been split into several loops
          # to be easier to follow, sacrificing a bit of efficiency.
          self.datasets_information = {}
          self.logger.info("Collecting information for all datasets to process")
          dataset_names = sorted(self.datasets_to_use.keys())
          for dataset_name in dataset_names:
  
              # Tell the user which dataset: nice with many datasets.
              sep_line = "-" * 30
              self.logger.info(sep_line)
              self.logger.info("  `%s'" % dataset_name)
              self.logger.info(sep_line)
  
              runs = self.dbs_resolve_runs(dataset_name)
              self.logger.info("    found %d run(s)" % len(runs))
              if len(runs) > 0:
                  self.logger.debug("      run number(s): %s" % \
                                    ", ".join([str(i) for i in runs]))
              else:
                  # DEBUG DEBUG DEBUG
                  # This should never happen after the DBS checks.
                  self.logger.warning("  --> skipping dataset "
                                      "without any runs")
                  assert False, "Panic: found a dataset without runs " \
                         "after DBS checks!"
                  # DEBUG DEBUG DEBUG end
  
              cmssw_version = self.dbs_resolve_cmssw_version(dataset_name)
              self.logger.info("    found CMSSW version `%s'" % cmssw_version)
  
              # Figure out if this is data or MC.
              datatype = self.dbs_resolve_datatype(dataset_name)
              self.logger.info("    sample is data or MC? --> %s" % \
                               datatype)
  
              ###
  
              # Try and figure out the GlobalTag to be used.
              if self.globaltag is None:
                  globaltag = self.dbs_resolve_globaltag(dataset_name)
              else:
                  globaltag = self.globaltag
  
              self.logger.info("    found GlobalTag `%s'" % globaltag)
  
              # DEBUG DEBUG DEBUG
              if globaltag == "":
                  # Actually we should not even reach this point, after
                  # our dataset sanity checks.
                  assert datatype == "data", \
                         "ERROR Empty GlobalTag for MC dataset!!!"
              # DEBUG DEBUG DEBUG end
  
              ###
  
              # DEBUG DEBUG DEBUG
              #tmp = self.dbs_check_dataset_spread_old(dataset_name)
              # DEBUG DEBUG DEBUG end
              sites_catalog = self.dbs_check_dataset_spread(dataset_name)
  
              # Extract the total event counts.
              num_events = {}
              for run_number in sites_catalog.keys():
                  num_events[run_number] = sites_catalog \
                                           [run_number]["all_sites"]
                  del sites_catalog[run_number]["all_sites"]
  
              # Extract the information about whether or not datasets
              # are mirrored.
              mirror_catalog = {}
              for run_number in sites_catalog.keys():
                  mirror_catalog[run_number] = sites_catalog \
                                               [run_number]["mirrored"]
                  del sites_catalog[run_number]["mirrored"]
  
              # BUG BUG BUG
              # I think I could now get rid of that and just fill the
              # "sites" entry with the `inverse' of this
              # num_events_catalog(?).
              #num_sites = self.dbs_resolve_dataset_number_of_sites(dataset_name)
              #sites_catalog = self.dbs_check_dataset_spread(dataset_name)
              #sites_catalog = dict(zip(num_events_catalog.keys(),
              #                         [[j for i in num_events_catalog.values() for j in i.keys()]]))
              # BUG BUG BUG end
  
  ##            # DEBUG DEBUG DEBUG
  ##            # This is probably only useful to make sure we don't muck
  ##            # things up, right?
  ##            # Figure out across how many sites this sample has been spread.
  ##            if num_sites == 1:
  ##                self.logger.info("    sample is contained at a single site")
  ##            else:
  ##                self.logger.info("    sample is spread across %d sites" % \
  ##                                 num_sites)
  ##            if num_sites < 1:
  ##                # NOTE: This _should not_ happen with any valid dataset.
  ##                self.logger.warning("  --> skipping dataset which is not " \
  ##                                    "hosted anywhere")
  ##            # DEBUG DEBUG DEBUG end
  
              # Now put everything in a place where we can find it again
              # if we need it.
              self.datasets_information[dataset_name] = {}
              self.datasets_information[dataset_name]["runs"] = runs
              self.datasets_information[dataset_name]["cmssw_version"] = \
                                                                       cmssw_version
              self.datasets_information[dataset_name]["globaltag"] = globaltag
              self.datasets_information[dataset_name]["datatype"] = datatype
              self.datasets_information[dataset_name]["num_events"] = num_events
              self.datasets_information[dataset_name]["mirrored"] = mirror_catalog
              self.datasets_information[dataset_name]["sites"] = sites_catalog
  
              # Each run of each dataset has a different CASTOR output
              # path.
              castor_path_common = self.create_castor_path_name_common(dataset_name)
              self.logger.info("    output will go into `%s'" % \
                               castor_path_common)
  
              castor_paths = dict(list(zip(runs,
                                      [self.create_castor_path_name_special(dataset_name, i, castor_path_common) \
                                       for i in runs])))
              for path_name in castor_paths.values():
                  self.logger.debug("      %s" % path_name)
              self.datasets_information[dataset_name]["castor_path"] = \
                                                                     castor_paths
function_tokens: def build_datasets_information ( self ) : # Get a list of runs in the dataset. # NOTE: The harvesting has to be done run-by-run, so we # split up datasets based on the run numbers. Strictly # speaking this is not (yet?) necessary for Monte Carlo # since all those samples use run number 1. Still, this # general approach should work for all samples. # Now loop over all datasets in the list and process them. # NOTE: This processing has been split into several loops # to be easier to follow, sacrificing a bit of efficiency. self . datasets_information = { } self . logger . info ( "Collecting information for all datasets to process" ) dataset_names = sorted ( self . datasets_to_use . keys ( ) ) for dataset_name in dataset_names : # Tell the user which dataset: nice with many datasets. sep_line = "-" * 30 self . logger . info ( sep_line ) self . logger . info ( "  `%s'" % dataset_name ) self . logger . info ( sep_line ) runs = self . dbs_resolve_runs ( dataset_name ) self . logger . info ( "    found %d run(s)" % len ( runs ) ) if len ( runs ) > 0 : self . logger . debug ( "      run number(s): %s" % ", " . join ( [ str ( i ) for i in runs ] ) ) else : # DEBUG DEBUG DEBUG # This should never happen after the DBS checks. self . logger . warning ( "  --> skipping dataset " "without any runs" ) assert False , "Panic: found a dataset without runs " "after DBS checks!" # DEBUG DEBUG DEBUG end cmssw_version = self . dbs_resolve_cmssw_version ( dataset_name ) self . logger . info ( "    found CMSSW version `%s'" % cmssw_version ) # Figure out if this is data or MC. datatype = self . dbs_resolve_datatype ( dataset_name ) self . logger . info ( "    sample is data or MC? --> %s" % datatype ) ### # Try and figure out the GlobalTag to be used. if self . globaltag is None : globaltag = self . dbs_resolve_globaltag ( dataset_name ) else : globaltag = self . globaltag self . logger . info ( "    found GlobalTag `%s'" % globaltag ) # DEBUG DEBUG DEBUG if globaltag == "" : # Actually we should not even reach this point, after # our dataset sanity checks. assert datatype == "data" , "ERROR Empty GlobalTag for MC dataset!!!" # DEBUG DEBUG DEBUG end ### # DEBUG DEBUG DEBUG #tmp = self.dbs_check_dataset_spread_old(dataset_name) # DEBUG DEBUG DEBUG end sites_catalog = self . dbs_check_dataset_spread ( dataset_name ) # Extract the total event counts. num_events = { } for run_number in sites_catalog . keys ( ) : num_events [ run_number ] = sites_catalog [ run_number ] [ "all_sites" ] del sites_catalog [ run_number ] [ "all_sites" ] # Extract the information about whether or not datasets # are mirrored. mirror_catalog = { } for run_number in sites_catalog . keys ( ) : mirror_catalog [ run_number ] = sites_catalog [ run_number ] [ "mirrored" ] del sites_catalog [ run_number ] [ "mirrored" ] # BUG BUG BUG # I think I could now get rid of that and just fill the # "sites" entry with the `inverse' of this # num_events_catalog(?). #num_sites = self.dbs_resolve_dataset_number_of_sites(dataset_name) #sites_catalog = self.dbs_check_dataset_spread(dataset_name) #sites_catalog = dict(zip(num_events_catalog.keys(), #                         [[j for i in num_events_catalog.values() for j in i.keys()]])) # BUG BUG BUG end ##            # DEBUG DEBUG DEBUG ##            # This is probably only useful to make sure we don't muck ##            # things up, right? ##            # Figure out across how many sites this sample has been spread. ##            if num_sites == 1: ##                self.logger.info("    sample is contained at a single site") ##            else: ##                self.logger.info("    sample is spread across %d sites" % \ ##                                 num_sites) ##            if num_sites < 1: ##                # NOTE: This _should not_ happen with any valid dataset. ##                self.logger.warning("  --> skipping dataset which is not " \ ##                                    "hosted anywhere") ##            # DEBUG DEBUG DEBUG end # Now put everything in a place where we can find it again # if we need it. self . datasets_information [ dataset_name ] = { } self . datasets_information [ dataset_name ] [ "runs" ] = runs self . datasets_information [ dataset_name ] [ "cmssw_version" ] = cmssw_version self . datasets_information [ dataset_name ] [ "globaltag" ] = globaltag self . datasets_information [ dataset_name ] [ "datatype" ] = datatype self . datasets_information [ dataset_name ] [ "num_events" ] = num_events self . datasets_information [ dataset_name ] [ "mirrored" ] = mirror_catalog self . datasets_information [ dataset_name ] [ "sites" ] = sites_catalog # Each run of each dataset has a different CASTOR output # path. castor_path_common = self . create_castor_path_name_common ( dataset_name ) self . logger . info ( "    output will go into `%s'" % castor_path_common ) castor_paths = dict ( list ( zip ( runs , [ self . create_castor_path_name_special ( dataset_name , i , castor_path_common ) for i in runs ] ) ) ) for path_name in castor_paths . values ( ) : self . logger . debug ( "      %s" % path_name ) self . datasets_information [ dataset_name ] [ "castor_path" ] = castor_paths