print_intro_line (fp, "Version", "%d.%d", mpiPi_vmajor, mpiPi_vminor);
print_intro_line (fp, "Version", "%d.%d.%d", mpiPi_vmajor, mpiPi_vminor, mpiPi_vpatch);
mpiPi_profile_print (FILE * fp){  int i;  char buf[256];  /*   * RecordIntroMaterial - Record all pertinant information to the final tracefile.   */  assert (fp);  fprintf (fp, "@ %s\n", mpiPi.toolname);  fprintf (fp, "@ Command : ");  for (i = 0; i < mpiPi.ac; i++)    {      fprintf (fp, "%s ", mpiPi.av[i]);    }  fprintf (fp, "\n");  print_intro_line (fp, "Version", "%d.%d", mpiPi_vmajor, mpiPi_vminor);  print_intro_line (fp, "MPIP Build date", "%s, %s", mpiPi_vdate,		    mpiPi_vtime);  {    char nowstr[128];    const struct tm *nowstruct;    char *fmtstr = "%Y %m %d %T";    nowstruct = localtime (&mpiPi.start_timeofday);    if (strftime (nowstr, 128, fmtstr, nowstruct) == (size_t) 0)      printf ("Could not get string from strftime()\n");    print_intro_line (fp, "Start time", "%s", nowstr);    nowstruct = localtime (&mpiPi.stop_timeofday);    if (strftime (nowstr, 128, fmtstr, nowstruct) == (size_t) 0)      printf ("Could not get string from strftime()\n");    print_intro_line (fp, "Stop time", "%s", nowstr);  }  print_intro_line (fp, "MPIP env var", "%s",		    mpiPi.envStr == NULL ? "[null]" : mpiPi.envStr);  print_intro_line (fp, "Collector Rank", "%d", mpiPi.collectorRank);  print_intro_line (fp, "Collector PID", "%d", mpiPi.procID);  print_intro_line (fp, "Final Output Dir", "%s", mpiPi.outputDir);  for (i = 0; i < mpiPi.size; i++)    {      print_intro_line (fp, "MPI Task Assignment", "%d %s",			mpiPi.global_task_info[i].rank,			mpiPi.global_task_info[i].hostname);    }  fprintf (fp, "\n");  /*****   ***** Start the statistics dump   *****/  /* -- calc the app runtime of each task */  mpiPi.global_app_time = 0;  mpiPi.global_mpi_time = 0;  for (i = 0; i < mpiPi.size; i++)    {      mpiPi_msg_debug ("app runtime for task %d is %g\n", i,		       mpiPi.global_task_info[i].app_time);      mpiPi.global_app_time += mpiPi.global_task_info[i].app_time;      mpiPi.global_task_info[i].mpi_time = 0;    }  /* -- sweep across the callsite (task indep) list and accumulate     each task's mpi time & sent message size */  {    int ac;    callsite_stats_t **av;    h_gather_data (mpiPi.global_callsite_stats, &ac, (void ***) &av);    for (i = 0; i < ac; i++)      {	mpiPi.global_task_info[av[i]->rank].mpi_time += av[i]->cumulativeTime;	mpiPi.global_mpi_time += av[i]->cumulativeTime;	mpiPi.global_mpi_size += av[i]->cumulativeDataSent;	mpiPi.global_mpi_io += av[i]->cumulativeIO;	mpiPi_msg_debug ("filename ptr: 0x%x functname ptr: 0x%x\n", av[i]->filename[0], av[i]->functname[0]);	mpiPi_msg_debug ("Callsite(%d,%d=[%s,%d,%s],%d) %g Cumulative=%g\n",			 av[i]->op,			 av[i]->csid,			 av[i]->filename[0],			 av[i]->lineno[0],			 av[i]->functname[0],			 av[i]->rank,			 av[i]->cumulativeTime,			 mpiPi.global_task_info[av[i]->rank].mpi_time);      }    free (av);  }  /* -- print the mpi/app times */  {    print_section_heading (fp, "MPI Time (seconds)");    fprintf (fp, "%-4s %10s %10s    %5s\n", "Task", "AppTime", "MPITime",	     "MPI%");    for (i = 0; i < mpiPi.size; i++)      {	double ratio;        if ( mpiPi.global_task_info[i].app_time > 0 )          {            ratio = (100.0 * mpiPi.global_task_info[i].mpi_time / 1e6) /              mpiPi.global_task_info[i].app_time;          }        else          ratio = 0;        	fprintf (fp, mpiP_Report_Formats[MPIP_MPI_TIME_FMT][mpiPi.reportFormat], i,		 mpiPi.global_task_info[i].app_time,		 mpiPi.global_task_info[i].mpi_time / 1e6, ratio);      }    fprintf (fp, mpiP_Report_Formats[MPIP_MPI_TIME_SUMMARY_FMT][mpiPi.reportFormat],	     mpiPi.global_app_time,	     mpiPi.global_mpi_time / 1e6,	     mpiPi.global_task_info[i].app_time > 0 ?              (100.0 * mpiPi.global_mpi_time / 1e6) / mpiPi.global_app_time :             0);  }  /* -- dump the source code locations */  {    int ac;    callsite_src_id_cache_entry_t **av;    int fileLenMax = 15;    int funcLenMax = 24;    h_gather_data (callsite_src_id_cache, &ac, (void ***) &av);    sprintf (buf, "Callsites: %d", ac);    qsort (av, ac, sizeof (void *), callsite_src_id_cache_sort_by_id);    print_section_heading (fp, buf);    /* Find longest file and function names for formatting */    for (i = 0; i < ac; i++)      {	int j, currlen;	for (j = 0;	     (j < MPIP_CALLSITE_STACK_DEPTH) && (av[i]->filename[j] != NULL);	     j++)	  {            currlen = strlen(av[i]->filename[j]);            fileLenMax = currlen > fileLenMax ? currlen : fileLenMax;            currlen = strlen(av[i]->functname[j]);            funcLenMax = currlen > funcLenMax ? currlen : funcLenMax;          }      }    fprintf (fp, "%3s %3s %-*s %5s %-*s %s\n",	     "ID", "Lev", fileLenMax, "File/Address", "Line", funcLenMax,              "Parent_Funct", "MPI_Call");    for (i = 0; i < ac; i++)      {	int j;	for (j = 0;	     (j < MPIP_CALLSITE_STACK_DEPTH) && (av[i]->filename[j] != NULL);	     j++)	  {            if ( av[i]->line[j] == 0 &&                  (strcmp(av[i]->filename[j], "[unknown]") == 0 ||                  strcmp(av[i]->functname[j], "[unknown]") == 0) )              {                fprintf (fp, "%3d %3d 0x%-*.*lx %-*s %s\n",                         av[i]->id,                         j,                         fileLenMax+4,                         sizeof(void*)*2,                         av[i]->pc[j],                          funcLenMax,                         av[i]->functname[j],                         (j ==                          0) ? &(mpiPi.lookup[av[i]->op -                                              mpiPi_BASE].name[4]) : "");              }            else              {                fprintf (fp, "%3d %3d %-*s %5d %-*s %s\n",                         av[i]->id,                         j,                         fileLenMax,                         av[i]->filename[j], av[i]->line[j],                          funcLenMax,                         av[i]->functname[j],                         (j ==                          0) ? &(mpiPi.lookup[av[i]->op -                                              mpiPi_BASE].name[4]) : "");              }          }      }    free (av);  }  {    int ac;    callsite_stats_t **av;    double timeCOV;    h_gather_data (mpiPi.global_callsite_stats_agg, &ac, (void ***) &av);    /* -- now that we have all the statistics in a queue, which is     * pointers to the data, we can sort it however we need to.     */    qsort (av, ac, sizeof (void *), callsite_sort_by_cumulative_time);    print_section_heading (fp,			   "Aggregate Time (top twenty, descending, milliseconds)");    if ( mpiPi.calcCOV )    {      fprintf (fp, "%-20s %4s %12s%6s  %6s  %6s\n", "Call", "Site", "Time  ",	       "App%", "MPI%", "COV");    }    else    {      fprintf (fp, "%-20s %4s %12s%6s  %6s\n", "Call", "Site", "Time  ",	       "App%", "MPI%");    }    for (i = 0; (i < 20) && (i < ac); i++)      {        if ( mpiPi.calcCOV )        {          timeCOV = calc_COV(av[i]->siteData, av[i]->siteDataIdx);          free(av[i]->siteData);  	  fprintf (fp, mpiP_Report_Formats[MPIP_AGGREGATE_COV_TIME_FMT][mpiPi.reportFormat],	  	   &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]), av[i]->csid,		   av[i]->cumulativeTime / 1000.0,		   100.0 * av[i]->cumulativeTime / (mpiPi.global_app_time * 1e6),		   mpiPi.global_mpi_time > 0 ?                    (100.0 * av[i]->cumulativeTime / mpiPi.global_mpi_time) : 0,                   timeCOV);	}	else	{          fprintf (fp, mpiP_Report_Formats[MPIP_AGGREGATE_TIME_FMT][mpiPi.reportFormat],                   &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]), av[i]->csid,                   av[i]->cumulativeTime / 1000.0,                   mpiPi.global_app_time > 0 ? 100.0 * av[i]->cumulativeTime / (mpiPi.global_app_time * 1e6):0,                   mpiPi.global_mpi_time > 0 ? 100.0 * av[i]->cumulativeTime / mpiPi.global_mpi_time : 0);        }      }    free (av);  }  {    int ac;    callsite_stats_t **av;    if ( mpiPi.global_mpi_size > 0 )    {      h_gather_data (mpiPi.global_callsite_stats_agg, &ac, (void ***) &av);      /* -- now that we have all the statistics in a queue, which is       * pointers to the data, we can sort it however we need to.       */      qsort (av, ac, sizeof (void *), callsite_sort_by_cumulative_size);      print_section_heading (fp,                             "Aggregate Sent Message Size (top twenty, descending, bytes)");      fprintf (fp, "%-20s %4s %10s %10s %10s %6s\n", "Call", "Site", "Count",                "Total", "Avrg", "Sent%");      for (i = 0; (i < 20) && (i < ac); i++)      {        if ( av[i]->cumulativeDataSent > 0 )        {          fprintf(fp, mpiP_Report_Formats[MPIP_AGGREGATE_MESS_FMT][mpiPi.reportFormat],                  &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]),                   av[i]->csid,                  av[i]->count,                  av[i]->cumulativeDataSent,                  av[i]->cumulativeDataSent/av[i]->count,                  av[i]->cumulativeDataSent * 100 / mpiPi.global_mpi_size);        }      }      free (av);    }  }#ifdef HAVE_MPI_IO  {    int ac;    callsite_stats_t **av;    if ( mpiPi.global_mpi_io > 0 )    {      h_gather_data (mpiPi.global_callsite_stats_agg, &ac, (void ***) &av);      /* -- now that we have all the statistics in a queue, which is       * pointers to the data, we can sort it however we need to.       */      qsort (av, ac, sizeof (void *), callsite_sort_by_cumulative_io);      print_section_heading (fp,                             "Aggregate I/O Size (top twenty, descending, bytes)");      fprintf (fp, "%-20s %4s %10s %10s %10s %6s\n", "Call", "Site", "Count",                "Total", "Avrg", "I/O%");      for (i = 0; (i < 20) && (i < ac); i++)      {        if ( av[i]->cumulativeIO > 0 )        {          fprintf(fp, mpiP_Report_Formats[MPIP_AGGREGATE_IO_FMT][mpiPi.reportFormat],                  &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]),                   av[i]->csid,                  av[i]->count,                  av[i]->cumulativeIO,                  av[i]->cumulativeIO/av[i]->count,                  av[i]->cumulativeIO * 100 / mpiPi.global_mpi_io);        }      }      free (av);    }  }#endif  {    int ac;    callsite_stats_t **av;    h_gather_data (mpiPi.global_callsite_stats, &ac, (void ***) &av);    /* -- now that we have all the statistics in a queue, which is     * pointers to the data, we can sort it however we need to.     */    qsort (av, ac, sizeof (void *), callsite_sort_by_name_id_rank);    sprintf (buf, "Callsite Time statistics (all, milliseconds): %d", ac);    print_section_heading (fp, buf);    fprintf (fp, "%-17s %4s %4s %6s %8s %8s %8s %6s %6s\n", "Name", "Site",	     "Rank", "Count", "Max", "Mean", "Min", "App%", "MPI%");    {      long long sCount = 0;      double sMin = DBL_MAX;      double sMax = 0;      double sCumulative = 0;      for (i = 0; i < ac; i++)	{	  if (i != 0 && (av[i]->csid != av[i - 1]->csid))	    {	      fprintf (fp,		       mpiP_Report_Formats[MPIP_CALLSITE_TIME_SUMMARY_FMT][mpiPi.reportFormat],		       &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),		       av[i - 1]->csid, "*", sCount, sMax / 1000.0,		       sCumulative / (sCount * 1000.0), sMin / 1000.0,		       mpiPi.global_app_time > 0 ? 100.0 * sCumulative / (mpiPi.global_app_time * 1e6) : 0,		       mpiPi.global_mpi_time > 0 ? 100.0 * sCumulative / mpiPi.global_mpi_time : 0);	      fprintf (fp, "\n");	      sCount = 0;	      sMax = 0;	      sMin = DBL_MAX;	      sCumulative = 0;	    }	  sCount += av[i]->count;	  sCumulative += av[i]->cumulativeTime;	  sMax = max (av[i]->maxDur, sMax);	  sMin = min (av[i]->minDur, sMin);	  if (mpiPi.global_task_info[av[i]->rank].mpi_time != 0 &&              (100.0 * av[i]->cumulativeTime /	       mpiPi.global_task_info[av[i]->rank].mpi_time)	      >= mpiPi.reportPrintThreshold)	    {	      fprintf (fp,		       mpiP_Report_Formats[MPIP_CALLSITE_TIME_RANK_FMT][mpiPi.reportFormat],		       &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]),		       av[i]->csid, av[i]->rank, av[i]->count,		       av[i]->maxDur / 1000.0,		       av[i]->cumulativeTime / (av[i]->count * 1000.0),		       av[i]->minDur / 1000.0,		       100.0 * av[i]->cumulativeTime /		       (mpiPi.global_task_info[av[i]->rank].app_time * 1e6),		       100.0 * av[i]->cumulativeTime /		       mpiPi.global_task_info[av[i]->rank].mpi_time);	    }	}      fprintf (fp, mpiP_Report_Formats[MPIP_CALLSITE_TIME_SUMMARY_FMT][mpiPi.reportFormat],	       &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),	       av[i - 1]->csid,	       "*",	       sCount,	       sMax / 1000.0,	       sCumulative / (sCount * 1000.0),	       sMin / 1000.0,	       mpiPi.global_app_time > 0 ? 100.0 * sCumulative / (mpiPi.global_app_time * 1e6) : 0,	       mpiPi.global_mpi_time > 0 ? 100.0 * sCumulative / mpiPi.global_mpi_time : 0);    }    free (av);  }  /*  Print Sent Data Section  */  {    int ac;    callsite_stats_t **av;    if ( mpiPi.global_mpi_size > 0 )    {      h_gather_data (mpiPi.global_callsite_stats, &ac, (void ***) &av);      /* -- now that we have all the statistics in a queue, which is       * pointers to the data, we can sort it however we need to.       */      qsort (av, ac, sizeof (void *), callsite_sort_by_name_id_rank);      sprintf (buf, "Callsite Message Sent statistics (all, sent bytes)");      print_section_heading (fp, buf);      fprintf (fp, "%-17s %4s %4s %7s %9s %9s %9s %9s\n", "Name", "Site",               "Rank", "Count", "Max", "Mean", "Min", "Sum");      {        long long sCount = 0;        double sMin = DBL_MAX;        double sMax = 0;        double sCumulative = 0;        int lastcsid = 0;        for (i = 0; i < ac; i++)	{	  if (i != 0 && sCumulative > 0 && (av[i]->csid != av[i - 1]->csid))          {            fprintf (fp,                     mpiP_Report_Formats[MPIP_CALLSITE_MESS_SUMMARY_FMT][mpiPi.reportFormat],                     &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),                     av[i - 1]->csid, "*", sCount, sMax,                     sCumulative / sCount, sMin,                     sCumulative);            sCount = 0;            sMax = 0;            sMin = DBL_MAX;            sCumulative = 0;          }	  if (av[i]->cumulativeDataSent > 0 )          {            sCount += av[i]->count;            sCumulative += av[i]->cumulativeDataSent;            sMax = max (av[i]->maxDataSent, sMax);            sMin = min (av[i]->minDataSent, sMin);            if ( lastcsid != 0 && lastcsid != av[i]->csid )              fprintf (fp, "\n");            fprintf (fp,                     mpiP_Report_Formats[MPIP_CALLSITE_MESS_RANK_FMT][mpiPi.reportFormat],                     &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]),                     av[i]->csid, av[i]->rank, av[i]->count,                     av[i]->maxDataSent,                     av[i]->cumulativeDataSent / av[i]->count,                     av[i]->minDataSent,                     av[i]->cumulativeDataSent);            lastcsid = av[i]->csid;          }	}              if ( sCumulative > 0 )        {          fprintf (fp, mpiP_Report_Formats[MPIP_CALLSITE_MESS_SUMMARY_FMT][mpiPi.reportFormat],                   &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),                   av[i - 1]->csid,                   "*",                   sCount,                   sMax,                   sCumulative / sCount,                   sMin,                   sCumulative);        }      }      free (av);    }  }#ifdef HAVE_MPI_IO  /*  Print I/O Section  */  {    int ac;    callsite_stats_t **av;    if ( mpiPi.global_mpi_io > 0 )    {      h_gather_data (mpiPi.global_callsite_stats, &ac, (void ***) &av);      /* -- now that we have all the statistics in a queue, which is       * pointers to the data, we can sort it however we need to.       */      qsort (av, ac, sizeof (void *), callsite_sort_by_name_id_rank);      sprintf (buf, "Callsite I/O statistics (all, I/O bytes)");      print_section_heading (fp, buf);      fprintf (fp, "%-17s %4s %4s %7s %9s %9s %9s %9s\n", "Name", "Site",               "Rank", "Count", "Max", "Mean", "Min", "Sum");      {        long long sCount = 0;        double sMin = DBL_MAX;        double sMax = 0;        double sCumulative = 0;        int lastcsid = 0;        for (i = 0; i < ac; i++)	{	  if (i != 0 && sCumulative > 0 && (av[i]->csid != av[i - 1]->csid))          {            fprintf (fp,                     mpiP_Report_Formats[MPIP_CALLSITE_IO_SUMMARY_FMT][mpiPi.reportFormat],                     &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),                     av[i - 1]->csid, "*", sCount, sMax,                     sCumulative / sCount, sMin,                     sCumulative);            sCount = 0;            sMax = 0;            sMin = DBL_MAX;            sCumulative = 0;          }	  if (av[i]->cumulativeIO > 0 )          {            sCount += av[i]->count;            sCumulative += av[i]->cumulativeIO;            sMax = max (av[i]->maxIO, sMax);            sMin = min (av[i]->minIO, sMin);            if ( lastcsid != 0 && lastcsid != av[i]->csid )              fprintf (fp, "\n");            fprintf (fp,                     mpiP_Report_Formats[MPIP_CALLSITE_IO_RANK_FMT][mpiPi.reportFormat],                     &(mpiPi.lookup[av[i]->op - mpiPi_BASE].name[4]),                     av[i]->csid, av[i]->rank, av[i]->count,                     av[i]->maxIO,                     av[i]->cumulativeIO / av[i]->count,                     av[i]->minIO,                     av[i]->cumulativeIO);            lastcsid = av[i]->csid;          }	}              if ( sCumulative > 0 )        {          fprintf (fp, mpiP_Report_Formats[MPIP_CALLSITE_IO_SUMMARY_FMT][mpiPi.reportFormat],                   &(mpiPi.lookup[av[i - 1]->op - mpiPi_BASE].name[4]),                   av[i - 1]->csid,                   "*",                   sCount,                   sMax,                   sCumulative / sCount,                   sMin,                   sCumulative);        }      }      free (av);    }  }#endif  print_section_heading (fp, "End of Report");  return 0;}