<commit_before>/*
 * This file is part of a small series of tutorial,
 * which aims to demonstrate key features of the GASPI
 * standard by means of small but expandable examples.
 * Conceptually the tutorial follows a MPI course
 * developed by EPCC and HLRS.
 *
 * Contact point for the MPI tutorial:
 *                 <EMAIL>
 * Contact point for the GASPI tutorial:
 *                 <EMAIL>
 *                 <EMAIL>
 *                 <EMAIL>
 */

#include "assert.h"
#include "constant.h"
#include "data.h"
#include "now.h"
#include "mm_pause.h"
#include "success_or_die.h"
#include "queue.h"
#include "threads.h"

#include <malloc.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <omp.h>

// threadprivate rangelist data
static int tStart = -1;
#pragma omp threadprivate(tStart)
static int tStop  = -2;
#pragma omp threadprivate(tStop)

int main (int argc, char *argv[])
{
  int i;
  int nProc, iProc;
  int provided, required = MPI_THREAD_FUNNELED;
  MPI_Init_thread(&argc, &argv, required, &provided);
  ASSERT(required == MPI_THREAD_FUNNELED);

  MPI_Comm_rank (MPI_COMM_WORLD, &iProc);
  MPI_Comm_size (MPI_COMM_WORLD, &nProc);

  gaspi_rank_t iProcG, nProcG;
  SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK));
  SUCCESS_OR_DIE (gaspi_proc_rank (&iProcG));
  SUCCESS_OR_DIE (gaspi_proc_num (&nProcG));
  ASSERT(iProc == iProcG);
  ASSERT(nProc == nProcG);

  // num threads
  omp_set_num_threads(nThreads);

  // assignment per proc, i-direction 
  int mSize = M_SZ/nProc;
  ASSERT(M_SZ % nProc == 0); 
  const int mStart = iProc*mSize;
  const int mStop  = (iProc+1)*mSize;

  // allocate segments for source, work
  gaspi_segment_id_t const source_id = 0;
  SUCCESS_OR_DIE ( gaspi_segment_create
                   ( source_id
                     , mSize * M_SZ * sizeof (double)
                     , GASPI_GROUP_ALL
                     , GASPI_BLOCK
                     , GASPI_MEM_UNINITIALIZED
                     ));
  gaspi_pointer_t source_array;
  SUCCESS_OR_DIE ( gaspi_segment_ptr ( source_id, &source_array) );
  ASSERT (source_array != 0);

  gaspi_segment_id_t const work_id = 1;
  SUCCESS_OR_DIE ( gaspi_segment_create
                   ( work_id
                     , mSize * M_SZ * sizeof (double)
                     , GASPI_GROUP_ALL
                     , GASPI_BLOCK
                     , GASPI_MEM_UNINITIALIZED
                     ));
  gaspi_pointer_t work_array;
  SUCCESS_OR_DIE ( gaspi_segment_ptr ( work_id, &work_array) );
  ASSERT (work_array != 0);

  // allocate target array
  ASSERT(mSize % CL == 0);
  double *target_array = memalign(CL* sizeof (double), mSize * M_SZ * sizeof (double));

  // assignment per thread
  int tSize = M_SZ/nThreads;
  if (M_SZ % nThreads != 0) 
    {
      tSize++;
    }

  // alloc (max) M_SZ blocks per process 
  block_t (*block) = malloc( M_SZ * sizeof(block_t));	  
  int block_num = 0;

  data_init(block
	    , &block_num
	    , tSize
	    , mSize
	    );

  // init thread local data, set thread range (tStart <= row <= tStop)
#pragma omp parallel default (none) shared(block, block_num, \
  	    mSize, source_array, work_array, target_array, stdout, stderr)
  {
    int const tid = omp_get_thread_num();  
    data_init_tlocal(mStart
		     , mStop
		     , block
		     , block_num
		     , &tStart
		     , &tStop
		     , tid
		     , source_array
		     , work_array
		     , target_array
		     , mSize
		     );
  }

  // mutual handshake schedule - requires even nProc 
  ASSERT(nProc % 2 == 0);
  gaspi_rank_t to[nProc-1][nProc];
  for (i = 0; i < nProc - 1 ; ++i)
    {
      int k;
      to[i][i] = nProc-1;
      to[i][nProc-1] = i;

      for (k = 1; k < nProc/2 ; ++k)
	{
	  int d1 = (i - k + (nProc-1)) % (nProc-1);	      
	  int d2 = (i + k + (nProc-1)) % (nProc-1);	      
	  to[i][d1] = d2;
	  to[i][d2] = d1;
	}
    }	  

  MPI_Barrier(MPI_COMM_WORLD);

  int iter;
  double median[NITER];

  for (iter = 0; iter < NITER; iter++) 
    {
      double time = -now();
      MPI_Barrier(MPI_COMM_WORLD);
#pragma omp parallel default (none) shared(iter, block_num, iProc, nProc, to, \
	    block, source_array, work_array, target_array, mSize, stdout,stderr)
      {
	// GASPI all-to-all - single comm phase
	if (this_is_the_first_thread())
	  {
	    gaspi_queue_id_t queue_id = 0;
	    int k;
	    for (k = 0; k < nProc - 1 ; ++k)
	      {
		gaspi_rank_t target = to[k][iProc];
		const int len = mSize * mSize * sizeof(double); 
		const int toffset = iProc * mSize;<commit_msg>minor fix<commit_after>/*
 * This file is part of a small series of tutorial,
 * which aims to demonstrate key features of the GASPI
 * standard by means of small but expandable examples.
 * Conceptually the tutorial follows a MPI course
 * developed by EPCC and HLRS.
 *
 * Contact point for the MPI tutorial:
 *                 <EMAIL>
 * Contact point for the GASPI tutorial:
 *                 <EMAIL>
 *                 <EMAIL>
 *                 <EMAIL>
 */

#include "assert.h"
#include "constant.h"
#include "data.h"
#include "now.h"
#include "mm_pause.h"
#include "success_or_die.h"
#include "queue.h"
#include "threads.h"

#include <malloc.h>
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <omp.h>

// threadprivate rangelist data
static int tStart = -1;
#pragma omp threadprivate(tStart)
static int tStop  = -2;
#pragma omp threadprivate(tStop)

int main (int argc, char *argv[])
{
  int i;
  int nProc, iProc;
  int provided, required = MPI_THREAD_FUNNELED;
  MPI_Init_thread(&argc, &argv, required, &provided);
  ASSERT(required == MPI_THREAD_FUNNELED);

  MPI_Comm_rank (MPI_COMM_WORLD, &iProc);
  MPI_Comm_size (MPI_COMM_WORLD, &nProc);

  gaspi_rank_t iProcG, nProcG;
  SUCCESS_OR_DIE (gaspi_proc_init (GASPI_BLOCK));
  SUCCESS_OR_DIE (gaspi_proc_rank (&iProcG));
  SUCCESS_OR_DIE (gaspi_proc_num (&nProcG));
  ASSERT(iProc == iProcG);
  ASSERT(nProc == nProcG);

  // num threads
  omp_set_num_threads(nThreads);

  // assignment per proc, i-direction 
  int mSize = M_SZ/nProc;
  ASSERT(M_SZ % nProc == 0); 
  const int mStart = iProc*mSize;
  const int mStop  = (iProc+1)*mSize;

  // allocate segments for source, work
  gaspi_segment_id_t const source_id = 0;
  SUCCESS_OR_DIE ( gaspi_segment_create
                   ( source_id
                     , mSize * M_SZ * sizeof (double)
                     , GASPI_GROUP_ALL
                     , GASPI_BLOCK
                     , GASPI_MEM_UNINITIALIZED
                     ));
  gaspi_pointer_t source_array;
  SUCCESS_OR_DIE ( gaspi_segment_ptr ( source_id, &source_array) );
  ASSERT (source_array != 0);

  gaspi_segment_id_t const work_id = 1;
  SUCCESS_OR_DIE ( gaspi_segment_create
                   ( work_id
                     , mSize * M_SZ * sizeof (double)
                     , GASPI_GROUP_ALL
                     , GASPI_BLOCK
                     , GASPI_MEM_UNINITIALIZED
                     ));
  gaspi_pointer_t work_array;
  SUCCESS_OR_DIE ( gaspi_segment_ptr ( work_id, &work_array) );
  ASSERT (work_array != 0);

  // allocate target array
  ASSERT(mSize % CL == 0);
  double *target_array = memalign(CL* sizeof (double), mSize * M_SZ * sizeof (double));

  // assignment per thread
  int tSize = M_SZ/nThreads;
  if (M_SZ % nThreads != 0) 
    {
      tSize++;
    }

  // alloc (max) M_SZ blocks per process 
  block_t (*block) = malloc( M_SZ * sizeof(block_t));	  
  int block_num = 0;

  data_init(block
	    , &block_num
	    , tSize
	    , mSize
	    );

  // init thread local data, set thread range (tStart <= row <= tStop)
#pragma omp parallel default (none) shared(block, block_num, \
  	    mSize, source_array, work_array, target_array, stdout, stderr)
  {
    int const tid = omp_get_thread_num();  
    data_init_tlocal(mStart
		     , mStop
		     , block
		     , block_num
		     , &tStart
		     , &tStop
		     , tid
		     , source_array
		     , work_array
		     , target_array
		     , mSize
		     );
  }

  // mutual handshake schedule - requires even nProc 
  ASSERT(nProc % 2 == 0);
  gaspi_rank_t to[nProc][nProc-1];
  for (i = 0; i < nProc - 1 ; ++i)
    {
      int k;
      to[i][i] = nProc-1;
      to[nProc-1][i] = i;

      for (k = 1; k < nProc/2 ; ++k)
	{
	  int d1 = (i - k + (nProc-1)) % (nProc-1);	      
	  int d2 = (i + k + (nProc-1)) % (nProc-1);	      
	  to[d1][i] = d2;
	  to[d2][i] = d1;
	}
    }	  

  MPI_Barrier(MPI_COMM_WORLD);

  int iter;
  double median[NITER];

  for (iter = 0; iter < NITER; iter++) 
    {
      double time = -now();
      MPI_Barrier(MPI_COMM_WORLD);
#pragma omp parallel default (none) shared(iter, block_num, iProc, nProc, to, \
	    block, source_array, work_array, target_array, mSize, stdout,stderr)
      {
	// GASPI all-to-all - single comm phase
	if (this_is_the_first_thread())
	  {
	    gaspi_queue_id_t queue_id = 0;
	    int k;
	    for (k = 0; k < nProc - 1 ; ++k)
	      {
		gaspi_rank_t target = to[iProc][k];
		const int len = mSize * mSize * sizeof(double); 
		const int toffset = iProc * mSize;
{'fraction_non_alphanumeric': 0.1157941437444543, 'fraction_numerical': 0.0075421472937000885, 'mean_word_length': 2.4129447388342165, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4596652', 'n_tokens_mistral': 3503, 'n_tokens_neox': 3260, 'n_words': 812}