<commit_before>import math
import comms

template `&&`(x: char): untyped = cast[pointer](unsafeAddr(x))
#template `&&`(x: int): untyped = cast[pointer](unsafeAddr(x))

type
  RankScatterDescriptor*[T] = object
    rank*: int
    data*: T
  RankScatterSeq*[T] = seq[RankScatterDescriptor[T]]
  RankScatterDescriptor2*[T] = object
    srank*: int
    desc*: RankScatterDescriptor[T]
  RankScatterMem* = object
    rank*: int
    bytes*: int
    data*: pointer
  RankScatterMemSeq* = seq[RankScatterMem]

proc exchange[T](desc: var seq[RankScatterDescriptor2[T]], comm: auto,
                 rank,srank,rrank,nranks: int, bitwise=true) =
  template send(r,d) = comm.pushSend(r,d)
  template recv(r,d) = comm.pushRecv(r,d)
  template waitSend() = comm.waitSends
  template waitRecv() = comm.waitRecvs
  template waitAll() = comm.waitAll
  #let n2 = nranks div 2
  #template rankDiff(x,y: int): int = abs(((nranks+x-y+n2) mod nranks) - n2)
  template rankDiff(x,y: int): int = ((nranks+x-y) mod nranks)

  var nmsgs = 0
  recv(rrank, nmsgs)
  var smsg = newSeq[RankScatterDescriptor2[T]](0)
  var rmsg = newSeq[RankScatterDescriptor2[T]](0)
  var newdesc = newSeq[RankScatterDescriptor2[T]](0)
  for i in 0..<desc.len:
    let drank = desc[i].desc.rank
    if bitwise:
      if (drank xor srank) < (drank xor rank):
        #echo "bsend: ", drank, "  ", rank, "  ", srank
        smsg.add desc[i]
      else:
        #echo "bsave: ", drank, "  ", rank, "  ", srank
        newdesc.add desc[i]
    else:
      let rd = rankDiff(drank,rank)
      let rds = rankDiff(drank,srank)
      let d = rd xor rds
      #if rds <= rd:
      if rd!=0 and (rds==0 or (d and (d-1)) == 0):
        #echo "send: ", drank, "  ", rank, "  ", srank
        smsg.add desc[i]
      else:
        #echo "save: ", drank, "  ", rank, "  ", srank
        newdesc.add desc[i]
  send(srank, smsg.len)
  waitRecv()
  if nmsgs>0:
    rmsg.setLen(nmsgs)
    recv(rrank, rmsg)
  waitSend()
  if smsg.len>0:
    send(srank, smsg)
  waitAll()

  if rrank > rank:
    newdesc.add rmsg
    shallowCopy desc, newdesc
  else:
    rmsg.add newdesc
    shallowCopy desc, rmsg

proc scatter*[T](desc0: RankScatterSeq[T], comm: auto): RankScatterSeq[T] =
  let myrank = comm.commrank
  let nranks = comm.commsize
  #template `&`(x: var char): untyped = cast[pointer](addr x)

  var desc = newSeq[RankScatterDescriptor2[T]](desc0.len)
  for i in 0..<desc0.len:
    desc[i].srank = myrank
    desc[i].desc = desc0[i]

  var d = 1
  while (nranks mod (2*d)) == 0:
    let srank = myrank xor d
    let rrank = srank
    exchange(desc, comm, myrank, srank, rrank, nranks)
    d = 2*d
<commit_msg>comms/rankScatter: replace shallowCopy with move for gcArc/gcOrc<commit_after>import math
import comms

template `&&`(x: char): untyped = cast[pointer](unsafeAddr(x))
#template `&&`(x: int): untyped = cast[pointer](unsafeAddr(x))

type
  RankScatterDescriptor*[T] = object
    rank*: int
    data*: T
  RankScatterSeq*[T] = seq[RankScatterDescriptor[T]]
  RankScatterDescriptor2*[T] = object
    srank*: int
    desc*: RankScatterDescriptor[T]
  RankScatterMem* = object
    rank*: int
    bytes*: int
    data*: pointer
  RankScatterMemSeq* = seq[RankScatterMem]

proc exchange[T](desc: var seq[RankScatterDescriptor2[T]], comm: auto,
                 rank,srank,rrank,nranks: int, bitwise=true) =
  template send(r,d) = comm.pushSend(r,d)
  template recv(r,d) = comm.pushRecv(r,d)
  template waitSend() = comm.waitSends
  template waitRecv() = comm.waitRecvs
  template waitAll() = comm.waitAll
  #let n2 = nranks div 2
  #template rankDiff(x,y: int): int = abs(((nranks+x-y+n2) mod nranks) - n2)
  template rankDiff(x,y: int): int = ((nranks+x-y) mod nranks)

  var nmsgs = 0
  recv(rrank, nmsgs)
  var smsg = newSeq[RankScatterDescriptor2[T]](0)
  var rmsg = newSeq[RankScatterDescriptor2[T]](0)
  var newdesc = newSeq[RankScatterDescriptor2[T]](0)
  for i in 0..<desc.len:
    let drank = desc[i].desc.rank
    if bitwise:
      if (drank xor srank) < (drank xor rank):
        #echo "bsend: ", drank, "  ", rank, "  ", srank
        smsg.add desc[i]
      else:
        #echo "bsave: ", drank, "  ", rank, "  ", srank
        newdesc.add desc[i]
    else:
      let rd = rankDiff(drank,rank)
      let rds = rankDiff(drank,srank)
      let d = rd xor rds
      #if rds <= rd:
      if rd!=0 and (rds==0 or (d and (d-1)) == 0):
        #echo "send: ", drank, "  ", rank, "  ", srank
        smsg.add desc[i]
      else:
        #echo "save: ", drank, "  ", rank, "  ", srank
        newdesc.add desc[i]
  send(srank, smsg.len)
  waitRecv()
  if nmsgs>0:
    rmsg.setLen(nmsgs)
    recv(rrank, rmsg)
  waitSend()
  if smsg.len>0:
    send(srank, smsg)
  waitAll()

  if rrank > rank:
    newdesc.add rmsg
    when defined(gcArc) or defined(gcOrc):
      desc = move newdesc
    else:
      shallowCopy desc, newdesc
  else:
    rmsg.add newdesc
    when defined(gcArc) or defined(gcOrc):
      desc = move rmsg
    else:
      shallowCopy desc, rmsg

proc scatter*[T](desc0: RankScatterSeq[T], comm: auto): RankScatterSeq[T] =
  let myrank = comm.commrank
  let nranks = comm.commsize
  #template `&`(x: var char): untyped = cast[pointer](addr x)

  var desc = newSeq[RankScatterDescriptor2[T]](desc0.len)
  for i in 0..<desc0.len:
    desc[i].srank = myrank
    desc[i].desc = desc0[i]

  var d = 1
  while (nranks mod (2*d)) == 0:
    let srank = myrank xor d
    let rrank = srank
    exchange(desc, comm, myrank, srank, rrank, nranks)
    d = 2*d

{'fraction_non_alphanumeric': 0.14527272727272728, 'fraction_numerical': 0.01090909090909091, 'mean_word_length': 3.07179866765359, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7525858', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2131, 'n_words': 520}