<commit_before>{-# LANGUAGE DeriveDataTypeable, TypeFamilies, TypeOperators,
             FlexibleContexts #-}

module Main where

import Control.Monad

import Flow

import Kernel.Binning
import Kernel.Data
import Kernel.Facet
import Kernel.FFT
import Kernel.Gridder
import Kernel.IO

-- ----------------------------------------------------------------------------
-- ---                             Functional                               ---
-- ----------------------------------------------------------------------------

-- Abstract kernel signatures.
createGrid :: Flow UVGrid
createGrid = flow "create grid"
grid :: Flow Vis -> Flow GCFs -> Flow UVGrid -> Flow UVGrid
grid = flow "grid"
idft :: Flow UVGrid -> Flow Image
idft = flow "idft"
gcf :: Flow Vis -> Flow GCFs
gcf = flow "gcf"

createImage :: Flow Image
createImage = flow "create image"
facetSum :: Flow Image -> Flow Image -> Flow Image
facetSum = flow "facet sum"

-- | Compound gridder actor
gridder :: Flow Vis -> Flow GCFs -> Flow Image
gridder vis gcfs = idft (grid vis gcfs createGrid)

-- | Facetted compound gridder actor
facetGridder :: Flow Vis -> Flow GCFs -> Flow Image
facetGridder vis gcfs = facetSum (gridder vis gcfs) createImage

-- ----------------------------------------------------------------------------
-- ---                               Strategy                               ---
-- ----------------------------------------------------------------------------

gridderStrat :: Config -> Strategy ()
gridderStrat cfg = do
  let gpar = cfgGrid cfg
      gcfpar = cfgGCF cfg

  -- Make point domain for visibilities
  tdom <- makeRangeDomain 0 (cfgPoints cfg)

  -- Create ranged domains for image coordinates
  ldoms <- makeRangeDomain 0 (gridImageWidth gpar)
  mdoms <- makeRangeDomain 0 (gridImageHeight gpar)
  ldom <- split ldoms (gridFacets gpar)
  mdom <- split mdoms (gridFacets gpar)
  let lmdom = (ldom, mdom)

  -- Create ranged domains for grid coordinates
  udoms <- makeRangeDomain 0 (gridWidth gpar)
  vdoms <- makeRangeDomain 0 (gridHeight gpar)
  udom <- split udoms (gridTiles gpar)
  vdom <- split vdoms (gridTiles gpar)
  let uvdoms = (udoms, vdoms); uvdom = (udom, vdom)

  -- Create data flow for visibilities, read in
  vis <- bindNew $ oskarReader tdom (cfgInput cfg) 0 0

  -- Data flows we want to calculate
  let gcfs = gcf vis
      gridded = grid vis gcfs createGrid
      facets = gridder vis gcfs
      result = facetGridder vis gcfs
  distribute ldom ParSchedule $ distribute mdom ParSchedule $ do
    let rkern :: IsKernelDef kf => kf -> kf
        rkern = regionKernel ldom . regionKernel mdom

    -- Rotate visibilities
    rebind vis (rotateKernel cfg lmdom tdom)

    -- Create w-binned domain, split
    wdoms <- makeBinDomain (rkern $ binSizer gpar tdom uvdom vis)
    wdom <- split wdoms (gridBins gpar)

    -- Load GCFs
    distribute wdom SeqSchedule $
      bind gcfs (rkern $ gcfKernel gcfpar wdom)

    -- Bin visibilities (could distribute, but there's no benefit)
    rebind vis (rkern $ binner gpar tdom uvdom wdom)

    -- Bind kernel rules
    bindRule createGrid (rkern $ gridInit gcfpar uvdom)
    bindRule grid (rkern $ hints [floatHint] $ gridKernel gpar gcfpar uvdoms wdom uvdom)

    -- Run gridding distributed
    distribute vdom ParSchedule $ distribute udom ParSchedule $ do
      calculate gridded
<commit_msg>Change facetting strategy to reading in Oskar data per node<commit_after>{-# LANGUAGE DeriveDataTypeable, TypeFamilies, TypeOperators,
             FlexibleContexts #-}

module Main where

import Control.Monad

import Flow

import Kernel.Binning
import Kernel.Data
import Kernel.Facet
import Kernel.FFT
import Kernel.Gridder
import Kernel.IO

-- ----------------------------------------------------------------------------
-- ---                             Functional                               ---
-- ----------------------------------------------------------------------------

-- Abstract kernel signatures.
createGrid :: Flow UVGrid
createGrid = flow "create grid"
grid :: Flow Vis -> Flow GCFs -> Flow UVGrid -> Flow UVGrid
grid = flow "grid"
idft :: Flow UVGrid -> Flow Image
idft = flow "idft"
gcf :: Flow Vis -> Flow GCFs
gcf = flow "gcf"

createImage :: Flow Image
createImage = flow "create image"
facetSum :: Flow Image -> Flow Image -> Flow Image
facetSum = flow "facet sum"

-- | Compound gridder actor
gridder :: Flow Vis -> Flow GCFs -> Flow Image
gridder vis gcfs = idft (grid vis gcfs createGrid)

-- | Facetted compound gridder actor
facetGridder :: Flow Vis -> Flow GCFs -> Flow Image
facetGridder vis gcfs = facetSum (gridder vis gcfs) createImage

-- ----------------------------------------------------------------------------
-- ---                               Strategy                               ---
-- ----------------------------------------------------------------------------

gridderStrat :: Config -> Strategy ()
gridderStrat cfg = do
  let gpar = cfgGrid cfg
      gcfpar = cfgGCF cfg

  -- Make point domain for visibilities
  tdom <- makeRangeDomain 0 (cfgPoints cfg)

  -- Create ranged domains for image coordinates
  ldoms <- makeRangeDomain 0 (gridImageWidth gpar)
  mdoms <- makeRangeDomain 0 (gridImageHeight gpar)
  ldom <- split ldoms (gridFacets gpar)
  mdom <- split mdoms (gridFacets gpar)
  let lmdom = (ldom, mdom)

  -- Create ranged domains for grid coordinates
  udoms <- makeRangeDomain 0 (gridWidth gpar)
  vdoms <- makeRangeDomain 0 (gridHeight gpar)
  udom <- split udoms (gridTiles gpar)
  vdom <- split vdoms (gridTiles gpar)
  let uvdoms = (udoms, vdoms); uvdom = (udom, vdom)

  -- Data flows we want to calculate
  vis <- uniq $ flow "vis"
  let gcfs = gcf vis
      gridded = grid vis gcfs createGrid
      facets = gridder vis gcfs
      result = facetGridder vis gcfs

  distribute ldom ParSchedule $ distribute mdom ParSchedule $ do
    let rkern :: IsKernelDef kf => kf -> kf
        rkern = regionKernel ldom . regionKernel mdom

    -- Read visibilities in
    bind vis $ oskarReader tdom (cfgInput cfg) 0 0

    -- Rotate visibilities
    rebind vis (rotateKernel cfg lmdom tdom)

    -- Create w-binned domain, split
    wdoms <- makeBinDomain (rkern $ binSizer gpar tdom uvdom vis)
    wdom <- split wdoms (gridBins gpar)

    -- Load GCFs
    distribute wdom SeqSchedule $
      bind gcfs (rkern $ gcfKernel gcfpar wdom)

    -- Bin visibilities (could distribute, but there's no benefit)
    rebind vis (rkern $ binner gpar tdom uvdom wdom)

    -- Bind kernel rules
    bindRule createGrid (rkern $ gridInit gcfpar uvdom)
    bindRule grid (rkern $ hints [floatHint] $ gridKernel gpar gcfpar uvdoms wdom uvdom)

    -- Run gridding distributed
    distribute vdom ParSchedule $ distribute udom ParSchedule $ do
      calculate gridded

{'fraction_non_alphanumeric': 0.16342182890855458, 'fraction_numerical': 0.002064896755162242, 'mean_word_length': 1.9611353711790394, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4118080', 'n_tokens_mistral': 2019, 'n_tokens_neox': 1904, 'n_words': 701}