<commit_before># Turbine builtin functions for blob manipulation


namespace eval turbine {

  proc blob_size_async { stack out blob } {
    rule "blob_size-$out-$blob" "$blob" \
        $turbine::LOCAL "blob_size_body $out $blob"
  }

  proc blob_size_body { out blob } {
    set blob_val [ retrieve_decr_blob $blob ]
    set sz [ blob_size $blob_val ]
    store_integer $out $sz
    adlb::blob_free $blob
  }

  proc blob_size { blob_val } {
    return [ lindex $blob_val 1 ]
  }

  proc blob_from_string { stack result input } {
    rule "bfs-$input-$result" $input $turbine::LOCAL \
      "blob_from_string_body $input $result"
  }
  proc blob_from_string_body { input result } {
    set t [ retrieve_decr $input ]
    store_blob_string $result $t
  }

  proc string_from_blob { stack result input } {
    rule "sfb-$input-$result" $input $turbine::LOCAL \
      "string_from_blob_body $input $result"
  }
  proc string_from_blob_body { input result } {
    set s [ retrieve_decr_blob_string $input ]
    store_string $result $s
  }

  proc floats_from_blob { stack result input } {
      rule "ffb-$result-$input" $input $turbine::LOCAL \
          "floats_from_blob_body $result $input"
  }
  proc floats_from_blob_body { result input } {
      set s      [ SwiftBlob_sizeof_float ]
      set length [ SwiftBlob_length_get $input ]
      set n      [ expr $length / $s ]
      for { set i 0 } { $i < $n } { incr i } {
          set d [ SwiftBlob_double_get $input ]
          puts "$i : $d"
      }
  }

  # Container must be indexed from 0,N-1
  proc blob_from_floats { stack result input } {
    rule "bff-$input-$result" $input $turbine::LOCAL \
      "blob_from_floats_body $input $result"
  }
  proc blob_from_floats_body { container result } {

      set type [ container_typeof $container ]
      set N  [ adlb::container_size $container ]
      c::log "bff_body start"
      complete_container $container \
          "blob_from_floats_store $result $container $N"
  }
  # This is called when every entry in container is set
  proc blob_from_floats_store { result container N } {
    set A [ list ]
    for { set i 0 } { $i < $N } { incr i } {
      set td [ container_lookup $container $i ]<commit_msg>Finish floats_from_blob()<commit_after># Turbine builtin functions for blob manipulation

namespace eval turbine {

  proc blob_size_async { stack out blob } {
    rule "blob_size-$out-$blob" "$blob" \
        $turbine::LOCAL "blob_size_body $out $blob"
  }

  proc blob_size_body { out blob } {
    set blob_val [ retrieve_decr_blob $blob ]
    set sz [ blob_size $blob_val ]
    store_integer $out $sz
    adlb::blob_free $blob
  }

  proc blob_size { blob_val } {
    return [ lindex $blob_val 1 ]
  }

  proc blob_from_string { stack result input } {
    rule "bfs-$input-$result" $input $turbine::LOCAL \
      "blob_from_string_body $input $result"
  }
  proc blob_from_string_body { input result } {
    set t [ retrieve_decr $input ]
    store_blob_string $result $t
  }

  proc string_from_blob { stack result input } {
    rule "sfb-$input-$result" $input $turbine::LOCAL \
      "string_from_blob_body $input $result"
  }
  proc string_from_blob_body { input result } {
    set s [ retrieve_decr_blob_string $input ]
    store_string $result $s
  }

  proc floats_from_blob { stack result input } {
      rule "ffb-$result-$input" $input $turbine::LOCAL \
          "floats_from_blob_body $result $input"
  }
  proc floats_from_blob_body { result input } {
      log "floats_from_blob_body: result=<$result> input=<$input>"
      set s      [ SwiftBlob_sizeof_float ]
      set L [ adlb::retrieve_blob $input ]
      set p      [ lindex $L 0 ]
      set length [ lindex $L 1 ]
      set blob   [ new_SwiftBlob ]
      SwiftBlob_pointer_set $blob [ SwiftBlob_cast_to_pointer $p ]
      SwiftBlob_length_set  $blob $length

      set n [ expr $length / $s ]
      for { set i 0 } { $i < $n } { incr i } {
          set d [ SwiftBlob_double_get $blob $i ]
          literal t float $d
          container_immediate_insert $result $i $t
      }
      adlb::refcount_incr $result $adlb::WRITE_REFCOUNT -1
      adlb::blob_free $input
      log "floats_from_blob_body: done"
  }

  # Container must be indexed from 0,N-1
  proc blob_from_floats { stack result input } {
    rule "bff-$input-$result" $input $turbine::LOCAL \
      "blob_from_floats_body $input $result"
  }
  proc blob_from_floats_body { container result } {

      set type [ container_typeof $container ]
      set N  [ adlb::container_size $container ]
      c::log "bff_body start"
      complete_container $container \
          "blob_from_floats_store $result $container $N"
  }
  # This is called when every entry in container is set
  proc blob_from_floats_store { result container N } {
    set A [ list ]
    for { set i 0 } { $i < $N } { incr i } {
      set td [ container_lookup $container $i ]
{'fraction_non_alphanumeric': 0.09887640449438202, 'fraction_numerical': 0.002655771195097038, 'mean_word_length': 2.801242236024845, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6092651', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1631, 'n_words': 450}