<commit_before># Copyright 2013 University of Chicago and Argonne National Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

# Turbine builtin container operations

# Rule debug token conventions:
#  1) Use shorthand notation from Turbine Internals Guide
#  2) Preferably, just specify output TDs.  Input TDs are reported by
#     the WAITING TRANSFORMS list and the rule debugging lines

namespace eval turbine {
    namespace export container_f_get container_f_insert
    namespace export f_reference
    namespace export f_container_create_nested

    # build array by inserting items into a container starting at 0
    # close: decrement writers count at end
    proc array_build { c elems close } {
      set n [ llength $elems ]
      log "container_build: <$c> $n elems, close $close"
      if { $n > 0 } {
        for { set i 0 } { $i < $n } { incr i } {
          set elem [ lindex $elems $i ]
          set drops 0
          if { $close && $i == [ expr $n - 1 ] } {
            set drops 1
          }
          adlb::insert $c $i $elem $drops
        }
      } else {
        adlb::slot_drop $c
      }
    }

    # Just like adlb::container_reference but add logging
    proc container_reference { c i r type } {
        log "creating reference: <$c>\[$i\] <- <*$r> ($type)"
        adlb::container_reference $c $i $r $type
        # TODO: need to move refcount from container to referenced item
        # once reference set
    }

    # Same as container_lookup, but fail if item does not exist
    proc container_lookup_checked { c i } {
        set res [ container_lookup $c $i ]
        if { $res == 0 } {
            error "lookup failed: container_lookup <$c>\[$i\]"
        }
        return $res
    }

    # When i is closed, set d := c[i] (by value copy)
    # d: the destination, an integer
    # inputs: [ list c i ]
    # c: the container
    # i: the subscript (any type)
    proc container_f_get_integer { parent d inputs } {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]

        rule "container_f_get-$c-$i" $i $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_f_get_integer_body $d $c $i"
    }

    proc container_f_get_integer_body { d c i } {
        set s [ retrieve $i ]
        set t [ container_lookup $c $s ]
        if { $t == 0 } {
            error "lookup failed: container_f_get <$c>\[$s\]"
        }
        set value [ retrieve_integer $t ]
        store_integer $d $value
    }

    # When i is closed, set c[i] := d (by insertion)
    # inputs: [ list c i d ]
    # c: the container
    # i: the subscript (any type)
    # d: the data
    # outputs: ignored.  To block on this, use turbine::reference
    # Note: assume slot kept open by other process
    proc container_f_insert { parent outputs inputs {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set d [ lindex $inputs 2 ]
        nonempty c i d

        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_f_insert-$c-$i" $i $turbine::LOCAL $adlb::RANK_ANY \
            [ list turbine::container_f_insert_body $c $i $d ]
    }

    proc container_f_insert_body { c i d } {
        set s [ retrieve $i ]
        container_insert $c $s $d 1
    }

    # When i and r are closed, set c[i] := *(r)
    # inputs: [ list c i r ]
    # r: a reference to a turbine ID
    proc container_f_deref_insert { parent outputs inputs {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set r [ lindex $inputs 2 ]

        nonempty c i r
        
        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_f_deref_insert-$c-$i" "$i $r" $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_f_deref_insert_body $c $i $r"
    }

    proc container_f_deref_insert_body { c i r } {
        set t1 [ retrieve_decr_integer $i ]
        set d [ retrieve_decr $r ]
        container_insert $c $t1 $d 1
    }

    # When r is closed, set c[i] := *(r)
    # inputs: [ list c i r ]
    # i: an integer which is the index to insert into
    # r: a reference to a turbine ID
    proc container_deref_insert { parent outputs inputs {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set r [ lindex $inputs 2 ]

        nonempty c i r
        
        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_deref_insert-$c-$i" "$r" $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_deref_insert_body $c $i $r"
    }

    proc container_deref_insert_body { c i r } {
        set d [ retrieve_decr $r ]
        # Refcount from reference passed to container
        container_insert $c $i $d
    }

    # Immediately insert data into container without affecting open slot count
    # c: the container
    # i: the subscript
    # d: the data
    # outputs: ignored.
    proc container_immediate_insert { c i d } {
        # adlb::slot_create $c
        container_insert $c $i $d 0
    }
<commit_msg>piggybacking for containers<commit_after># Copyright 2013 University of Chicago and Argonne National Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

# Turbine builtin container operations

# Rule debug token conventions:
#  1) Use shorthand notation from Turbine Internals Guide
#  2) Preferably, just specify output TDs.  Input TDs are reported by
#     the WAITING TRANSFORMS list and the rule debugging lines

namespace eval turbine {
    namespace export container_f_get container_f_insert
    namespace export f_reference
    namespace export f_container_create_nested

    # build array by inserting items into a container starting at 0
    # close: decrement writers count at end
    proc array_build { c elems close } {
      set n [ llength $elems ]
      log "container_build: <$c> $n elems, close $close"
      if { $n > 0 } {
        for { set i 0 } { $i < $n } { incr i } {
          set elem [ lindex $elems $i ]
          set drops 0
          if { $close && $i == [ expr $n - 1 ] } {
            set drops 1
          }
          adlb::insert $c $i $elem $drops
        }
      } else {
        adlb::slot_drop $c
      }
    }

    # Just like adlb::container_reference but add logging
    proc container_reference { c i r type } {
        log "creating reference: <$c>\[$i\] <- <*$r> ($type)"
        adlb::container_reference $c $i $r $type
        # TODO: need to move refcount from container to referenced item
        # once reference set
    }

    # Same as container_lookup, but fail if item does not exist
    proc container_lookup_checked { c i } {
        set res [ container_lookup $c $i ]
        if { $res == 0 } {
            error "lookup failed: container_lookup <$c>\[$i\]"
        }
        return $res
    }

    # When i is closed, set d := c[i] (by value copy)
    # d: the destination, an integer
    # inputs: [ list c i ]
    # c: the container
    # i: the subscript (any type)
    proc container_f_get_integer { parent d inputs } {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]

        rule "container_f_get-$c-$i" $i $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_f_get_integer_body $d $c $i"
    }

    proc container_f_get_integer_body { d c i } {
        set s [ retrieve $i ]
        set t [ container_lookup $c $s ]
        if { $t == 0 } {
            error "lookup failed: container_f_get <$c>\[$s\]"
        }
        set value [ retrieve_integer $t ]
        store_integer $d $value
    }

    # When i is closed, set c[i] := d (by insertion)
    # inputs: [ list c i d ]
    # c: the container
    # i: the subscript (any type)
    # d: the data
    # outputs: ignored.  To block on this, use turbine::reference
    # Note: assume slot kept open by other process
    proc container_f_insert { parent outputs inputs {slot_drops 1} {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set d [ lindex $inputs 2 ]
        nonempty c i d

        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_f_insert-$c-$i" $i $turbine::LOCAL $adlb::RANK_ANY \
            [ list turbine::container_f_insert_body $c $i $d $slot_drops ]
    }

    proc container_f_insert_body { c i d slot_drops } {
        set s [ retrieve $i ]
        container_insert $c $s $d $slot_drops
    }

    # When i and r are closed, set c[i] := *(r)
    # inputs: [ list c i r ]
    # r: a reference to a turbine ID
    proc container_f_deref_insert { parent outputs inputs {slot_drops 1} {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set r [ lindex $inputs 2 ]

        nonempty c i r
        
        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_f_deref_insert-$c-$i" "$i $r" $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_f_deref_insert_body $c $i $r $slot_drops"
    }

    proc container_f_deref_insert_body { c i r slot_drops } {
        set t1 [ retrieve_decr_integer $i ]
        set d [ retrieve_decr $r ]
        container_insert $c $t1 $d $slot_drops
    }

    # When r is closed, set c[i] := *(r)
    # inputs: [ list c i r ]
    # i: an integer which is the index to insert into
    # r: a reference to a turbine ID
    proc container_deref_insert { parent outputs inputs {slot_drops 1} {slot_create 1}} {
        set c [ lindex $inputs 0 ]
        set i [ lindex $inputs 1 ]
        set r [ lindex $inputs 2 ]

        nonempty c i r
        
        if { $slot_create } {
            adlb::slot_create $c
        }

        rule "container_deref_insert-$c-$i" "$r" $turbine::LOCAL $adlb::RANK_ANY \
            "turbine::container_deref_insert_body $c $i $r $slot_drops"
    }

    proc container_deref_insert_body { c i r slot_drops } {
        set d [ retrieve_decr $r ]
        # Refcount from reference passed to container
        container_insert $c $i $d $slot_drops
    }

    # Immediately insert data into container without affecting open slot count
    # c: the container
    # i: the subscript
    # d: the data
    # outputs: ignored.
    proc container_immediate_insert { c i d {drops 0} } {
        # adlb::slot_create $c
        container_insert $c $i $d $drops
    }

{'fraction_non_alphanumeric': 0.0924769027716674, 'fraction_numerical': 0.006599208095028597, 'mean_word_length': 2.208921513269339, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4588222', 'n_tokens_mistral': 3652, 'n_tokens_neox': 3508, 'n_words': 1307}