<commit_before>require 'active_record'
module Chawk
  # Models used in Chawk.  ActiveRecord classes.
  module Models
    class NodeInvalidator
      extend Forwardable
      def_delegators :sweeps, :size, :map

      def initialize(node)
        @node = node
        @sweeps = []
        @ranges = []
      end

      def <<(time)
        @node.ranges.where("start_ts <= ? AND stop_ts >= ?",time,time).each do |range|
          @ranges << range.id unless @ranges.include?(range.id)
        end
      end

      def invalidate!()
        @ranges.each{|r|Chawk::Models::Range.find(r).populate!}
      end
    end

    class NodeAggregator

      attr_reader :dataset

      def initialize(node)
        node.check_read_access
        if node.points.length > 0
          @dataset = node.points.to_a.reduce([]) {|ary,p| ary << p.value}
        end
      end

      def max()
        @dataset.max
      end

      def min()
        @dataset.min
      end

      def mean
        sum.to_f / @dataset.length
      end

      def sum
        @dataset.reduce(0) {|sum,p| sum+=p}
      end

      def count
        @dataset.length
      end

      def sumsqr
        @dataset.map {|x| x * x}.reduce(&:+)
      end

      def stdev
        m = mean
        Math.sqrt((sumsqr - count * m * m)/(count-1))
      end
    end

    # The Node, where most Chawk:Addr information is persisted..
    class Node < ActiveRecord::Base
      attr_accessor :agent
      after_initialize :init
      self.table_name_prefix = "chawk_"
      belongs_to :agent
      has_many :points
      has_many :values
      has_many :relations
      has_many :ranges, foreign_key: :parent_node_id

      attr_accessor :access

      def init
        @agent = nil
      end

      def clear_points!
        check_admin_access
        points.destroy_all
      end

      def _prepare_insert(val, ts, options)
        values = {value:val,observed_at:ts.to_f}
        if options[:meta]
          if options[:meta].is_a?(Hash)
            values[:meta] = options[:meta].to_json
          else
            raise ArgumentError, "Meta must be a JSON-representable Hash. #{options[:meta].inspect}"
          end
        end
        values
      end

      def _insert_value(val,ts,options={})
        self.values.create(_prepare_insert(val, ts, options))
        ts
      end

      def value_recognizer(item, dt, options={})
        case 
        when item.is_a?(String)
          _insert_value item,dt, options
        else
          raise ArgumentError, "Can't recognize format of data item. #{item.inspect}"
        end
      end

      def _add(args, options={})
        check_write_access
        options[:observed_at] ? dt = options[:observed_at] : dt = Time.now
        if args.is_a?(Array)
          args.each do |arg|
            yield arg, dt, options
          end
        else
            yield args, dt, options
        end
      end

      # @param args [Object, Array of Objects]
      # @param options [Hash] You can also pass in :meta and :timestamp 
      # Add an item or an array of items (one at a time) to the datastore.
      def add_values(args,options={})
        ni = NodeInvalidator.new(self)
        _add(args,options) do |arg,dt,options|
          ni << value_recognizer(arg, dt, options)
        end
        ni.invalidate!
      end

      # @param args [Object, Array of Objects]
      # @param options [Hash] You can also pass in :meta and :timestamp 
      # Add an item or an array of items (one at a time) to the datastore.
      def add_points(args,options={})
        ni = NodeInvalidator.new(self)
        _add(args,options) do |arg,dt,options|
          ni << point_recognizer(arg, dt, options)
        end
        ni.invalidate!
      end

      def _insert_point(val,ts,options={})        
        self.points.create(_prepare_insert(val, ts, options))
        ts
      end

      def _insert_point_hash(item,ts,options)
        if item['v'] && item['v'].is_a?(Integer)
          if item['t']
            _insert_point item['v'],item['t'], options
          else
            _insert_point item['v'],ts, options
          end
        else
          raise ArgumentError, "Hash must have 'v' key set to proper type.. #{item.inspect}"
        end
      end

      def _insert_point_array(item,options)
        if item.length == 2 && item[0].is_a?(Integer)
          _insert_point item[0],item[1], options
        else
          raise ArgumentError, "Array Items must be in [value,timestamp] format. #{item.inspect}"
        end<commit_msg>Refactoring<commit_after>require 'active_record'
module Chawk
  # Models used in Chawk.  ActiveRecord classes.
  module Models
    class NodeInvalidator
      extend Forwardable
      def_delegators :sweeps, :size, :map

      def initialize(node)
        @node = node
        @sweeps = []
        @ranges = []
      end

      def <<(time)
        @node.ranges.where("start_ts <= ? AND stop_ts >= ?",time,time).each do |range|
          @ranges << range.id unless @ranges.include?(range.id)
        end
      end

      def invalidate!()
        @ranges.each{|r|Chawk::Models::Range.find(r).populate!}
      end
    end

    class NodeAggregator

      attr_reader :dataset

      def initialize(node)
        node.check_read_access
        if node.points.length > 0
          @dataset = node.points.to_a.reduce([]) {|ary,p| ary << p.value}
        end
      end

      def max()
        @dataset.max
      end

      def min()
        @dataset.min
      end

      def mean
        sum.to_f / @dataset.length
      end

      def sum
        @dataset.reduce(0) {|sum,p| sum+=p}
      end

      def count
        @dataset.length
      end

      def sumsqr
        @dataset.map {|x| x * x}.reduce(&:+)
      end

      def stdev
        m = mean
        Math.sqrt((sumsqr - count * m * m)/(count-1))
      end
    end

    # The Node, where most Chawk:Addr information is persisted..
    class Node < ActiveRecord::Base
      attr_accessor :agent
      after_initialize :init
      self.table_name_prefix = "chawk_"
      belongs_to :agent
      has_many :points
      has_many :values
      has_many :relations
      has_many :ranges, foreign_key: :parent_node_id

      attr_accessor :access

      def init
        @agent = nil
      end

      def clear_points!
        check_admin_access
        points.destroy_all
      end

      def _prepare_insert(val, ts, options)
        values = {value:val,observed_at:ts.to_f}
        if options[:meta]
          if options[:meta].is_a?(Hash)
            values[:meta] = options[:meta].to_json
          else
            raise ArgumentError, "Meta must be a JSON-representable Hash. #{options[:meta].inspect}"
          end
        end
        values
      end

      def _insert_value(val,ts,options={})
        self.values.create(_prepare_insert(val, ts, options))
        ts
      end

      def value_recognizer(item, dt, options={})
        case 
        when item.is_a?(String)
          _insert_value item,dt, options
        else
          raise ArgumentError, "Can't recognize format of data item. #{item.inspect}"
        end
      end

      def _unravel(items)
        if items.is_a?(Array)
          items.each do |item|
            yield item
          end
        else
          yield items
        end
      end

      def _add(args, type, options={})
        check_write_access
        ni = NodeInvalidator.new(self)
        options[:observed_at] ? dt = options[:observed_at] : dt = Time.now
        _unravel(args) do |arg|
          case type
          when :point
            ni << point_recognizer(arg, dt, options)
          when :value
            ni << value_recognizer(arg, dt, options)
          end
        end
        ni.invalidate!
      end

      # @param args [Object, Array of Objects]
      # @param options [Hash] You can also pass in :meta and :timestamp 
      # Add an item or an array of items (one at a time) to the datastore.
      def add_values(args,options={})
        _add(args,:value, options)
      end

      # @param args [Object, Array of Objects]
      # @param options [Hash] You can also pass in :meta and :timestamp 
      # Add an item or an array of items (one at a time) to the datastore.
      def add_points(args,options={})
        _add(args,:point,options)
      end

      def _insert_point(val,ts,options={})        
        self.points.create(_prepare_insert(val, ts, options))
        ts
      end

      def _insert_point_hash(item,ts,options)
        if item['v'] && item['v'].is_a?(Integer)
          if item['t']
            _insert_point item['v'],item['t'], options
          else
            _insert_point item['v'],ts, options
          end
        else
          raise ArgumentError, "Hash must have 'v' key set to proper type.. #{item.inspect}"
        end
      end

      def _insert_point_array(item,options)
        if item.length == 2 && item[0].is_a?(Integer)
          _insert_point item[0],item[1], options
        else
          raise ArgumentError, "Array Items must be in [value,timestamp] format. #{item.inspect}"
        end
{'fraction_non_alphanumeric': 0.10025310883679982, 'fraction_numerical': 0.0015406624848684934, 'mean_word_length': 1.870499052432091, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2036128', 'n_tokens_mistral': 2797, 'n_tokens_neox': 2709, 'n_words': 752}