<commit_before>class Firebase
  def [](*names)
    if names.length == 0
      childByAutoId
    else
      childByAppendingPath(names.join('/'))
    end
  end
end

class FQuery

  EVENT_TYPES_MAP = {
    :child_added => FEventTypeChildAdded,
    :added => FEventTypeChildAdded,
    :child_moved => FEventTypeChildMoved,
    :moved => FEventTypeChildMoved,
    :child_changed => FEventTypeChildChanged,
    :changed => FEventTypeChildChanged,
    :child_removed => FEventTypeChildRemoved,
    :removed => FEventTypeChildRemoved,
    :value => FEventTypeValue
  }

  class Holder
    
    attr_accessor :map
    
    def initialize
      @map = NSMapTable.weakToStrongObjectsMapTable
      @own = NSMapTable.weakToStrongObjectsMapTable
    end

    def own_snapshot(snap)
      ref = snap.ref
      unless owned = @own.objectForKey(ref)
        owned = []
        @own.setObject(owned, forKey:ref)
      end
      owned.push snap
    end

    def track(fquery, handler, leaf)
      unless query_handles = @map.objectForKey(leaf)
        query_handles = {}
        @map.setObject(query_handles, forKey:leaf)
      end
      query_handles[handler] = fquery
    end

    def off(handle=nil, cleanup_only=false)
      object_context = !handle.is_a?(Integer) && handle
      handle = nil if object_context
      keyEnum = @map.keyEnumerator
      keys = []
      while key = keyEnum.nextObject
        keys.push key
      end
      while other = keys.pop
        next if object_context && other != object_context
        if hash = @map.objectForKey(other)
          hash_keys = [] + hash.keys
          while handler = hash_keys.pop
            if !handle || handler == handle
              fquery = hash[handler]
              @own.removeObjectForKey(fquery)
              hash.delete(handler)
              unless cleanup_only
                # p fquery, "removeObserverWithHandle", handler, caller
                fquery.removeObserverWithHandle(handler)
              end
              other_handlers = other.instance_variable_get("@_firebase_handlers")
              other_handlers.off(handler, true)
            end
          end
        end
      end
      nil
    end
    def dealloc
      off
      super
    end
  end

  def ensure_firebase_handlers(object)
    unless _firebase_handlers = object.instance_variable_get("@_firebase_handlers")
      _firebase_handlers = object.instance_variable_set("@_firebase_handlers", Holder.new)
    end
    _firebase_handlers
  end

  def own_snapshot(object, snap)
    _firebase_handlers = ensure_firebase_handlers(object)
    _firebase_handlers.own_snapshot(snap)
    nil
  end

  def track_handler(object, handler, leaf)
    _firebase_handlers = ensure_firebase_handlers(object)
    _firebase_handlers.track(self, handler, leaf)
    nil
  end

  Dispatch.once do
    $firebase_masters_by_url = NSMapTable.strongToWeakObjectsMapTable
    $firebase_masters_by_fquery = NSMapTable.weakToWeakObjectsMapTable
  end

  def print_masters
    puts "$firebase_masters_by_url:"
    for k in $firebase_masters_by_url.keyEnumerator
      puts "  - #{k}: #{$firebase_masters_by_url.objectForKey(k).rmext_object_desc}"
    end
    puts "$firebase_masters_by_fquery:"
    for k in $firebase_masters_by_fquery.keyEnumerator
      puts "  - #{k.rmext_object_desc}: #{$firebase_masters_by_fquery.objectForKey(k).rmext_object_desc}"
    end
    nil
  end

  def master
    if m = $firebase_masters_by_url.objectForKey(description)
      m
    else
      $firebase_masters_by_url.setObject(self, forKey:description)
      self
    end
  end

  def on(event_type, options={}, &and_then)
    master._on(event_type, options, &and_then)
  end

  def _on(_event_type, options={}, &and_then)
    and_then = (and_then || options[:completion]).weak!
    raise "event handler is required" unless and_then
    raise "event handler must accept one or two arguments" unless and_then.arity == 1 || and_then.arity == 2

    owner = and_then.owner
    weak_owner = WeakRef.new(owner)
    event_type = EVENT_TYPES_MAP[_event_type]
    raise "event handler is unknown: #{_event_type.inspect}" unless event_type
    # if event_type == FEventTypeValue
    #   if initial_snap = $firebase_masters_by_fquery.objectForKey(self)
    #     # p "HIT!", description
    #     rmext_inline_or_on_main_q do
    #       and_then.call(initial_snap)
    #     end
    #   else
    #     # p "MISS!", description
    #   end
    # end

    disconnect_block = options[:disconnect]
    raise ":disconnect handler must not accept any arguments" if disconnect_block && disconnect_block.arity > 0

    handler = if and_then.arity == 1
      wrapped_block = lambda do |snap|
        if weak_owner.weakref_alive?
          datasnap = FirebaseExt::DataSnapshot.new(snap)
          $firebase_masters_by_fquery.setObject(datasnap, forKey:self)
          own_snapshot(weak_owner, datasnap)
          and_then.call(datasnap)
        end
      end.weak!
      if disconnect_block
        disconnect_block.weak!
        if options[:once]
          observeSingleEventOfType(event_type, withBlock:wrapped_block, withCancelBlock:disconnect_block)
        else
          observeEventType(event_type, withBlock:wrapped_block, withCancelBlock:disconnect_block)
        end
      else
        if options[:once]
          observeSingleEventOfType(event_type, withBlock:wrapped_block)
        else
          observeEventType(event_type, withBlock:wrapped_block)
        end
      end
    else
      wrapped_block = lambda do |snap, prev|
        if weak_owner.weakref_alive?
          datasnap = FirebaseExt::DataSnapshot.new(snap)
          $firebase_masters_by_fquery.setObject(datasnap, forKey:self)
          and_then.call(datasnap, prev)
        end
      end.weak!
      if disconnect_block
        disconnect_block.weak!
        if options[:once]
          observeSingleEventOfType(event_type, andPreviousSiblingNameWithBlock:wrapped_block, withCancelBlock:disconnect_block)
        else
          observeEventType(event_type, andPreviousSiblingNameWithBlock:wrapped_block, withCancelBlock:disconnect_block)
        end
      else
        if options[:once]
          observeSingleEventOfType(event_type, andPreviousSiblingNameWithBlock:wrapped_block)
        else
          observeEventType(event_type, andPreviousSiblingNameWithBlock:wrapped_block)
        end
      end
    end
    unless options[:once]
      track_handler(owner, handler, self)
      track_handler(self, handler, owner)
      handler
    end
  end

  def once(event_type, options={}, &and_then)
    on(event_type, options.merge(:once => true), &and_then)
  end

  def off(handle=nil)
    master._off(handle)
  end

  def _off(handle=nil)
    if @_firebase_handlers
      @_firebase_handlers.off(handle)
    else
      @_firebase_handlers.off
    end
    return self
  end

  def self.off_context(owner, handle=nil)
    if _firebase_handlers = owner.instance_variable_get("@_firebase_handlers")
      _firebase_handlers.off(handle)
    end
  end

  def description
    AFHTTPRequestSerializer.serializer.requestWithMethod("GET", URLString: "https://#{repo.repoInfo.host}#{path.toString}", parameters:queryParams.queryObject).URL.absoluteString
  end

end

module FirebaseExt

  class DataSnapshot

    include RMExtensions::CommonMethods

    attr_accessor :value, :ref, :name

    def dealloc
      dealloc_inspect
      super
    end

    def initialize(snap=nil)
      if snap
        @snap = snap
        @ref = snap.ref
        @name = snap.name
        @value = snap.value
      end
    end

    def hasValue?
      !value.nil?
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      value && value[key]
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Listener

    include RMExtensions::CommonMethods

    attr_accessor :snapshot, :ref, :callback, :handle

    def dealloc
      dealloc_inspect
      stop!
      super
    end

    def ready?
      !!@ready
    end

    def start!
      cancel_block = lambda do
        p "cancel block"
        raise "unhandled cancel"
      end
      @handle = ref.on(:value, { :disconnect => cancel_block }) do |snap|
        @snapshot = snap
        @callback.call(snap) if @callback
        @ready = true
        rmext_trigger2(:ready, self)
        # p "ready__"
      end
    end

    def stop!
      @ready = false
      if ref && handle
        ref.off(handle)
      end
    end

    def hasValue?
      !!(snapshot && snapshot.hasValue?)
    end

    def toValue
      snapshot && snapshot.value
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      snapshot && snapshot.valueForKey(key)
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Coordinator

    include RMExtensions::CommonMethods

    attr_accessor :watches

    def dealloc
      dealloc_inspect
      super
    end

    def clear!
      @ready = false
      keys = @watches.keys.dup
      while name = keys.pop
        watch = @watches[name]
        watch.stop!
      end
      @watches.clear
      self
    end

    def initialize
      @watches = {}
    end

    def ready?
      !!@ready
    end

    def ready!
      @ready = true
      rmext_trigger2(:ready, self)
    end

    def stub(name)
      @watches[name] ||= Listener.new
    end

    def watch_data(name, data)
      if current = @watches[name]
        if current == data || current.description == data.description
          return
        end
        current.stop!
        current.rmext_off(:ready, self)
      end
      data.rmext_on(:ready) do
        if @watches.values.all?(&:ready?)
          ready!
          # p "ready__"
        end
      end
      @watches[name] = data
    end

    def watch(name, ref, &block)
      data = Listener.new
      data.ref = ref
      data.callback = block && RMExtensions::WeakBlock.new(block)
      watch_data(name, data)
      data.start!
      data
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      @watches[key]
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Model
    
    include RMExtensions::CommonMethods

    def dealloc
      dealloc_inspect
      super
    end

    attr_accessor :opts

    attr_reader :api, :root

    def initialize(opts=nil)
      @opts = opts || {}
      @dependencies = {}
      internal_setup
    end

    def ready?
      !!@ready
    end

    def ready!
      @ready = true
      rmext_trigger2(:ready, self)
      @waiting_once = nil
    end

    # override
    def setup
    end

    def internal_setup
      rmext_cleanup
      @api = Coordinator.new
      @api.rmext_on(:ready) do
        check_ready
      end
      setup
    end

    def check_ready
      if @api.ready? && @dependencies.values.all?(&:ready?)
        ready!
      end
    end

    def reload
      @ready = false
      old_watches = @api.watches # keep them in memory momentarily
      internal_setup
      old_watches = nil # let them go
      self
    end

    def watch(name, ref, &block)
      rmext_ivar(name, @api.watch(name, ref, &block))
    end

    # add another model as a dependency and ivar,
    # which will affect ready.  should be done before ready has
    # has a chance to be triggered.  the user can add an
    # attr_reader for easy access if desired.
    #
    # Book
    #   attr_reader :author
    #   model.watch(:root, ref) do |data|
    #     model.depend(:author, Author.get(data[:author_id]))
    #   end
    # ...
    # book = Book.get(1)
    # book.attr("name") #=> "My Book"
    # book.author.attr("name") #=> "<NAME>"
    def depend(name, model)
      @dependencies[name] = model
      rmext_ivar(name, model)
      model.rmext_on(:ready) do
        check_ready
      end
    end

    def always(&block)
      block.weak!
      rmext_on(:ready, &block)
      if ready?
        rmext_block_on_main_q(block, self)
      end
      # return an unbinder
      weak_self = WeakRef.new(self)
      lambda do
        if weak_self.weakref_alive?
          rmext_off(:ready, &block)
        end
      end
    end

    def once(&block)
      @waiting_once = self
      if ready?
        rmext_block_on_main_q(block, self)
      else
        rmext_once(:ready, &block)
      end
      self
    end

    def cancel_block(&block)
      rmext_off(:ready, &block)
    end

    def attr(keypath)
      root.attr(keypath)
    end

    def hasValue?
      root.hasValue?
    end

    def self.create(&block)
      x = new
      block.call(x)
      x
    end

    # this is the method you should call
    def self.get(opts=nil)
      if opts && existing = get_in_memory(opts)
        # p "HIT!", opts<commit_msg>add Model#toValue<commit_after>class Firebase
  def [](*names)
    if names.length == 0
      childByAutoId
    else
      childByAppendingPath(names.join('/'))
    end
  end
end

class FQuery

  EVENT_TYPES_MAP = {
    :child_added => FEventTypeChildAdded,
    :added => FEventTypeChildAdded,
    :child_moved => FEventTypeChildMoved,
    :moved => FEventTypeChildMoved,
    :child_changed => FEventTypeChildChanged,
    :changed => FEventTypeChildChanged,
    :child_removed => FEventTypeChildRemoved,
    :removed => FEventTypeChildRemoved,
    :value => FEventTypeValue
  }

  class Holder
    
    attr_accessor :map
    
    def initialize
      @map = NSMapTable.weakToStrongObjectsMapTable
      @own = NSMapTable.weakToStrongObjectsMapTable
    end

    def own_snapshot(snap)
      ref = snap.ref
      unless owned = @own.objectForKey(ref)
        owned = []
        @own.setObject(owned, forKey:ref)
      end
      owned.push snap
    end

    def track(fquery, handler, leaf)
      unless query_handles = @map.objectForKey(leaf)
        query_handles = {}
        @map.setObject(query_handles, forKey:leaf)
      end
      query_handles[handler] = fquery
    end

    def off(handle=nil, cleanup_only=false)
      object_context = !handle.is_a?(Integer) && handle
      handle = nil if object_context
      keyEnum = @map.keyEnumerator
      keys = []
      while key = keyEnum.nextObject
        keys.push key
      end
      while other = keys.pop
        next if object_context && other != object_context
        if hash = @map.objectForKey(other)
          hash_keys = [] + hash.keys
          while handler = hash_keys.pop
            if !handle || handler == handle
              fquery = hash[handler]
              @own.removeObjectForKey(fquery)
              hash.delete(handler)
              unless cleanup_only
                # p fquery, "removeObserverWithHandle", handler, caller
                fquery.removeObserverWithHandle(handler)
              end
              other_handlers = other.instance_variable_get("@_firebase_handlers")
              other_handlers.off(handler, true)
            end
          end
        end
      end
      nil
    end
    def dealloc
      off
      super
    end
  end

  def ensure_firebase_handlers(object)
    unless _firebase_handlers = object.instance_variable_get("@_firebase_handlers")
      _firebase_handlers = object.instance_variable_set("@_firebase_handlers", Holder.new)
    end
    _firebase_handlers
  end

  def own_snapshot(object, snap)
    _firebase_handlers = ensure_firebase_handlers(object)
    _firebase_handlers.own_snapshot(snap)
    nil
  end

  def track_handler(object, handler, leaf)
    _firebase_handlers = ensure_firebase_handlers(object)
    _firebase_handlers.track(self, handler, leaf)
    nil
  end

  Dispatch.once do
    $firebase_masters_by_url = NSMapTable.strongToWeakObjectsMapTable
    $firebase_masters_by_fquery = NSMapTable.weakToWeakObjectsMapTable
  end

  def print_masters
    puts "$firebase_masters_by_url:"
    for k in $firebase_masters_by_url.keyEnumerator
      puts "  - #{k}: #{$firebase_masters_by_url.objectForKey(k).rmext_object_desc}"
    end
    puts "$firebase_masters_by_fquery:"
    for k in $firebase_masters_by_fquery.keyEnumerator
      puts "  - #{k.rmext_object_desc}: #{$firebase_masters_by_fquery.objectForKey(k).rmext_object_desc}"
    end
    nil
  end

  def master
    if m = $firebase_masters_by_url.objectForKey(description)
      m
    else
      $firebase_masters_by_url.setObject(self, forKey:description)
      self
    end
  end

  def on(event_type, options={}, &and_then)
    master._on(event_type, options, &and_then)
  end

  def _on(_event_type, options={}, &and_then)
    and_then = (and_then || options[:completion]).weak!
    raise "event handler is required" unless and_then
    raise "event handler must accept one or two arguments" unless and_then.arity == 1 || and_then.arity == 2

    owner = and_then.owner
    weak_owner = WeakRef.new(owner)
    event_type = EVENT_TYPES_MAP[_event_type]
    raise "event handler is unknown: #{_event_type.inspect}" unless event_type
    # if event_type == FEventTypeValue
    #   if initial_snap = $firebase_masters_by_fquery.objectForKey(self)
    #     # p "HIT!", description
    #     rmext_inline_or_on_main_q do
    #       and_then.call(initial_snap)
    #     end
    #   else
    #     # p "MISS!", description
    #   end
    # end

    disconnect_block = options[:disconnect]
    raise ":disconnect handler must not accept any arguments" if disconnect_block && disconnect_block.arity > 0

    handler = if and_then.arity == 1
      wrapped_block = lambda do |snap|
        if weak_owner.weakref_alive?
          datasnap = FirebaseExt::DataSnapshot.new(snap)
          $firebase_masters_by_fquery.setObject(datasnap, forKey:self)
          own_snapshot(weak_owner, datasnap)
          and_then.call(datasnap)
        end
      end.weak!
      if disconnect_block
        disconnect_block.weak!
        if options[:once]
          observeSingleEventOfType(event_type, withBlock:wrapped_block, withCancelBlock:disconnect_block)
        else
          observeEventType(event_type, withBlock:wrapped_block, withCancelBlock:disconnect_block)
        end
      else
        if options[:once]
          observeSingleEventOfType(event_type, withBlock:wrapped_block)
        else
          observeEventType(event_type, withBlock:wrapped_block)
        end
      end
    else
      wrapped_block = lambda do |snap, prev|
        if weak_owner.weakref_alive?
          datasnap = FirebaseExt::DataSnapshot.new(snap)
          $firebase_masters_by_fquery.setObject(datasnap, forKey:self)
          and_then.call(datasnap, prev)
        end
      end.weak!
      if disconnect_block
        disconnect_block.weak!
        if options[:once]
          observeSingleEventOfType(event_type, andPreviousSiblingNameWithBlock:wrapped_block, withCancelBlock:disconnect_block)
        else
          observeEventType(event_type, andPreviousSiblingNameWithBlock:wrapped_block, withCancelBlock:disconnect_block)
        end
      else
        if options[:once]
          observeSingleEventOfType(event_type, andPreviousSiblingNameWithBlock:wrapped_block)
        else
          observeEventType(event_type, andPreviousSiblingNameWithBlock:wrapped_block)
        end
      end
    end
    unless options[:once]
      track_handler(owner, handler, self)
      track_handler(self, handler, owner)
      handler
    end
  end

  def once(event_type, options={}, &and_then)
    on(event_type, options.merge(:once => true), &and_then)
  end

  def off(handle=nil)
    master._off(handle)
  end

  def _off(handle=nil)
    if @_firebase_handlers
      @_firebase_handlers.off(handle)
    else
      @_firebase_handlers.off
    end
    return self
  end

  def self.off_context(owner, handle=nil)
    if _firebase_handlers = owner.instance_variable_get("@_firebase_handlers")
      _firebase_handlers.off(handle)
    end
  end

  def description
    AFHTTPRequestSerializer.serializer.requestWithMethod("GET", URLString: "https://#{repo.repoInfo.host}#{path.toString}", parameters:queryParams.queryObject).URL.absoluteString
  end

end

module FirebaseExt

  class DataSnapshot

    include RMExtensions::CommonMethods

    attr_accessor :value, :ref, :name

    def dealloc
      dealloc_inspect
      super
    end

    def initialize(snap=nil)
      if snap
        @snap = snap
        @ref = snap.ref
        @name = snap.name
        @value = snap.value
      end
    end

    def hasValue?
      !value.nil?
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      value && value[key]
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Listener

    include RMExtensions::CommonMethods

    attr_accessor :snapshot, :ref, :callback, :handle

    def dealloc
      dealloc_inspect
      stop!
      super
    end

    def ready?
      !!@ready
    end

    def start!
      cancel_block = lambda do
        p "cancel block"
        raise "unhandled cancel"
      end
      @handle = ref.on(:value, { :disconnect => cancel_block }) do |snap|
        @snapshot = snap
        @callback.call(snap) if @callback
        @ready = true
        rmext_trigger2(:ready, self)
        # p "ready__"
      end
    end

    def stop!
      @ready = false
      if ref && handle
        ref.off(handle)
      end
    end

    def hasValue?
      !!(snapshot && snapshot.hasValue?)
    end

    def toValue
      snapshot && snapshot.value
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      snapshot && snapshot.valueForKey(key)
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Coordinator

    include RMExtensions::CommonMethods

    attr_accessor :watches

    def dealloc
      dealloc_inspect
      super
    end

    def clear!
      @ready = false
      keys = @watches.keys.dup
      while name = keys.pop
        watch = @watches[name]
        watch.stop!
      end
      @watches.clear
      self
    end

    def initialize
      @watches = {}
    end

    def ready?
      !!@ready
    end

    def ready!
      @ready = true
      rmext_trigger2(:ready, self)
    end

    def stub(name)
      @watches[name] ||= Listener.new
    end

    def watch_data(name, data)
      if current = @watches[name]
        if current == data || current.description == data.description
          return
        end
        current.stop!
        current.rmext_off(:ready, self)
      end
      data.rmext_on(:ready) do
        if @watches.values.all?(&:ready?)
          ready!
          # p "ready__"
        end
      end
      @watches[name] = data
    end

    def watch(name, ref, &block)
      data = Listener.new
      data.ref = ref
      data.callback = block && RMExtensions::WeakBlock.new(block)
      watch_data(name, data)
      data.start!
      data
    end

    def attr(keypath)
      valueForKeyPath(keypath)
    end

    def valueForKey(key)
      @watches[key]
    end

    def valueForUndefinedKey(key)
      nil
    end

  end

  class Model
    
    include RMExtensions::CommonMethods

    def dealloc
      dealloc_inspect
      super
    end

    attr_accessor :opts

    attr_reader :api, :root

    def initialize(opts=nil)
      @opts = opts || {}
      @dependencies = {}
      internal_setup
    end

    def ready?
      !!@ready
    end

    def ready!
      @ready = true
      rmext_trigger2(:ready, self)
      @waiting_once = nil
    end

    # override
    def setup
    end

    def internal_setup
      rmext_cleanup
      @api = Coordinator.new
      @api.rmext_on(:ready) do
        check_ready
      end
      setup
    end

    def check_ready
      if @api.ready? && @dependencies.values.all?(&:ready?)
        ready!
      end
    end

    def reload
      @ready = false
      old_watches = @api.watches # keep them in memory momentarily
      internal_setup
      old_watches = nil # let them go
      self
    end

    def watch(name, ref, &block)
      rmext_ivar(name, @api.watch(name, ref, &block))
    end

    # add another model as a dependency and ivar,
    # which will affect ready.  should be done before ready has
    # has a chance to be triggered.  the user can add an
    # attr_reader for easy access if desired.
    #
    # Book
    #   attr_reader :author
    #   model.watch(:root, ref) do |data|
    #     model.depend(:author, Author.get(data[:author_id]))
    #   end
    # ...
    # book = Book.get(1)
    # book.attr("name") #=> "My Book"
    # book.author.attr("name") #=> "<NAME>"
    def depend(name, model)
      @dependencies[name] = model
      rmext_ivar(name, model)
      model.rmext_on(:ready) do
        check_ready
      end
    end

    def always(&block)
      block.weak!
      rmext_on(:ready, &block)
      if ready?
        rmext_block_on_main_q(block, self)
      end
      # return an unbinder
      weak_self = WeakRef.new(self)
      lambda do
        if weak_self.weakref_alive?
          rmext_off(:ready, &block)
        end
      end
    end

    def once(&block)
      @waiting_once = self
      if ready?
        rmext_block_on_main_q(block, self)
      else
        rmext_once(:ready, &block)
      end
      self
    end

    def cancel_block(&block)
      rmext_off(:ready, &block)
    end

    def attr(keypath)
      root.attr(keypath)
    end

    def hasValue?
      root.hasValue?
    end

    def toValue
      root.toValue
    end

    def self.create(&block)
      x = new
      block.call(x)
      x
    end

    # this is the method you should call
    def self.get(opts=nil)
      if opts && existing = get_in_memory(opts)
        # p "HIT!", opts
{'fraction_non_alphanumeric': 0.08302626421011368, 'fraction_numerical': 0.0007056056448451588, 'mean_word_length': 2.326943140323422, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2070189', 'n_tokens_mistral': 8298, 'n_tokens_neox': 8050, 'n_words': 1784}