<commit_before>require 'spec_helper'

module Ransack
  module Adapters
    module ActiveRecord
      describe Base do

        subject { ::ActiveRecord::Base }

        it { should respond_to :ransack }
        it { should respond_to :search }

        describe '#search' do
          subject { Person.ransack }

          it { should be_a Search }
          it 'has a Relation as its object' do
            expect(subject.object).to be_an ::ActiveRecord::Relation
          end

          context 'with scopes' do
            before do
              Person.stub :ransackable_scopes =>
                [:active, :over_age, :of_age, :article_count_equals]
            end

            it "applies true scopes" do
              search = Person.ransack('active' => true)
              search.result.to_sql.should include "active = 1"
            end

            it "applies stringy true scopes" do
              search = Person.ransack('active' => 'true')
              search.result.to_sql.should include "active = 1"
            end

            it "applies stringy boolean scopes with true value in an array" do
              search = Person.ransack('of_age' => ['true'])
              search.result.to_sql.should include "age >= 18"
            end

            it "applies stringy boolean scopes with false value in an array" do
              search = Person.ransack('of_age' => ['false'])
              search.result.to_sql.should include "age < 18"
            end

            it "ignores unlisted scopes" do
              search = Person.ransack('restricted' => true)
              search.result.to_sql.should_not include "restricted"
            end

            it "ignores false scopes" do
              search = Person.ransack('active' => false)
              search.result.to_sql.should_not include "active"
            end

            it "ignores stringy false scopes" do
              search = Person.ransack('active' => 'false')
              search.result.to_sql.should_not include "active"
            end

            it "passes values to scopes" do
              search = Person.ransack('over_age' => 18)
              search.result.to_sql.should include "age > 18"
            end

            it "chains scopes" do
              search = Person.ransack('over_age' => 18, 'active' => true)
              search.result.to_sql.should include "age > 18"
              search.result.to_sql.should include "active = 1"
            end

            unless AR_VERSION =~ /^3\.0\./

              context "applying joins/group/having scope" do
                it "applies scope correctly when input is 0" do
                  search = Person.ransack('article_count_equals' => 0)
                  search.result.to_sql.should include
                  "HAVING count(articles.id) = 0"
                end

                it "applies scope correctly when input is 1" do
                  search = Person.ransack('article_count_equals' => 1)
                  search.result.to_sql.should include
                  "HAVING count(articles.id) = 1"
                end

                it "applies scope correctly when input is 1337" do
                  search = Person.ransack('article_count_equals' => 1337)
                  search.result.to_sql.should include
                  "HAVING count(articles.id) = 1337"
                end
              end

            end
          end

          it 'does not raise exception for string :params argument' do
            lambda { Person.ransack('') }.should_not raise_error
          end

          it 'does not modify the parameters' do
            params = { :name_eq => '' }
            expect { Person.ransack(params) }.not_to change { params }
          end
        end

        describe '#ransacker' do
          # For infix tests
          def self.sane_adapter?
            case ::ActiveRecord::Base.connection.adapter_name
            when "SQLite3", "PostgreSQL"
              true
            else
              false
            end
          end
          # in schema.rb, class Person:
          # ransacker :reversed_name, formatter: proc { |v| v.reverse } do |parent|
          #   parent.table[:name]
          # end
          #
          # ransacker :doubled_name do |parent|
          #   Arel::Nodes::InfixOperation.new(
          #     '||', parent.table[:name], parent.table[:name]
          #   )
          # end

          it 'creates ransack attributes' do
            s = Person.ransack(:reversed_name_eq => 'htimS cirA')
            expect(s.result.size).to eq(1)

            expect(s.result.first).to eq Person.where(name: '<NAME>').first
          end

          it 'can be accessed through associations' do
            s = Person.ransack(:children_reversed_name_eq => 'htimS cirA')
            expect(s.result.to_sql).to match(
              /#{quote_table_name("children_people")}.#{
                 quote_column_name("name")} = '<NAME>'/
            )
          end

          it 'allows an "attribute" to be an InfixOperation' do
            s = Person.ransack(:doubled_name_eq => '<NAME>')
            expect(s.result.first).to eq Person.where(name: '<NAME>').first
          end if defined?(Arel::Nodes::InfixOperation) && sane_adapter?

          it "doesn't break #count if using InfixOperations" do
            s = Person.ransack(:doubled_name_eq => '<NAME>')
            expect(s.result.count).to eq 1
          end if defined?(Arel::Nodes::InfixOperation) && sane_adapter?

          it "should remove empty key value pairs from the params hash" do
            s = Person.ransack(:children_reversed_name_eq => '')
            expect(s.result.to_sql).not_to match /LEFT OUTER JOIN/
          end

          it "should keep proper key value pairs in the params hash" do
            s = Person.ransack(:children_reversed_name_eq => 'Testing')
            expect(s.result.to_sql).to match /LEFT OUTER JOIN/
          end

          it "should function correctly when nil is passed in" do
            s = Person.ransack(nil)
          end

          it "should function correctly when a blank string is passed in" do
            s = Person.ransack('')
          end

          it "should function correctly with a multi-parameter attribute" do
            date = Date.current
            s = Person.ransack(
              { "created_at_gteq(1i)" => date.year,
                "created_at_gteq(2i)" => date.month,
                "created_at_gteq(3i)" => date.day
              }
            )
            expect(s.result.to_sql).to match />=/
            expect(s.result.to_sql).to match date.to_s
          end

          it "should function correctly when using fields with dots in them" do
            s = Person.ransack(:email_cont => "example.com")
            expect(s.result.exists?).to be true
          end

          it "should function correctly when using fields with % in them" do
            p = Person.create!(:name => "110%-er")
            s = Person.ransack(:name_cont => "10%")
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when using fields with backslashes in them" do
            p = Person.create!(:name => "\\WINNER\\")
            s = Person.ransack(:name_cont => "\\WINNER\\")
            expect(s.result.to_a).to eq [p]
          end

          context "searching on an `in` predicate with a ransacker" do
            it "should function correctly when passing an array of ids" do
              s = Person.ransack(array_users_in: true)
              expect(s.result.count).to be > 0
            end

            it "should function correctly when passing an array of strings" do
              p = Person.create!(name: Person.first.id.to_s)
              s = Person.ransack(array_names_in: true)
              expect(s.result.count).to be > 0
            end

            it 'should function correctly with an Arel SqlLiteral' do
              s = Person.search(sql_literal_id_in: 1)
              expect(s.result.count).to be 1
              s = Person.search(sql_literal_id_in: ['2', 4, '5', 8])
              expect(s.result.count).to be 4
            end
          end

          context "search on an `in` predicate with an array" do
            it "should function correctly when passing an array of ids" do
              array = Person.all.map(&:id)
              s = Person.ransack(id_in: array)
              expect(s.result.count).to eq array.size
            end
          end

          it "should function correctly when an attribute name ends with '_start'" do
            p = Person.create!(:new_start => 'Bar and foo', :name => 'Xiang')

            s = Person.ransack(:new_start_end => ' and foo')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:name_or_new_start_start => 'Xia')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:new_start_or_name_end => 'iang')
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when an attribute name ends with '_end'" do
            p = Person.create!(:stop_end => 'Foo and bar', :name => 'Marianne')

            s = Person.ransack(:stop_end_start => 'Foo and')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:stop_end_or_name_end => 'anne')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:name_or_stop_end_end => ' bar')
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when an attribute name has 'and' in it" do
          # FIXME: this test does not pass!
            p = Person.create!(:terms_and_conditions => true)
            s = Person.ransack(:terms_and_conditions_eq => true)
          # search is not detecting the attribute
            puts "
            FIXME: Search not detecting the `terms_and_conditions` attribute in
            base_spec.rb, line 178: #{s.result.to_sql}"
          # expect(s.result.to_a).to eq [p]
          end

          it 'allows sort by "only_sort" field' do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_sort" } }
            )
            expect(s.result.to_sql).to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_sort")} ASC/
            )
          end

          it "doesn't sort by 'only_search' field" do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_search" } }
            )
            expect(s.result.to_sql).not_to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_search")} ASC/
            )
          end

          it 'allows search by "only_search" field' do
            s = Person.ransack(:only_search_eq => 'htimS cirA')
            expect(s.result.to_sql).to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_search")} = 'htimS cirA'/
            )
          end

          it "can't be searched by 'only_sort'" do
            s = Person.ransack(:only_sort_eq => 'htimS cirA')
            expect(s.result.to_sql).not_to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_sort")} = 'htimS cirA'/
            )
          end

          it 'allows sort by "only_admin" field, if auth_object: :admin' do
            s = Person.ransack(
              { "s" => { "0" => { "dir" => "asc", "name" => "only_admin" } } },
              { auth_object: :admin }
            )
            expect(s.result.to_sql).to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} ASC/
            )
          end

          it "doesn't sort by 'only_admin' field, if auth_object: nil" do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_admin" } }
            )
            expect(s.result.to_sql).not_to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} ASC/
            )
          end

          it 'allows search by "only_admin" field, if auth_object: :admin' do
            s = Person.ransack(
              { :only_admin_eq => 'htimS cirA' },
              { :auth_object => :admin }
            )
            expect(s.result.to_sql).to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} = 'htimS cirA'/
            )
          end

          it "can't be searched by 'only_admin', if auth_object: nil" do
            s = Person.ransack(:only_admin_eq => 'htimS cirA')
            expect(s.result.to_sql).not_to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} = 'htimS cirA'/
            )
          end
        end

        describe '#ransackable_attributes' do
          context 'when auth_object is nil' do
            subject { Person.ransackable_attributes }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_search' }
            it { should_not include 'only_sort' }
            it { should_not include 'only_admin' }
          end

          context 'with auth_object :admin' do
            subject { Person.ransackable_attributes(:admin) }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_search' }
            it { should_not include 'only_sort' }
            it { should include 'only_admin' }
          end
        end

        describe '#ransortable_attributes' do
          context 'when auth_object is nil' do
            subject { Person.ransortable_attributes }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_sort' }
            it { should_not include 'only_search' }
            it { should_not include 'only_admin' }
          end

          context 'with auth_object :admin' do
            subject { Person.ransortable_attributes(:admin) }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_sort' }
            it { should_not include 'only_search' }
            it { should include 'only_admin' }
          end
        end

        describe '#ransackable_associations' do
          subject { Person.ransackable_associations }

          it { should include 'parent' }
          it { should include 'children' }
          it { should include 'articles' }
        end

        describe '#ransackable_scopes' do
          subject { Person.ransackable_scopes }

          it { should eq [] }
        end

      end
    end
  end
end
<commit_msg>base_spec: use the same local var & search methods<commit_after>require 'spec_helper'

module Ransack
  module Adapters
    module ActiveRecord
      describe Base do

        subject { ::ActiveRecord::Base }

        it { should respond_to :ransack }
        it { should respond_to :search }

        describe '#search' do
          subject { Person.ransack }

          it { should be_a Search }
          it 'has a Relation as its object' do
            expect(subject.object).to be_an ::ActiveRecord::Relation
          end

          context 'with scopes' do
            before do
              Person.stub :ransackable_scopes =>
                [:active, :over_age, :of_age, :article_count_equals]
            end

            it "applies true scopes" do
              s = Person.ransack('active' => true)
              s.result.to_sql.should include "active = 1"
            end

            it "applies stringy true scopes" do
              s = Person.ransack('active' => 'true')
              s.result.to_sql.should include "active = 1"
            end

            it "applies stringy boolean scopes with true value in an array" do
              s = Person.ransack('of_age' => ['true'])
              s.result.to_sql.should include "age >= 18"
            end

            it "applies stringy boolean scopes with false value in an array" do
              s = Person.ransack('of_age' => ['false'])
              s.result.to_sql.should include "age < 18"
            end

            it "ignores unlisted scopes" do
              s = Person.ransack('restricted' => true)
              s.result.to_sql.should_not include "restricted"
            end

            it "ignores false scopes" do
              s = Person.ransack('active' => false)
              s.result.to_sql.should_not include "active"
            end

            it "ignores stringy false scopes" do
              s = Person.ransack('active' => 'false')
              s.result.to_sql.should_not include "active"
            end

            it "passes values to scopes" do
              s = Person.ransack('over_age' => 18)
              s.result.to_sql.should include "age > 18"
            end

            it "chains scopes" do
              s = Person.ransack('over_age' => 18, 'active' => true)
              s.result.to_sql.should include "age > 18"
              s.result.to_sql.should include "active = 1"
            end

            unless AR_VERSION =~ /^3\.0\./
              context "applying joins/group/having scope" do
                it "applies scope correctly when input is 0" do
                  s = Person.ransack('article_count_equals' => 0)
                  s.result.to_sql.should include "HAVING count(articles.id) = 0"
                end

                it "applies scope correctly when input is 1" do
                  s = Person.ransack('article_count_equals' => 1)
                  s.result.to_sql.should include "HAVING count(articles.id) = 1"
                end

                it "applies scope correctly when input is 1337" do
                  s = Person.ransack('article_count_equals' => 1337)
                  s.result.to_sql.should include "HAVING count(articles.id) = 1337"
                end
              end
            end
          end

          it 'does not raise exception for string :params argument' do
            lambda { Person.ransack('') }.should_not raise_error
          end

          it 'does not modify the parameters' do
            params = { :name_eq => '' }
            expect { Person.ransack(params) }.not_to change { params }
          end
        end

        describe '#ransacker' do
          # For infix tests
          def self.sane_adapter?
            case ::ActiveRecord::Base.connection.adapter_name
            when "SQLite3", "PostgreSQL"
              true
            else
              false
            end
          end
          # in schema.rb, class Person:
          # ransacker :reversed_name, formatter: proc { |v| v.reverse } do |parent|
          #   parent.table[:name]
          # end
          #
          # ransacker :doubled_name do |parent|
          #   Arel::Nodes::InfixOperation.new(
          #     '||', parent.table[:name], parent.table[:name]
          #   )
          # end

          it 'creates ransack attributes' do
            s = Person.ransack(:reversed_name_eq => 'htimS cirA')
            expect(s.result.size).to eq(1)

            expect(s.result.first).to eq Person.where(name: '<NAME>').first
          end

          it 'can be accessed through associations' do
            s = Person.ransack(:children_reversed_name_eq => 'htimS cirA')
            expect(s.result.to_sql).to match(
              /#{quote_table_name("children_people")}.#{
                 quote_column_name("name")} = '<NAME>'/
            )
          end

          it 'allows an "attribute" to be an InfixOperation' do
            s = Person.ransack(:doubled_name_eq => '<NAME> <NAME>')
            expect(s.result.first).to eq Person.where(name: '<NAME>').first
          end if defined?(Arel::Nodes::InfixOperation) && sane_adapter?

          it "doesn't break #count if using InfixOperations" do
            s = Person.ransack(:doubled_name_eq => '<NAME> <NAME>')
            expect(s.result.count).to eq 1
          end if defined?(Arel::Nodes::InfixOperation) && sane_adapter?

          it "should remove empty key value pairs from the params hash" do
            s = Person.ransack(:children_reversed_name_eq => '')
            expect(s.result.to_sql).not_to match /LEFT OUTER JOIN/
          end

          it "should keep proper key value pairs in the params hash" do
            s = Person.ransack(:children_reversed_name_eq => 'Testing')
            expect(s.result.to_sql).to match /LEFT OUTER JOIN/
          end

          it "should function correctly when nil is passed in" do
            s = Person.ransack(nil)
          end

          it "should function correctly when a blank string is passed in" do
            s = Person.ransack('')
          end

          it "should function correctly with a multi-parameter attribute" do
            date = Date.current
            s = Person.ransack(
              { "created_at_gteq(1i)" => date.year,
                "created_at_gteq(2i)" => date.month,
                "created_at_gteq(3i)" => date.day
              }
            )
            expect(s.result.to_sql).to match />=/
            expect(s.result.to_sql).to match date.to_s
          end

          it "should function correctly when using fields with dots in them" do
            s = Person.ransack(:email_cont => "example.com")
            expect(s.result.exists?).to be true
          end

          it "should function correctly when using fields with % in them" do
            p = Person.create!(:name => "110%-er")
            s = Person.ransack(:name_cont => "10%")
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when using fields with backslashes in them" do
            p = Person.create!(:name => "\\WINNER\\")
            s = Person.ransack(:name_cont => "\\WINNER\\")
            expect(s.result.to_a).to eq [p]
          end

          context "searching on an `in` predicate with a ransacker" do
            it "should function correctly when passing an array of ids" do
              s = Person.ransack(array_users_in: true)
              expect(s.result.count).to be > 0
            end

            it "should function correctly when passing an array of strings" do
              p = Person.create!(name: Person.first.id.to_s)
              s = Person.ransack(array_names_in: true)
              expect(s.result.count).to be > 0
            end

            it 'should function correctly with an Arel SqlLiteral' do
              s = Person.ransack(sql_literal_id_in: 1)
              expect(s.result.count).to be 1
              s = Person.ransack(sql_literal_id_in: ['2', 4, '5', 8])
              expect(s.result.count).to be 4
            end
          end

          context "search on an `in` predicate with an array" do
            it "should function correctly when passing an array of ids" do
              array = Person.all.map(&:id)
              s = Person.ransack(id_in: array)
              expect(s.result.count).to eq array.size
            end
          end

          it "should function correctly when an attribute name ends with '_start'" do
            p = Person.create!(:new_start => 'Bar and foo', :name => 'Xiang')

            s = Person.ransack(:new_start_end => ' and foo')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:name_or_new_start_start => 'Xia')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:new_start_or_name_end => 'iang')
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when an attribute name ends with '_end'" do
            p = Person.create!(:stop_end => 'Foo and bar', :name => 'Marianne')

            s = Person.ransack(:stop_end_start => 'Foo and')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:stop_end_or_name_end => 'anne')
            expect(s.result.to_a).to eq [p]

            s = Person.ransack(:name_or_stop_end_end => ' bar')
            expect(s.result.to_a).to eq [p]
          end

          it "should function correctly when an attribute name has 'and' in it" do
          # FIXME: this test does not pass!
            p = Person.create!(:terms_and_conditions => true)
            s = Person.ransack(:terms_and_conditions_eq => true)
          # search is not detecting the attribute
            puts "
            FIXME: Search not detecting the `terms_and_conditions` attribute in
            base_spec.rb, line 178: #{s.result.to_sql}"
          # expect(s.result.to_a).to eq [p]
          end

          it 'allows sort by "only_sort" field' do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_sort" } }
            )
            expect(s.result.to_sql).to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_sort")} ASC/
            )
          end

          it "doesn't sort by 'only_search' field" do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_search" } }
            )
            expect(s.result.to_sql).not_to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_search")} ASC/
            )
          end

          it 'allows search by "only_search" field' do
            s = Person.ransack(:only_search_eq => 'htimS cirA')
            expect(s.result.to_sql).to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_search")} = 'htimS cirA'/
            )
          end

          it "can't be searched by 'only_sort'" do
            s = Person.ransack(:only_sort_eq => 'htimS cirA')
            expect(s.result.to_sql).not_to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_sort")} = 'htimS cirA'/
            )
          end

          it 'allows sort by "only_admin" field, if auth_object: :admin' do
            s = Person.ransack(
              { "s" => { "0" => { "dir" => "asc", "name" => "only_admin" } } },
              { auth_object: :admin }
            )
            expect(s.result.to_sql).to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} ASC/
            )
          end

          it "doesn't sort by 'only_admin' field, if auth_object: nil" do
            s = Person.ransack(
              "s" => { "0" => { "dir" => "asc", "name" => "only_admin" } }
            )
            expect(s.result.to_sql).not_to match(
              /ORDER BY #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} ASC/
            )
          end

          it 'allows search by "only_admin" field, if auth_object: :admin' do
            s = Person.ransack(
              { :only_admin_eq => 'htimS cirA' },
              { :auth_object => :admin }
            )
            expect(s.result.to_sql).to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} = 'htimS cirA'/
            )
          end

          it "can't be searched by 'only_admin', if auth_object: nil" do
            s = Person.ransack(:only_admin_eq => 'htimS cirA')
            expect(s.result.to_sql).not_to match(
              /WHERE #{quote_table_name("people")}.#{
                quote_column_name("only_admin")} = 'htimS cirA'/
            )
          end
        end

        describe '#ransackable_attributes' do
          context 'when auth_object is nil' do
            subject { Person.ransackable_attributes }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_search' }
            it { should_not include 'only_sort' }
            it { should_not include 'only_admin' }
          end

          context 'with auth_object :admin' do
            subject { Person.ransackable_attributes(:admin) }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_search' }
            it { should_not include 'only_sort' }
            it { should include 'only_admin' }
          end
        end

        describe '#ransortable_attributes' do
          context 'when auth_object is nil' do
            subject { Person.ransortable_attributes }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_sort' }
            it { should_not include 'only_search' }
            it { should_not include 'only_admin' }
          end

          context 'with auth_object :admin' do
            subject { Person.ransortable_attributes(:admin) }

            it { should include 'name' }
            it { should include 'reversed_name' }
            it { should include 'doubled_name' }
            it { should include 'only_sort' }
            it { should_not include 'only_search' }
            it { should include 'only_admin' }
          end
        end

        describe '#ransackable_associations' do
          subject { Person.ransackable_associations }

          it { should include 'parent' }
          it { should include 'children' }
          it { should include 'articles' }
        end

        describe '#ransackable_scopes' do
          subject { Person.ransackable_scopes }

          it { should eq [] }
        end

      end
    end
  end
end

{'fraction_non_alphanumeric': 0.10405279296093854, 'fraction_numerical': 0.0041327822956939075, 'mean_word_length': 1.6170955080680331, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 151, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5993630', 'n_tokens_mistral': 8847, 'n_tokens_neox': 8335, 'n_words': 2291}