<commit_before># encoding: utf-8
require 'spec_helper'

describe PassaporteWeb::Identity do

  describe "constants" do
    it { PassaporteWeb::Identity::ATTRIBUTES.should == [:accounts, :birth_date, :country, :cpf, :email, :first_name, :gender, :is_active, :language, :last_name, :nickname, :notifications, :send_myfreecomm_news, :send_partner_news, :services, :timezone, :update_info_url, :uuid, :password, :<PASSWORD>, :must_change_password, :inhibit_activation_message, :tos] }
    it { PassaporteWeb::Identity::UPDATABLE_ATTRIBUTES.should == [:first_name, :last_name, :nickname, :cpf, :birth_date, :gender, :send_myfreecomm_news, :send_partner_news, :country, :language, :timezone] }
    it { PassaporteWeb::Identity::CREATABLE_ATTRIBUTES.should == [:first_name, :last_name, :nickname, :cpf, :birth_date, :gender, :send_myfreecomm_news, :send_partner_news, :country, :language, :timezone, :email, :password, :<PASSWORD>, :must_change_password, :tos] }
  end

  describe ".new" do
    it "should instanciate an empty object" do
      identity = PassaporteWeb::Identity.new
      identity.attributes.should == {:accounts=>nil, :birth_date=>nil, :country=>nil, :cpf=>nil, :email=>nil, :first_name=>nil, :gender=>nil, :is_active=>nil, :language=>nil, :last_name=>nil, :nickname=>nil, :notifications=>nil, :send_myfreecomm_news=>nil, :send_partner_news=>nil, :services=>nil, :timezone=>nil, :update_info_url=>nil, :uuid=>nil, :password=><PASSWORD>, :password2=><PASSWORD>, :must_change_password=><PASSWORD>, :inhibit_activation_message=>nil, :tos=>nil}
    end
    it "should instanciate an object with attributes set" do
      attributes = {
        "last_name" => "<NAME>",
        "is_active" => true,
        "timezone" => "GMT-3",
        "nickname" => "Lula",
        "first_name" => "<NAME>",
        "send_partner_news" => false,
        "uuid" => "a5868d14-6529-477a-9c6b-a09dd42a7cd2",
        "language" => "pt_BR",
        "country" => "Brasil",
        "update_info_url" => "/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/",
        "send_myfreecomm_news" => false,
        "gender" => "M",
        "birth_date" => "1945-10-27",
        "email" => "<EMAIL>",
        "notifications" => {
          "count" => 0,
          "list" => "/notifications/api/"
        },
        "accounts" => [],
        "services" => {
          "myfinance" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/",
          "account_manager" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"
        },
        "password"=><PASSWORD>,
        "password2"=><PASSWORD>,
        "must_change_password"=>nil,
        "inhibit_activation_message"=>nil,
        "tos"=>nil
      }
      identity = PassaporteWeb::Identity.new(attributes)
      identity.attributes.should == {:accounts=>[], :birth_date=>"1945-10-27", :country=>"Brasil", :cpf=>nil, :email=>"<EMAIL>", :first_name=>"<NAME>", :gender=>"M", :is_active=>true, :language=>"pt_BR", :last_name=>"<NAME>", :nickname=>"Lula", :notifications=>{"count"=>0, "list"=>"/notifications/api/"}, :send_myfreecomm_news=>false, :send_partner_news=>false, :services=>{"myfinance"=>"/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/", "account_manager"=>"/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"}, :timezone=>"GMT-3", :update_info_url=>"/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/", :uuid=>"a5868d14-6529-477a-9c6b-a09dd42a7cd2", :password=><PASSWORD>, :password2=><PASSWORD>, :must_change_password=><PASSWORD>, :inhibit_activation_message=>nil, :tos=>nil}
      identity.last_name.should == "<NAME>"
      identity.is_active.should == true
      identity.timezone.should == "GMT-3"
      identity.nickname.should == "Lula"
      identity.first_name.should == "<NAME>"
      identity.send_partner_news.should == false
      identity.uuid.should == "a5868d14-6529-477a-9c6b-a09dd42a7cd2"
      identity.language.should == "pt_BR"
      identity.country.should == "Brasil"
      identity.update_info_url.should == "/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/"
      identity.send_myfreecomm_news.should == false
      identity.gender.should == "M"
      identity.birth_date.should == "1945-10-27"
      identity.email.should == "<EMAIL>"
      identity.notifications.should == {"count" => 0, "list" => "/notifications/api/"}
      identity.accounts.should == []
      identity.services.should == {"myfinance" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/", "account_manager" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"}
      identity.password.should be_nil
      identity.password2.should be_nil
      identity.must_change_password.should be_nil
      identity.inhibit_activation_message.should be_nil
      identity.tos.should be_nil
    end
  end

  describe "#== and #===" do
    it "should identify two profiles with the same uuid as equal" do
      p1 = PassaporteWeb::Identity.new('uuid' => 'some-uuid')
      p2 = PassaporteWeb::Identity.new('uuid' => 'some-uuid')
      p1.should == p2
      p1.should_not === p2
      p2.should == p1
      p2.should_not === p1
    end
    it "should identify two profiles with different uuids as different" do
      p1 = PassaporteWeb::Identity.new('uuid' => 'some-uuid-1')
      p2 = PassaporteWeb::Identity.new('uuid' => 'some-uuid-2')
      p1.should_not == p2
      p1.should_not === p2
      p2.should_not == p1
      p2.should_not === p1
      p3 = PassaporteWeb::Identity.new
      p1.should_not == p3
      p3.should_not == p1
      p3.should_not === p1
    end
  end

  describe ".find", :vcr => true do
    it "should find the requested profile by uuid" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56")
      identity.should be_instance_of(PassaporteWeb::Identity)
      identity.uuid.should == '5e32f927-c4ab-404e-a91c-b2abc05afb56'
      identity.email.should == '<EMAIL>'
      identity.update_info_url.should == '/accounts/api/identities/5e32f927-c4ab-404e-a91c-b2abc05afb56/'
      identity.accounts.size.should == 9
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil] # FIXME não vem com o único serviço ?
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by uuid, including expired accounts" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil] # FIXME não vem com o único serviço ?
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by uuid, including other services" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", false, true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
      # FIXME não está vindo com todos os serviços
      # identity.services.size.should > 1
      # identity.services.keys.sort.should == ['identity_client', 'vc-promove']
    end
    it "should find the requested profile by uuid, including other services and expired accounts" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", true, true)
      identity.accounts.size.should == 11
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
      # FIXME não está vindo com todos os serviços
      # identity.services.size.should > 1
      # identity.services.keys.sort.should == ['identity_client', 'vc-promove']
    end
    it "should raise an error if no profiles exist with that uuid" do
      expect {
        PassaporteWeb::Identity.find("invalid-uuid")
      }.to raise_error(RestClient::ResourceNotFound, '404 Resource Not Found')
    end
  end

  describe ".find_by_email", :vcr => true do
    it "should find the requested profile by email" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>")
      identity.should be_instance_of(PassaporteWeb::Identity)
      identity.uuid.should == '5e32f927-c4ab-404e-a91c-b2abc05afb56'
      identity.email.should == '<EMAIL>'
      identity.update_info_url.should == '/accounts/api/identities/5e32f927-c4ab-404e-a91c-b2abc05afb56/'
      identity.accounts.size.should == 9
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil] # FIXME não vem com o único serviço ?
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by email, including expired accounts" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil] # FIXME não vem com o único serviço ?
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by email, including other services" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", false, true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
      # FIXME não está vindo com todos os serviços
      # identity.services.size.should > 1
      # identity.services.keys.sort.should == ['identity_client', 'vc-promove']
    end
    it "should find the requested profile by email, including other services and expired accounts" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", true, true)
      identity.accounts.size.should == 11
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
      # FIXME não está vindo com todos os serviços
      # identity.services.size.should > 1
      # identity.services.keys.sort.should == ['identity_client', 'vc-promove']
    end
    it "should raise an error if no profiles exist with that email" do
      expect {
        PassaporteWeb::Identity.find("<EMAIL>")
      }.to raise_error(RestClient::ResourceNotFound, '404 Resource Not Found')
    end
  end

  describe "#save", :vcr => true do
    describe "PUT" do
      let(:identity) { PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56") }
      context "on success" do
        it "should update the profile attributes on the server" do
          identity.first_name.should == 'Testador'
          identity.first_name = '<NAME>'
          identity.save.should be_true
          identity.first_name.should == '<NAME>'<commit_msg>Removes comments<commit_after># encoding: utf-8
require 'spec_helper'

describe PassaporteWeb::Identity do

  describe "constants" do
    it { PassaporteWeb::Identity::ATTRIBUTES.should == [:accounts, :birth_date, :country, :cpf, :email, :first_name, :gender, :is_active, :language, :last_name, :nickname, :notifications, :send_myfreecomm_news, :send_partner_news, :services, :timezone, :update_info_url, :uuid, :password, :<PASSWORD>, :must_change_password, :inhibit_activation_message, :tos] }
    it { PassaporteWeb::Identity::UPDATABLE_ATTRIBUTES.should == [:first_name, :last_name, :nickname, :cpf, :birth_date, :gender, :send_myfreecomm_news, :send_partner_news, :country, :language, :timezone] }
    it { PassaporteWeb::Identity::CREATABLE_ATTRIBUTES.should == [:first_name, :last_name, :nickname, :cpf, :birth_date, :gender, :send_myfreecomm_news, :send_partner_news, :country, :language, :timezone, :email, :password, :<PASSWORD>, :must_change_password, :tos] }
  end

  describe ".new" do
    it "should instanciate an empty object" do
      identity = PassaporteWeb::Identity.new
      identity.attributes.should == {:accounts=>nil, :birth_date=>nil, :country=>nil, :cpf=>nil, :email=>nil, :first_name=>nil, :gender=>nil, :is_active=>nil, :language=>nil, :last_name=>nil, :nickname=>nil, :notifications=>nil, :send_myfreecomm_news=>nil, :send_partner_news=>nil, :services=>nil, :timezone=>nil, :update_info_url=>nil, :uuid=>nil, :password=><PASSWORD>, :password2=><PASSWORD>, :must_change_password=><PASSWORD>, :inhibit_activation_message=>nil, :tos=>nil}
    end
    it "should instanciate an object with attributes set" do
      attributes = {
        "last_name" => "<NAME>",
        "is_active" => true,
        "timezone" => "GMT-3",
        "nickname" => "Lula",
        "first_name" => "<NAME>",
        "send_partner_news" => false,
        "uuid" => "a5868d14-6529-477a-9c6b-a09dd42a7cd2",
        "language" => "pt_BR",
        "country" => "Brasil",
        "update_info_url" => "/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/",
        "send_myfreecomm_news" => false,
        "gender" => "M",
        "birth_date" => "1945-10-27",
        "email" => "<EMAIL>",
        "notifications" => {
          "count" => 0,
          "list" => "/notifications/api/"
        },
        "accounts" => [],
        "services" => {
          "myfinance" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/",
          "account_manager" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"
        },
        "password"=><PASSWORD>,
        "password2"=><PASSWORD>,
        "must_change_password"=><PASSWORD>,
        "inhibit_activation_message"=>nil,
        "tos"=>nil
      }
      identity = PassaporteWeb::Identity.new(attributes)
      identity.attributes.should == {:accounts=>[], :birth_date=>"1945-10-27", :country=>"Brasil", :cpf=>nil, :email=>"<EMAIL>", :first_name=>"<NAME>", :gender=>"M", :is_active=>true, :language=>"pt_BR", :last_name=>"<NAME>", :nickname=>"Lula", :notifications=>{"count"=>0, "list"=>"/notifications/api/"}, :send_myfreecomm_news=>false, :send_partner_news=>false, :services=>{"myfinance"=>"/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/", "account_manager"=>"/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"}, :timezone=>"GMT-3", :update_info_url=>"/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/", :uuid=>"a5868d14-6529-477a-9c6b-a09dd42a7cd2", :password=><PASSWORD>, :password2=><PASSWORD>, :must_change_password=>nil, :inhibit_activation_message=>nil, :tos=>nil}
      identity.last_name.should == "<NAME>"
      identity.is_active.should == true
      identity.timezone.should == "GMT-3"
      identity.nickname.should == "Lula"
      identity.first_name.should == "<NAME>"
      identity.send_partner_news.should == false
      identity.uuid.should == "a5868d14-6529-477a-9c6b-a09dd42a7cd2"
      identity.language.should == "pt_BR"
      identity.country.should == "Brasil"
      identity.update_info_url.should == "/profile/api/info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/"
      identity.send_myfreecomm_news.should == false
      identity.gender.should == "M"
      identity.birth_date.should == "1945-10-27"
      identity.email.should == "<EMAIL>"
      identity.notifications.should == {"count" => 0, "list" => "/notifications/api/"}
      identity.accounts.should == []
      identity.services.should == {"myfinance" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/myfinance/", "account_manager" => "/accounts/api/service-info/a5868d14-6529-477a-9c6b-a09dd42a7cd2/account_manager/"}
      identity.password.should be_nil
      identity.password2.should be_nil
      identity.must_change_password.should be_nil
      identity.inhibit_activation_message.should be_nil
      identity.tos.should be_nil
    end
  end

  describe "#== and #===" do
    it "should identify two profiles with the same uuid as equal" do
      p1 = PassaporteWeb::Identity.new('uuid' => 'some-uuid')
      p2 = PassaporteWeb::Identity.new('uuid' => 'some-uuid')
      p1.should == p2
      p1.should_not === p2
      p2.should == p1
      p2.should_not === p1
    end
    it "should identify two profiles with different uuids as different" do
      p1 = PassaporteWeb::Identity.new('uuid' => 'some-uuid-1')
      p2 = PassaporteWeb::Identity.new('uuid' => 'some-uuid-2')
      p1.should_not == p2
      p1.should_not === p2
      p2.should_not == p1
      p2.should_not === p1
      p3 = PassaporteWeb::Identity.new
      p1.should_not == p3
      p3.should_not == p1
      p3.should_not === p1
    end
  end

  describe ".find", :vcr => true do
    it "should find the requested profile by uuid" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56")
      identity.should be_instance_of(PassaporteWeb::Identity)
      identity.uuid.should == '5e32f927-c4ab-404e-a91c-b2abc05afb56'
      identity.email.should == '<EMAIL>'
      identity.update_info_url.should == '/accounts/api/identities/5e32f927-c4ab-404e-a91c-b2abc05afb56/'
      identity.accounts.size.should == 9
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil]
    end
    it "should find the requested profile by uuid, including expired accounts" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil]
    end
    it "should find the requested profile by uuid, including other services" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", false, true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
    end
    it "should find the requested profile by uuid, including other services and expired accounts" do
      identity = PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56", true, true)
      identity.accounts.size.should == 11
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
    end
    it "should raise an error if no profiles exist with that uuid" do
      expect {
        PassaporteWeb::Identity.find("invalid-uuid")
      }.to raise_error(RestClient::ResourceNotFound, '404 Resource Not Found')
    end
  end

  describe ".find_by_email", :vcr => true do
    it "should find the requested profile by email" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>")
      identity.should be_instance_of(PassaporteWeb::Identity)
      identity.uuid.should == '5e32f927-c4ab-404e-a91c-b2abc05afb56'
      identity.email.should == '<EMAIL>'
      identity.update_info_url.should == '/accounts/api/identities/5e32f927-c4ab-404e-a91c-b2abc05afb56/'
      identity.accounts.size.should == 9
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil]
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by email, including expired accounts" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] rescue nil }.sort.should == [nil]
      identity.services.size.should == 1
      identity.services.keys.should == ['identity_client']
    end
    it "should find the requested profile by email, including other services" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", false, true)
      identity.accounts.size.should == 10
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
    end
    it "should find the requested profile by email, including other services and expired accounts" do
      identity = PassaporteWeb::Identity.find_by_email("<EMAIL>", true, true)
      identity.accounts.size.should == 11
      identity.accounts.map { |a| a['expiration'] }.uniq.should == [nil, "2013-04-02 00:00:00", "2014-05-01 00:00:00"]
      identity.accounts.map { |a| a['services'] }.flatten.uniq.map { |s| s['slug'] }.sort.should == ["identity_client", "vc-promove"]
    end
    it "should raise an error if no profiles exist with that email" do
      expect {
        PassaporteWeb::Identity.find("<EMAIL>")
      }.to raise_error(RestClient::ResourceNotFound, '404 Resource Not Found')
    end
  end

  describe "#save", :vcr => true do
    describe "PUT" do
      let(:identity) { PassaporteWeb::Identity.find("5e32f927-c4ab-404e-a91c-b2abc05afb56") }
      context "on success" do
        it "should update the profile attributes on the server" do
          identity.first_name.should == 'Testador'
          identity.first_name = '<NAME>'
          identity.save.should be_true
          identity.first_name.should == '<NAME>'
{'fraction_non_alphanumeric': 0.16078175895765473, 'fraction_numerical': 0.05750271444082519, 'mean_word_length': 3.70879345603272, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 237, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7303418', 'n_tokens_mistral': 8769, 'n_tokens_neox': 8116, 'n_words': 1490}