<commit_before>require 'spec_helper'

describe DataURL do
  context :parse do
    it "should return nils if URL is nil" do
      DataURL.parse(nil).should == [nil, nil, nil]
    end
    
    it "should return empty string for empty data" do
      DataURL.parse('data:,').first.should == ''
    end
    
    it "should parse base64'd data as base64" do
      DataURL.parse('data:;base64,YWJj').should == ['abc', 'application/octet-stream', true]
    end
    
    it "should parse URL-encoded data as URL-encoded" do
      DataURL.parse('data:,abc%20').should == ['abc ', 'application/octet-stream', false]
    end
    
    it "should parse out content types" do
      DataURL.parse('data:text/plain,').should == ['', 'text/plain', false]
    end
    
    it "should raise an exception for invalid strings" do
      expect { DataURL.parse('gibberish') }.to raise_error DataURL::InvalidURLError
    end
    
    it "should raise an exception for non-data URLs" do
      expect { DataURL.parse('http://google.com/some,thing') }.to raise_error DataURL::InvalidURLError
    end
  end
end<commit_msg>Test that we ignore gibberish in base64 strings<commit_after>require 'spec_helper'

describe DataURL do
  context :parse do
    it "should return nils if URL is nil" do
      DataURL.parse(nil).should == [nil, nil, nil]
    end
    
    it "should return empty string for empty data" do
      DataURL.parse('data:,').first.should == ''
    end
    
    it "should parse base64'd data as base64" do
      DataURL.parse('data:;base64,YWJj').should == ['abc', 'application/octet-stream', true]
    end
    
    it "should parse URL-encoded data as URL-encoded" do
      DataURL.parse('data:,abc%20').should == ['abc ', 'application/octet-stream', false]
    end
    
    it "should parse out content types" do
      DataURL.parse('data:text/plain,').should == ['', 'text/plain', false]
    end
    
    it "should raise an exception for invalid strings" do
      expect { DataURL.parse('gibberish') }.to raise_error DataURL::InvalidURLError
    end
    
    it "should raise an exception for non-data URLs" do
      expect { DataURL.parse('http://google.com/some,thing') }.to raise_error DataURL::InvalidURLError
    end
    
    it "should ignore invalid crap in base64'd data" do
      DataURL.parse('data:;base64,YWJj!YWJj').first.should == 'abcabc'
    end
  end
end
{'fraction_non_alphanumeric': 0.12203389830508475, 'fraction_numerical': 0.009322033898305085, 'mean_word_length': 3.0221465076660987, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3665139', 'n_tokens_mistral': 767, 'n_tokens_neox': 743, 'n_words': 222}