<commit_before>
require "multi_json"
require "escape_escape_escape"

BRACKET = " < %3C &lt &lt; &LT &LT; &#60 &#060 &#0060
&#00060 &#000060 &#0000060 &#60; &#060; &#0060; &#00060;
&#000060; &#0000060; &#x3c &#x03c &#x003c &#x0003c &#x00003c
&#x000003c &#x3c; &#x03c; &#x003c; &#x0003c; &#x00003c;
&#x000003c; &#X3c &#X03c &#X003c &#X0003c &#X00003c &#X000003c
&#X3c; &#X03c; &#X003c; &#X0003c; &#X00003c; &#X000003c;
&#x3C &#x03C &#x003C &#x0003C &#x00003C &#x000003C &#x3C; &#x03C;
&#x003C; &#x0003C; &#x00003C; &#x000003C; &#X3C &#X03C
&#X003C &#X0003C &#X00003C &#X000003C &#X3C; &#X03C; &#X003C; &#X0003C;
&#X00003C; &#X000003C; \x3c \x3C \u003c \u003C ".strip

Dir.glob("specs/as_json/*.json").sort.each { |f|
  contents = MultiJson.load(File.read f)
  method_name = File.basename(f).gsub(/\A\d+-|\.json\Z/, '')
  describe ":#{method_name}" do
    contents.each { |t|
      it t["it"] do
        i      = t["input"]
        o      = t["output"]
        actual = Escape_Escape_Escape.send(method_name, i)

        case o
        when String
          actual.should == o
        when Array
          target = o.pop
          begin
            if o[1].is_a?(Array)
              meth = o.shift
              args = o.shift
              actual = actual.send(meth, *args)
            else
              fail "Unknown method: #{o[0].inspect}"
            end
          end while !o.empty?

          actual.should == target
        end # === case
      end # === it
    }
  end
}



describe ":clean_utf8" do

  it "replaces nb spaces (160 codepoint) with regular ' ' spaces" do
    s = [160, 160,64, 116, 119, 101, 108, 108, 121, 109, 101, 160, 102, 105, 108, 109].
      inject('', :<<)

    "@twellyme film".should == E.clean_utf8(s).strip
  end

  it "replaces tabs with spaces" do
    s = "a\t \ta"
    "a   a".should == E.clean_utf8(s)
  end

end # === describe :clean_utf8 ===



describe ':un_e' do

  it 'un-escapes escaped text mixed with HTML' do
    s = "<p>Hi&amp;</p>"
    "<p>Hi&</p>".should == E.un_e(s)
  end

  it 'un-escapes special chars: "Hello ©®∆"' do
    s = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello & World ©®∆"
    t.should == E.un_e(s)
  end

  it 'un-escapes all 70 different combos of "<"' do
    "< %3C".should == E.un_e(BRACKET).split.uniq.join(' ')
  end

end # === describe :un_e


describe ':escape' do

  it 'does not re-escape already escaped text mixed with HTML' do
    h = "<p>Hi</p>"
    e = E.escape(h)
    o = e + h
    E.escape(o).shold == E.escape(h + h)
  end

  it 'escapes special chars: "Hello ©®∆"' do
    s = "Hello & World ©®∆"
    t = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello &amp; World &copy;&reg;&#x2206;"
    t.should == E.escape(s)
  end

  it 'escapes all 70 different combos of "<"' do
    "&lt; %3C".should == E.escape(BRACKET).split.uniq.join(' ')
  end

  it 'escapes all keys in nested objects' do
    html = "<b>test</b>"
    t    = {" a &gt;" => {" a &gt;" => E.escape(html) }}
    t.should == E.escape({" a >" => {" a >" => html}})
  end

  it 'escapes all values in nested objects' do
    html = "<b>test</b>"
    t    = {name: {name: E.escape(html)}}
    t.should == E.escape({name:{name: html}})
  end

  it 'escapes all values in nested arrays' do
    html = "<b>test</b>"
    [{name: {name: E.escape(html)}}].should == E.escape([{name:{name: html}}])
  end

  'uri url href'.split.each { |k| # ==============================================

    it "escapes values of keys :#{k} that are valid /path" do
      a = {:key=>{:"#{k}" => "/path/mine/&"}}
      t = {:key=>{:"#{k}" => "/path/mine/&amp;"}}
      t.should == E.escape(a)
    end

    it "sets nil any keys ending with :#{k} and have invalid uri" do
      a = {:key=>{:"#{k}" => "javascript:alert(s)"}}
      t = {:key=>{:"#{k}" => nil                  }}
      t.should == E.escape(a)
    end

    it "sets nil any keys ending with _#{k} and have invalid uri" do
      a = {:key=>{:"my_#{k}" => "javascript:alert(s)"}}
      t = {:key=>{:"my_#{k}" => nil                  }}
      t.should == E.escape(a)
    end

    it "escapes values of keys with _#{k} that are valid https uri" do
      a = {:key=>{:"my_#{k}" => "https://www.yahoo.com/&"}}
      t = {:key=>{:"my_#{k}" => "https://www.yahoo.com/&amp;"}}
      t.should == E.escape(a)
    end

    it "escapes values of keys with _#{k} that are valid uri" do
      a = {:key=>{:"my_#{k}" => "http://www.yahoo.com/&"}}
      t = {:key=>{:"my_#{k}" => "http://www.yahoo.com/&amp;"}}
      t.should == E.escape(a)
    end

    it "escapes values of keys ending with _#{k} that are valid /path" do
      a = {:key=>{:"my_#{k}" => "/path/mine/&"}}
      t = {:key=>{:"my_#{k}" => "/path/mine/&amp;"}}
      t.should == E.escape(a)
    end

    it "allows unicode uris" do
      a = {:key=>{:"my_#{k}" => "http://кц.рф"}}
      t = {:key=>{:"my_#{k}" => "http://&#x43a;&#x446;.&#x440;&#x444;"}}
      t.should == E.escape(a)
    end
  }

  # === password field
  it "does not escape :pass_word key" do
    a = {:pass_word=>"&&&"}
    a.should == E.escape(a)
  end

  it "does not escape :confirm_pass_word key" do
    a = {:confirm_pass_word=>"&&&"}
    a.should == E.escape(a)
  end

  [true, false].each do |v|
    it "does not escape #{v.inspect}" do
      a = {:something=>v}
      a.should == E.escape(a)
    end
  end

  it "does not escape numbers" do
    a = {:something=>1}
    E.escape(a).should == a
  end

end # === end desc



describe( 'uri' ) {

  it( 'normalizes address' ) {
    s = "hTTp://wWw.test.com/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains: <' ) {
    s = "http://www.test.com/<something/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains HTML entities' ) {
    s = "http://6&#9;6.000146.0x7.147/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains HTML entities' ) {
    s = "http://www.test.com/&nbsp;s/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if query string contains HTML entities' ) {
    s = "http://www.test.com/s/test?t&nbsp;test"
    E.uri(s).should == s.upcase
  }

} # === end desc

describe :plaintext do

  it "normalizes string" do
    Escape_Escape_Escape.plaintext("\u2167").should == "VIII"
  end

end # === describe :plaintext

describe( 'Sanitize') {

  it( 'un-escapes escaped text mixed with HTML') {
    s = "<p>Hi&amp;</p>"
    U(s).should == "<p>Hi&</p>"
  }

  it( 'does not re-escape already escaped text mixed with HTML') {
    h = "<p>Hi</p>"
    e = _s.escapeHTML(h)
    o = e + h
    E(o).should == _s.escapeHTML(h + h)
  }

  it( 'escapes special chars: "Hello ©®∆"') {
    s = "Hello & World ©®∆"
    t = "Hello &amp; World &#169;&#174;&#8710;"
    E(s).should == t
  }

  it( 'un-escapes special chars: "Hello ©®∆"') {
    s = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello & World ©®∆"
    U(s).should == t
  }

  it( 'escapes all 70 different combos of "<"') {
    E(BRACKET.strip).split.uniq.join(' ').should == "&lt;"
  }

  it( 'un-escapes all 70 different combos of "<"') {
    U(BRACKET.strip).split.join(' ').should == "<"
  }

  it( 'escapes all keys in nested objects') {
    HTML = "<b>test</b>"
    input = {
      " a >" => {" a >"=> HTML}
    }
    output = {
      " a &gt;" => {
        " a &gt;" => string.escapeHTML(HTML)
      }
    }
    E(input).should == output
  }

  it( 'escapes all values in nested objects') {
    HTML = "<b>test</b>"
    E({name:{name: HTML}}).should == {name: {name: _s.escapeHTML(HTML)}}
  }

  it( 'escapes all values in nested arrays') {
    HTML = "<b>test</b>"
    E([{name:{name: HTML}}]).should == [{name: {name: _s.escapeHTML(HTML)}}]
  }

} # // === end desc



describe :other do

  it 'sanitizes :css :expression regardless of the case' do
    input = "eXprEssioN(alert('xss!'));"
    E(input).should == nil
  end

  it 'sanitizes :css :expression when ( or ) is an html entity: &#40; &#41;' do
    input = "eXprEssioN&#40;alert('xss!')&#41;"
    E(input).should == nil
  end

  it 'sanitizes :css :expression when ( is html entity regardless of car: &rPaR;' do
    input = "eXprEssioN&rPaR;alert('xss!'))"
    E(input).should == nil
  end

  it 'sanitizes css_href' do
    input = "smtp://file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &#47;' do
    input = "smtp:&#47;&#47;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &#x0002F;' do
    input = "smtp:&#x0002F;&#x0002F;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &sol;' do
    input = "smtp:&sol;&sol;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href with encoded slashes' do
    input = "smtp:&#047;&#047;file.com&#047;img.png"
    E(input).should == nil
  end

  it 'sanitizes javascript: href' do
    input = 'jAvAscript://alert()'
    E(input).should == nil
  end

  it 'sanitizes javascript: href with encoded colons' do
    input = "javascript&#058;//alert()"
    E(input).should == nil
  end

  it 'sanitizes javascript: href with encoded slashes' do
    input = "javascript:&#047;&#047;alert()"
    E(input).should == nil
  end

  describe :html do
    it 'removes Unicode characters that do not belong in html' do
      input = "\u0340\u0341\u17a3\u17d3\u2028\u2029\u202a"
      E(input).should == ''
    end
  end

end # === describe :other


<commit_msg>Dev. checkpoint.<commit_after>
require "multi_json"
require "escape_escape_escape"

BRACKET = " < %3C &lt &lt; &LT &LT; &#60 &#060 &#0060
&#00060 &#000060 &#0000060 &#60; &#060; &#0060; &#00060;
&#000060; &#0000060; &#x3c &#x03c &#x003c &#x0003c &#x00003c
&#x000003c &#x3c; &#x03c; &#x003c; &#x0003c; &#x00003c;
&#x000003c; &#X3c &#X03c &#X003c &#X0003c &#X00003c &#X000003c
&#X3c; &#X03c; &#X003c; &#X0003c; &#X00003c; &#X000003c;
&#x3C &#x03C &#x003C &#x0003C &#x00003C &#x000003C &#x3C; &#x03C;
&#x003C; &#x0003C; &#x00003C; &#x000003C; &#X3C &#X03C
&#X003C &#X0003C &#X00003C &#X000003C &#X3C; &#X03C; &#X003C; &#X0003C;
&#X00003C; &#X000003C; \x3c \x3C \u003c \u003C ".strip

Dir.glob("specs/as_json/*.json").sort.each { |f|
  contents = MultiJson.load(File.read f)
  method_name = File.basename(f).gsub(/\A\d+-|\.json\Z/, '')
  describe ":#{method_name}" do
    contents.each { |t|
      it t["it"] do
        i      = t["input"]
        o      = t["output"]
        actual = Escape_Escape_Escape.send(method_name, i)

        case o
        when String
          actual.should == o
        when Array
          target = o.pop
          begin
            if o[1].is_a?(Array)
              meth = o.shift
              args = o.shift
              actual = actual.send(meth, *args)
            else
              fail "Unknown method: #{o[0].inspect}"
            end
          end while !o.empty?

          actual.should == target
        end # === case
      end # === it
    }
  end
}



describe ":clean_utf8" do

  it "replaces nb spaces (160 codepoint) with regular ' ' spaces" do
    s = [160, 160,64, 116, 119, 101, 108, 108, 121, 109, 101, 160, 102, 105, 108, 109].
      inject('', :<<)

    "@twellyme film".should == E.clean_utf8(s).strip
  end

  it "replaces tabs with spaces" do
    s = "a\t \ta"
    "a   a".should == E.clean_utf8(s)
  end

end # === describe :clean_utf8 ===



describe ':un_e' do

  it 'un-escapes escaped text mixed with HTML' do
    s = "<p>Hi&amp;</p>"
    "<p>Hi&</p>".should == E.un_e(s)
  end

  it 'un-escapes special chars: "Hello ©®∆"' do
    s = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello & World ©®∆"
    t.should == E.un_e(s)
  end

  it 'un-escapes all 70 different combos of "<"' do
    "< %3C".should == E.un_e(BRACKET).split.uniq.join(' ')
  end

end # === describe :un_e


describe ':escape' do

  it 'does not re-escape already escaped text mixed with HTML' do
    h = "<p>Hi</p>"
    e = E.escape(h)
    o = e + h
    E.escape(o).shold == E.escape(h + h)
  end

  it 'escapes special chars: "Hello ©®∆"' do
    s = "Hello & World ©®∆"
    t = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello &amp; World &copy;&reg;&#x2206;"
    t.should == E.escape(s)
  end

  it 'escapes all 70 different combos of "<"' do
    "&lt; %3C".should == E.escape(BRACKET).split.uniq.join(' ')
  end

  it 'escapes all keys in nested objects' do
    html = "<b>test</b>"
    t    = {" a &gt;" => {" a &gt;" => E.escape(html) }}
    t.should == E.escape({" a >" => {" a >" => html}})
  end

  it 'escapes all values in nested objects' do
    html = "<b>test</b>"
    t    = {name: {name: E.escape(html)}}
    t.should == E.escape({name:{name: html}})
  end

  it 'escapes all values in nested arrays' do
    html = "<b>test</b>"
    [{name: {name: E.escape(html)}}].should == E.escape([{name:{name: html}}])
  end

  'uri url href'.split.each { |k| # ==============================================

    it "escapes values of keys :#{k} that are valid /path" do
      a = {:key=>{:"#{k}" => "/path/mine/&"}}
      t = {:key=>{:"#{k}" => "/path/mine/&amp;"}}
      t.should == E.escape(a)
    end

    it "sets nil any keys ending with :#{k} and have invalid uri" do
      a = {:key=>{:"#{k}" => "javascript:alert(s)"}}
      t = {:key=>{:"#{k}" => nil                  }}
      t.should == E.escape(a)
    end

    it "sets nil any keys ending with _#{k} and have invalid uri" do
      a = {:key=>{:"my_#{k}" => "javascript:alert(s)"}}
      t = {:key=>{:"my_#{k}" => nil                  }}
      t.should == E.escape(a)
    end

    it "escapes values of keys with _#{k} that are valid https uri" do
      a = {:key=>{:"my_#{k}" => "https://www.yahoo.com/&"}}
      t = {:key=>{:"my_#{k}" => "https://www.yahoo.com/&amp;"}}
      t.should == E.escape(a)
    end

    it "escapes values of keys with _#{k} that are valid uri" do
      a = {:key=>{:"my_#{k}" => "http://www.yahoo.com/&"}}
      t = {:key=>{:"my_#{k}" => "http://www.yahoo.com/&amp;"}}
      t.should == E.escape(a)
    end

    it "escapes values of keys ending with _#{k} that are valid /path" do
      a = {:key=>{:"my_#{k}" => "/path/mine/&"}}
      t = {:key=>{:"my_#{k}" => "/path/mine/&amp;"}}
      t.should == E.escape(a)
    end

    it "allows unicode uris" do
      a = {:key=>{:"my_#{k}" => "http://кц.рф"}}
      t = {:key=>{:"my_#{k}" => "http://&#x43a;&#x446;.&#x440;&#x444;"}}
      t.should == E.escape(a)
    end
  }

  # === password field
  it "does not escape :pass_word key" do
    a = {:pass_word=>"&&&"}
    a.should == E.escape(a)
  end

  it "does not escape :confirm_pass_word key" do
    a = {:confirm_pass_word=>"&&&"}
    a.should == E.escape(a)
  end

  [true, false].each do |v|
    it "does not escape #{v.inspect}" do
      a = {:something=>v}
      a.should == E.escape(a)
    end
  end

  it "does not escape numbers" do
    a = {:something=>1}
    E.escape(a).should == a
  end

end # === end desc



describe( 'uri' ) {

  it( 'normalizes address' ) {
    s = "hTTp://wWw.test.com/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains: <' ) {
    s = "http://www.test.com/<something/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains HTML entities' ) {
    s = "http://6&#9;6.000146.0x7.147/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if path contains HTML entities' ) {
    s = "http://www.test.com/&nbsp;s/"
    E.uri(s).should == s.upcase
  }

  it( 'returns an Error if query string contains HTML entities' ) {
    s = "http://www.test.com/s/test?t&nbsp;test"
    E.uri(s).should == s.upcase
  }

} # === end desc

describe :plaintext do

  it "normalizes string" do
    Escape_Escape_Escape.plaintext("\u2167").should == "VIII"
  end

end # === describe :plaintext

describe( 'Sanitize') {

  it( 'un-escapes escaped text mixed with HTML') {
    s = "<p>Hi&amp;</p>"
    U(s).should == "<p>Hi&</p>"
  }

  it( 'does not re-escape already escaped text mixed with HTML') {
    h = "<p>Hi</p>"
    e = _s.escapeHTML(h)
    o = e + h
    E(o).should == _s.escapeHTML(h + h)
  }

  it( 'escapes special chars: "Hello ©®∆"') {
    s = "Hello & World ©®∆"
    t = "Hello &amp; World &#169;&#174;&#8710;"
    E(s).should == t
  }

  it( 'un-escapes special chars: "Hello ©®∆"') {
    s = "Hello &amp; World &#169;&#174;&#8710;"
    t = "Hello & World ©®∆"
    U(s).should == t
  }

  it( 'escapes all 70 different combos of "<"') {
    E(BRACKET.strip).split.uniq.join(' ').should == "&lt;"
  }

  it( 'un-escapes all 70 different combos of "<"') {
    U(BRACKET.strip).split.join(' ').should == "<"
  }

  it( 'escapes all keys in nested objects') {
    HTML = "<b>test</b>"
    input = {
      " a >" => {" a >"=> HTML}
    }
    output = {
      " a &gt;" => {
        " a &gt;" => string.escapeHTML(HTML)
      }
    }
    E(input).should == output
  }

  it( 'escapes all values in nested objects') {
    HTML = "<b>test</b>"
    E({name:{name: HTML}}).should == {name: {name: _s.escapeHTML(HTML)}}
  }

  it( 'escapes all values in nested arrays') {
    HTML = "<b>test</b>"
    E([{name:{name: HTML}}]).should == [{name: {name: _s.escapeHTML(HTML)}}]
  }

} # // === end desc



describe :other do

  it 'sanitizes :css :expression regardless of the case' do
    input = "eXprEssioN(alert('xss!'));"
    E(input).should == nil
  end

  it 'sanitizes :css :expression when ( or ) is an html entity: &#40; &#41;' do
    input = "eXprEssioN&#40;alert('xss!')&#41;"
    E(input).should == nil
  end

  it 'sanitizes :css :expression when ( is html entity regardless of car: &rPaR;' do
    input = "eXprEssioN&rPaR;alert('xss!'))"
    E(input).should == nil
  end

  it 'sanitizes css_href' do
    input = "smtp://file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &#47;' do
    input = "smtp:&#47;&#47;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &#x0002F;' do
    input = "smtp:&#x0002F;&#x0002F;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href event if slash is html entity: &sol;' do
    input = "smtp:&sol;&sol;file.com/img.png"
    E(input).should == nil
  end

  it 'sanitizes css_href with encoded slashes' do
    input = "smtp:&#047;&#047;file.com&#047;img.png"
    E(input).should == nil
  end

  it 'sanitizes javascript: href' do
    input = 'jAvAscript://alert()'
    E(input).should == nil
  end

  it 'sanitizes javascript: href with encoded colons' do
    input = "javascript&#058;//alert()"
    E(input).should == nil
  end

  it 'sanitizes javascript: href with encoded slashes' do
    input = "javascript:&#047;&#047;alert()"
    E(input).should == nil
  end

  describe :html do

    it 'has the same REGEX_UNSUITABLE_CHARS as Sanitize' do
      Escape_Escape_Escape::REGEX_UNSUITABLE_CHARS.
        should == Sanitize::REGEX_UNSUITABLE_CHARS 
    end

    it 'removes Unicode characters that do not belong in html' do
      input = "\u0340\u0341\u17a3\u17d3\u2028\u2029\u202a"
      E(input).should == ''
    end

  end

end # === describe :other



{'fraction_non_alphanumeric': 0.20459060964132594, 'fraction_numerical': 0.04601066610896162, 'mean_word_length': 2.622537878787879, 'pattern_counts': {'":': 4, '<': 73, '<?xml version=': 0, '>': 139, 'https://': 4, 'lorem ipsum': 0, 'www.': 14, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7294367', 'n_tokens_mistral': 8214, 'n_tokens_neox': 7379, 'n_words': 1903}