<commit_before>require 'test_helper'

class MediaTest < ActiveSupport::TestCase
  test "should create media" do
    assert_kind_of Media, create_media
  end

  test "should normalize URL" do
    expected = 'https://ca.ios.ba/'
    variations = %w(
      https://ca.ios.ba
      ca.ios.ba
      https://ca.ios.ba:443
      https://ca.ios.ba//
      https://ca.ios.ba/?
      https://ca.ios.ba/#foo
      https://ca.ios.ba/
      https://ca.ios.ba
      https://ca.ios.ba/foo/..
      https://ca.ios.ba/?#
    )
    variations.each do |url|
      media = Media.new(url: url)
      assert_equal expected, media.url
    end

    media = Media.new(url: 'http://ca.ios.ba/a%c3%82/%7Euser?a=b')
    assert_match '//ca.ios.ba/a%C3%82/~user?a=b', media.url

  end

  test "should not normalize URL" do
    urls = %w(
      https://meedan.com/
      https://example.com/
      https://ca.ios.ba/?foo=bar
    )
    urls.each do |url|
      media = Media.new(url: url)
      assert_equal url, media.url
    end
  end

  test "should follow redirection of relative paths" do
    WebMock.enable!
    WebMock.stub_request(:any, /https?:\/\/www.almasryalyoum.com\/node\/517699/).to_return(body: '', headers: { location: '/editor/details/968' }, status: 302)
    Media.any_instance.stubs(:get_canonical_url).returns(false)
    m = create_media url: 'https://www.almasryalyoum.com/node/517699'
    assert_match /almasryalyoum.com\/editor\/details\/968/, m.url
    WebMock.disable!
  end

  test "should parse URL including cloudflare credentials on header" do
      host = ENV['hosts']
      url = 'https://example.com/'
      parsed_url = RequestHelper.parse_url url
      m = Media.new url: url
      header_options_without_cf = RequestHelper.html_options(url).merge(read_timeout: PenderConfig.get('timeout', 30).to_i)
      assert_nil header_options_without_cf['CF-Access-Client-Id']
      assert_nil header_options_without_cf['CF-Access-Client-Secret']
  
      PenderConfig.current = nil
      ENV['hosts'] = {"example.com"=>{"cf_credentials"=>"1234:5678"}}.to_json
      header_options_with_cf = RequestHelper.html_options(url).merge(read_timeout: PenderConfig.get('timeout', 30).to_i)
      assert_equal '1234', header_options_with_cf['CF-Access-Client-Id']
      assert_equal '5678', header_options_with_cf['CF-Access-Client-Secret']
      OpenURI.stubs(:open_uri).with(parsed_url, header_options_without_cf).raises(RuntimeError.new('unauthorized'))
      OpenURI.stubs(:open_uri).with(parsed_url, header_options_with_cf)
      assert_equal Nokogiri::HTML::Document, m.send(:get_html, RequestHelper.html_options(m.url)).class
  ensure
    ENV['hosts'] = host
  end

  test "should get relative canonical URL parsed from html tags" do
    m = create_media url: 'https://www.bbc.com'
    data = m.as_json
    assert_match 'https://www.bbc.com', m.url
    assert_match 'BBC', data['title']
    assert_match /Breaking news/, data['description']
    assert_equal '', data['published_at']
    assert_equal '', data['username']
    assert_equal 'https://www.bbc.com', data['author_url']
    assert_equal '', data['picture']
  end
  
  test "should get canonical URL parsed from html tags" do
    doc = ''
    open('test/data/page-with-url-on-tag.html') { |f| doc = f.read }
    Media.any_instance.stubs(:doc).returns(Nokogiri::HTML(doc))

    media1 = create_media url: 'http://example.com/2016/08/bom-dia-2.html'
    media2 = create_media url: 'http://example.com/?p=6704&fake=123'
    assert_equal media1.url, media2.url
    Media.any_instance.unstub(:doc)
  end

  test "should store the picture address" do
    m = create_media url: 'http://xkcd.com/448/'
    data = m.as_json
    assert_match /Good Morning/, data['title']
    assert_equal '', data['description']
    assert_equal '', data['published_at']
    assert_equal '', data['username']
    assert_match 'https://xkcd.com', data['author_url']
    assert_equal '', data['screenshot']
    assert_match /imgs/, data['picture']
  end

  test "should return author picture" do
    Media.any_instance.stubs(:doc).returns(Nokogiri::HTML("<meta property='og:image' content='https://github.githubassets.com/images/modules/open_graph/github-logo.png'>"))
    url = 'http://github.com'
    id = Media.get_id url
    m = create_media url: url<commit_msg>Stub HTTP request for flaky Media test<commit_after>require 'test_helper'

class MediaTest < ActiveSupport::TestCase
  test "should create media" do
    assert_kind_of Media, create_media
  end

  test "should normalize URL" do
    expected = 'https://ca.ios.ba/'
    variations = %w(
      https://ca.ios.ba
      ca.ios.ba
      https://ca.ios.ba:443
      https://ca.ios.ba//
      https://ca.ios.ba/?
      https://ca.ios.ba/#foo
      https://ca.ios.ba/
      https://ca.ios.ba
      https://ca.ios.ba/foo/..
      https://ca.ios.ba/?#
    )
    variations.each do |url|
      media = Media.new(url: url)
      assert_equal expected, media.url
    end

    media = Media.new(url: 'http://ca.ios.ba/a%c3%82/%7Euser?a=b')
    assert_match '//ca.ios.ba/a%C3%82/~user?a=b', media.url

  end

  test "should not normalize URL" do
    urls = %w(
      https://meedan.com/
      https://example.com/
      https://ca.ios.ba/?foo=bar
    )
    urls.each do |url|
      media = Media.new(url: url)
      assert_equal url, media.url
    end
  end

  test "should follow redirection of relative paths" do
    WebMock.enable!
    WebMock.stub_request(:any, /https?:\/\/www.almasryalyoum.com\/node\/517699/).to_return(body: '', headers: { location: '/editor/details/968' }, status: 302)
    Media.any_instance.stubs(:get_canonical_url).returns(false)
    m = create_media url: 'https://www.almasryalyoum.com/node/517699'
    assert_match /almasryalyoum.com\/editor\/details\/968/, m.url
    WebMock.disable!
  end

  test "should parse URL including cloudflare credentials on header" do
      host = ENV['hosts']
      url = 'https://example.com/'
      parsed_url = RequestHelper.parse_url url
      m = Media.new url: url
      header_options_without_cf = RequestHelper.html_options(url).merge(read_timeout: PenderConfig.get('timeout', 30).to_i)
      assert_nil header_options_without_cf['CF-Access-Client-Id']
      assert_nil header_options_without_cf['CF-Access-Client-Secret']
  
      PenderConfig.current = nil
      ENV['hosts'] = {"example.com"=>{"cf_credentials"=>"1234:5678"}}.to_json
      header_options_with_cf = RequestHelper.html_options(url).merge(read_timeout: PenderConfig.get('timeout', 30).to_i)
      assert_equal '1234', header_options_with_cf['CF-Access-Client-Id']
      assert_equal '5678', header_options_with_cf['CF-Access-Client-Secret']
      OpenURI.stubs(:open_uri).with(parsed_url, header_options_without_cf).raises(RuntimeError.new('unauthorized'))
      OpenURI.stubs(:open_uri).with(parsed_url, header_options_with_cf)
      assert_equal Nokogiri::HTML::Document, m.send(:get_html, RequestHelper.html_options(m.url)).class
  ensure
    ENV['hosts'] = host
  end

  test "should get relative canonical URL parsed from html tags" do
    m = create_media url: 'https://www.bbc.com'
    data = m.as_json
    assert_match 'https://www.bbc.com', m.url
    assert_match 'BBC', data['title']
    assert_match /Breaking news/, data['description']
    assert_equal '', data['published_at']
    assert_equal '', data['username']
    assert_equal 'https://www.bbc.com', data['author_url']
    assert_equal '', data['picture']
  end
  
  test "should get canonical URL parsed from html tags" do
    doc = ''
    open('test/data/page-with-url-on-tag.html') { |f| doc = f.read }
    Media.any_instance.stubs(:doc).returns(Nokogiri::HTML(doc))

    media1 = create_media url: 'http://example.com/2016/08/bom-dia-2.html'
    media2 = create_media url: 'http://example.com/?p=6704&fake=123'
    assert_equal media1.url, media2.url
    Media.any_instance.unstub(:doc)
  end

  test "should store the picture address" do
    m = create_media url: 'http://xkcd.com/448/'
    data = m.as_json
    assert_match /Good Morning/, data['title']
    assert_equal '', data['description']
    assert_equal '', data['published_at']
    assert_equal '', data['username']
    assert_match 'https://xkcd.com', data['author_url']
    assert_equal '', data['screenshot']
    assert_match /imgs/, data['picture']
  end

  test "should return author picture" do
    WebMock.stub_request(:get, /github.com/).to_return(status: 200, body: "<meta property='og:image' content='https://github.githubassets.com/images/modules/open_graph/github-logo.png'>")
    url = 'http://github.com'
    id = Media.get_id url
    m = create_media url: url
{'fraction_non_alphanumeric': 0.1495240306477827, 'fraction_numerical': 0.017065242628279544, 'mean_word_length': 3.344427634896621, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 40, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '782848', 'n_tokens_mistral': 3045, 'n_tokens_neox': 2925, 'n_words': 547}