# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


import unittest
from recommender import url_util

  def testDeduplicate(self):
    self.assertEqual(['https://a.test'],
                     url_util.DeduplicateUrls(
                         ['https://a.test', 'http://a.test']))
    self.assertEqual(['http://a.test', 'http://b.test'],
                     url_util.DeduplicateUrls([
                         'http://a.test', 'https://a.test///', 'HTTP://a.test/',
                         'http://www.a.test/', 'http://b.test'
                     ]))

  def testRemoveUtmParameters(self):
    # Unchanged URL:
    self.assertEqual(
        'https://a.test/some/path?param1=a&param2=b#fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/path?param1=a&param2=b#fragment'))

    self.assertEqual(
        'https://a.test/some/path?param1=a#fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/path?param1=a&utm_campaign=bla#fragment'))

    self.assertEqual(
        'https://a.test/some/utm_path#utm_fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/utm_path?utm_campaign=bla&utm_source=bla#utm_fragment'
        ))

    self.assertEqual(

# Path: recommender/url_util.py
# class PageMetadata(object):
# class Error(Exception):
# class InvalidURLError(Error):
# def DeduplicateUrls(urls):
# def Normalize(url):
# def RemoveUtmParameters(url):
# def Resolve(base_url, url):
# def _GetContentTypeAndCharset(headers):
# def GetPageContent(url):
# def GetPageMetadata(url):
# def FindContent(tree, paths, filter_fn=None):
# def FindFeedUrl(url, tree, path, result):
# URL_FETCH_DEADLINE_SECONDS = 15
# FEED_CONTENT_TYPES = set([
#     'application/atom+xml', 'application/rdf+xml', 'application/rss+xml',
#     'application/x-netcdf', 'application/xml', 'text/xml'
# ])


# Path: recommender/url_util_test.py
import unittest
from recommender import url_util


  def testDeduplicate(self):
    self.assertEqual(['https://a.test'],
                     url_util.DeduplicateUrls(
                         ['https://a.test', 'http://a.test']))
    self.assertEqual(['http://a.test', 'http://b.test'],
                     url_util.DeduplicateUrls([
                         'http://a.test', 'https://a.test///', 'HTTP://a.test/',
                         'http://www.a.test/', 'http://b.test'
                     ]))

  def testRemoveUtmParameters(self):
    # Unchanged URL:
    self.assertEqual(
        'https://a.test/some/path?param1=a&param2=b#fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/path?param1=a&param2=b#fragment'))

    self.assertEqual(
        'https://a.test/some/path?param1=a#fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/path?param1=a&utm_campaign=bla#fragment'))

    self.assertEqual(
        'https://a.test/some/utm_path#utm_fragment',
        url_util.RemoveUtmParameters(
            'https://a.test/some/utm_path?utm_campaign=bla&utm_source=bla#utm_fragment'
        ))

    self.assertEqual(

        'invalid_url_with_utm_string',