<commit_before>[
    "http://*youtube.com/watch*", 
    "http://*.youtube.com/v/*", 
    "http://youtu.be/*", 
    "http://*.youtube.com/user/*#*", 
    "http://*.youtube.com/*#*/*", 
    "http://www.veoh.com/*/watch/*", 
    "http://*justin.tv/*", 
    "http://*justin.tv/*/b/*", 
    "http://www.ustream.tv/recorded/*", 
    "http://www.ustream.tv/channel/*", 
    "http://qik.com/video/*", 
    "http://qik.com/*", 
    "http://*revision3.com/*", 
    "http://*.dailymotion.com/video/*", 
    "http://*.dailymotion.com/*/video/*", 
    "http://www.collegehumor.com/video:*", 
    "http://*twitvid.com/*", 
    "http://www.break.com/*/*", 
    "http://vids.myspace.com/index.cfm?fuseaction=vids.individual&videoid*", 
    "http://www.myspace.com/index.cfm?fuseaction=*&videoid*", 
    "http://www.metacafe.com/watch/*", 
    "http://blip.tv/file/*", 
    "http://*.blip.tv/file/*", 
    "http://video.google.com/videoplay?*", 
    "http://*revver.com/video/*", 
    "http://video.yahoo.com/watch/*/*", 
    "http://video.yahoo.com/network/*", 
    "http://*viddler.com/explore/*/videos/*", 
    "http://liveleak.com/view?*", 
    "http://www.liveleak.com/view?*", 
    "http://animoto.com/play/*", 
    "http://dotsub.com/view/*", 
    "http://www.overstream.net/view.php?oid=*", 
    "http://*yfrog.*/*", 
    "http://tweetphoto.com/*", 
    "http://www.flickr.com/photos/*", 
    "http://*twitpic.com/*", 
    "http://*imgur.com/*", 
    "http://*.posterous.com/*", 
    "http://post.ly/*", 
    "http://twitgoo.com/*", 
    "http://i*.photobucket.com/albums/*", 
    "http://gi*.photobucket.com/groups/*", 
    "http://phodroid.com/*/*/*", 
    "http://www.mobypicture.com/user/*/view/*", 
    "http://moby.to/*", 
    "http://xkcd.com/*", 
    "http://www.xkcd.com/*", 
    "http://www.asofterworld.com/index.php?id=*", 
    "http://www.qwantz.com/index.php?comic=*", 
    "http://23hq.com/*/photo/*", 
    "http://www.23hq.com/*/photo/*", 
    "http://*dribbble.com/shots/*", 
    "http://drbl.in/*", 
    "http://*.smugmug.com/*", 
    "http://*.smugmug.com/*#*", 
    "http://emberapp.com/*/images/*", 
    "http://emberapp.com/*/images/*/sizes/*", 
    "http://emberapp.com/*/collections/*/*", 
    "http://emberapp.com/*/categories/*/*/*", 
    "http://embr.it/*", 
    "http://picasaweb.google.com*/*/*#*", 
    "http://picasaweb.google.com*/lh/photo/*", 
    "http://picasaweb.google.com*/*/*", 
    "http://www.whitehouse.gov/photos-and-video/video/*", 
    "http://www.whitehouse.gov/video/*", 
    "http://wh.gov/photos-and-video/video/*", 
    "http://wh.gov/video/*", 
    "http://www.hulu.com/watch*", 
    "http://www.hulu.com/w/*", 
    "http://hulu.com/watch*", 
    "http://hulu.com/w/*", 
    "http://movieclips.com/watch/*/*/", 
    "http://movieclips.com/watch/*/*/*/*", 
    "http://*crackle.com/c/*", 
    "http://www.fancast.com/*/videos", 
    "http://www.funnyordie.com/videos/*", 
    "http://www.vimeo.com/groups/*/videos/*", 
    "http://www.vimeo.com/*", 
    "http://vimeo.com/groups/*/videos/*", 
    "http://vimeo.com/*", 
    "http://www.ted.com/talks/*.html*", 
    "http://www.ted.com/talks/lang/*/*.html*", 
    "http://www.ted.com/index.php/talks/*.html*", 
    "http://www.ted.com/index.php/talks/lang/*/*.html*", 
    "http://*omnisio.com/*", 
    "http://*nfb.ca/film/*", 
    "http://www.thedailyshow.com/watch/*", 
    "http://www.thedailyshow.com/full-episodes/*", 
    "http://www.thedailyshow.com/collection/*/*/*", 
    "http://movies.yahoo.com/movie/*/video/*", 
    "http://movies.yahoo.com/movie/*/info", 
    "http://movies.yahoo.com/movie/*/trailer", 
    "http://www.colbertnation.com/the-colbert-report-collections/*", 
    "http://www.colbertnation.com/full-episodes/*", 
    "http://www.colbertnation.com/the-colbert-report-videos/*", 
    "http://www.comedycentral.com/videos/index.jhtml?*", 
    "http://www.theonion.com/video/*", 
    "http://theonion.com/video/*", 
    "http://wordpress.tv/*/*/*/*/", 
    "http://www.traileraddict.com/trailer/*", 
    "http://www.traileraddict.com/clip/*", 
    "http://www.traileraddict.com/poster/*", 
    "http://www.escapistmagazine.com/videos/*", 
    "http://www.trailerspy.com/trailer/*/*", 
    "http://www.trailerspy.com/trailer/*", 
    "http://www.trailerspy.com/view_video.php*", 
    "http://soundcloud.com/*", 
    "http://soundcloud.com/*/*", 
    "http://soundcloud.com/*/sets/*", 
    "http://soundcloud.com/groups/*", 
    "http://www.lala.com/#album/*", 
    "http://www.lala.com/album/*", 
    "http://www.lala.com/#song/*", 
    "http://www.lala.com/song/*", 
    "http://www.mixcloud.com/*/*/", 
    "http://*amazon.*/gp/product/*", 
    "http://*amazon.*/*/dp/*", 
    "http://*amazon.*/dp/*", 
    "http://*amazon.*/o/ASIN/*", 
    "http://*amazon.*/gp/offer-listing/*", 
    "http://*amazon.*/*/ASIN/*", 
    "http://*amazon.*/gp/product/images/*", 
    "http://www.amzn.com/*", 
    "http://amzn.com/*", 
    "http://twitter.com/*/status/*", 
    "http://twitter.com/*/statuses/*", 
    "http://www.slideshare.net/*/*", 
    "http://*.scribd.com/doc/*", 
    "http://screenr.com/*", 
    "http://www.5min.com/Video/*", 
    "http://www.howcast.com/videos/*", 
    "http://www.screencast.com/*/media/*", 
    "http://screencast.com/*/media/*", 
    "http://www.screencast.com/t/*", 
    "http://screencast.com/t/*", 
    "http://www.clearspring.com/widgets/*", 
    "http://my.opera.com/*/albums/show.dml?id=*", 
    "http://my.opera.com/*/albums/showpic.dml?album=*&picture=*", 
    "http://tumblr.com/*", 
    "http://*.tumblr.com/post/*"
]<commit_msg>added new supported urls from embeddly, there may be provider overlap<commit_after>[
    "http://*youtube.com/watch*",
    "http://*.youtube.com/v/*",
    "http://youtu.be/*",
    "http://*.youtube.com/user/*#*",
    "http://*.youtube.com/*#*/*",
    "http://*justin.tv/*",
    "http://*justin.tv/*/b/*",
    "http://www.ustream.tv/recorded/*",
    "http://www.ustream.tv/channel/*",
    "http://www.ustream.tv/*",
    "http://qik.com/video/*",
    "http://qik.com/*",
    "http://*revision3.com/*",
    "http://*.dailymotion.com/video/*",
    "http://*.dailymotion.com/*/video/*",
    "http://www.collegehumor.com/video:*",
    "http://*twitvid.com/*",
    "http://www.break.com/*/*",
    "http://vids.myspace.com/index.cfm?fuseaction=vids.individual&videoid*",
    "http://www.myspace.com/index.cfm?fuseaction=*&videoid*",
    "http://www.metacafe.com/watch/*",
    "http://blip.tv/file/*",
    "http://*.blip.tv/file/*",
    "http://video.google.com/videoplay?*",
    "http://*revver.com/video/*",
    "http://video.yahoo.com/watch/*/*",
    "http://video.yahoo.com/network/*",
    "http://*viddler.com/explore/*/videos/*",
    "http://liveleak.com/view?*",
    "http://www.liveleak.com/view?*",
    "http://animoto.com/play/*",
    "http://dotsub.com/view/*",
    "http://www.overstream.net/view.php?oid=*",
    "http://www.livestream.com/*",
    "http://www.worldstarhiphop.com/videos/video*.php?v=*",
    "http://worldstarhiphop.com/videos/video*.php?v=*",
    "http://teachertube.com/viewVideo.php*",
    "http://teachertube.com/viewVideo.php*",
    "http://bambuser.com/v/*",
    "http://bambuser.com/channel/*",
    "http://bambuser.com/channel/*/broadcast/*",
    "http://www.schooltube.com/video/*/*",
    "http://*yfrog.*/*",
    "http://tweetphoto.com/*",
    "http://www.flickr.com/photos/*",
    "http://flic.kr/*",
    "http://*twitpic.com/*",
    "http://*imgur.com/*",
    "http://*.posterous.com/*",
    "http://post.ly/*",
    "http://twitgoo.com/*",
    "http://i*.photobucket.com/albums/*",
    "http://gi*.photobucket.com/groups/*",
    "http://phodroid.com/*/*/*",
    "http://www.mobypicture.com/user/*/view/*",
    "http://moby.to/*",
    "http://xkcd.com/*",
    "http://www.xkcd.com/*",
    "http://www.asofterworld.com/index.php?id=*",
    "http://www.qwantz.com/index.php?comic=*",
    "http://23hq.com/*/photo/*",
    "http://www.23hq.com/*/photo/*",
    "http://*dribbble.com/shots/*",
    "http://drbl.in/*",
    "http://*.smugmug.com/*",
    "http://*.smugmug.com/*#*",
    "http://emberapp.com/*/images/*",
    "http://emberapp.com/*/images/*/sizes/*",
    "http://emberapp.com/*/collections/*/*",
    "http://emberapp.com/*/categories/*/*/*",
    "http://embr.it/*",
    "http://picasaweb.google.com*/*/*#*",
    "http://picasaweb.google.com*/lh/photo/*",
    "http://picasaweb.google.com*/*/*",
    "http://dailybooth.com/*/*",
    "http://brizzly.com/pic/*",
    "http://pics.brizzly.com/*.jpg",
    "http://img.ly/*",
    "http://www.facebook.com/photo.php*",
    "http://www.tinypic.com/view.php*",
    "http://tinypic.com/view.php*",
    "http://www.tinypic.com/player.php*",
    "http://tinypic.com/player.php*",
    "http://www.tinypic.com/r/*/*",
    "http://tinypic.com/r/*/*",
    "http://*.tinypic.com/*.jpg",
    "http://*.tinypic.com/*.png",
    "http://meadd.com/*/*",
    "http://meadd.com/*",
    "http://*.deviantart.com/art/*",
    "http://*.deviantart.com/gallery/*",
    "http://*.deviantart.com/#/*",
    "http://fav.me/*",
    "http://*.deviantart.com",
    "http://*.deviantart.com/gallery",
    "http://*.deviantart.com/*/*.jpg",
    "http://*.deviantart.com/*/*.gif",
    "http://*.deviantart.net/*/*.jpg",
    "http://*.deviantart.net/*/*.gif",
    "http://www.whitehouse.gov/photos-and-video/video/*",
    "http://www.whitehouse.gov/video/*",
    "http://wh.gov/photos-and-video/video/*",
    "http://wh.gov/video/*",
    "http://www.hulu.com/watch*",
    "http://www.hulu.com/w/*",
    "http://hulu.com/watch*",
    "http://hulu.com/w/*",
    "http://movieclips.com/watch/*/*/",
    "http://movieclips.com/watch/*/*/*/*",
    "http://*crackle.com/c/*",
    "http://www.fancast.com/*/videos",
    "http://www.funnyordie.com/videos/*",
    "http://www.vimeo.com/groups/*/videos/*",
    "http://www.vimeo.com/*",
    "http://vimeo.com/groups/*/videos/*",
    "http://vimeo.com/*",
    "http://www.ted.com/talks/*.html*",
    "http://www.ted.com/talks/lang/*/*.html*",
    "http://www.ted.com/index.php/talks/*.html*",
    "http://www.ted.com/index.php/talks/lang/*/*.html*",
    "http://*omnisio.com/*",
    "http://*nfb.ca/film/*",
    "http://www.thedailyshow.com/watch/*",
    "http://www.thedailyshow.com/full-episodes/*",
    "http://www.thedailyshow.com/collection/*/*/*",
    "http://movies.yahoo.com/movie/*/video/*",
    "http://movies.yahoo.com/movie/*/info",
    "http://movies.yahoo.com/movie/*/trailer",
    "http://www.colbertnation.com/the-colbert-report-collections/*",
    "http://www.colbertnation.com/full-episodes/*",
    "http://www.colbertnation.com/the-colbert-report-videos/*",
    "http://www.comedycentral.com/videos/index.jhtml?*",
    "http://www.theonion.com/video/*",
    "http://theonion.com/video/*",
    "http://wordpress.tv/*/*/*/*/",
    "http://www.traileraddict.com/trailer/*",
    "http://www.traileraddict.com/clip/*",
    "http://www.traileraddict.com/poster/*",
    "http://www.escapistmagazine.com/videos/*",
    "http://www.trailerspy.com/trailer/*/*",
    "http://www.trailerspy.com/trailer/*",
    "http://www.trailerspy.com/view_video.php*",
    "http://www.atom.com/*/*/",
    "http://fora.tv/*/*/*/*",
    "http://www.spike.com/video/*",
    "http://www.gametrailers.com/video/*",
    "http://gametrailers.com/video/*",
    "http://www.koldcast.tv/video/*",
    "http://www.koldcast.tv/#video:*",
    "http://techcrunch.tv/watch*",
    "http://techcrunch.tv/*/watch*",
    "http://www.godtube.com/featured/video/*",
    "http://www.tangle.com/view_video*",
    "http://soundcloud.com/*",
    "http://soundcloud.com/*/*",
    "http://soundcloud.com/*/sets/*",
    "http://soundcloud.com/groups/*",
    "http://www.last.fm/music/*",
    "http://www.last.fm/music/+videos/*",
    "http://www.last.fm/music/+images/*",
    "http://www.last.fm/music/*/_/*",
    "http://www.last.fm/music/*/*",
    "http://www.mixcloud.com/*/*/",
    "http://espn.go.com/video/clip*",
    "http://espn.go.com/*/story*",
    "http://cnbc.com/id/*",
    "http://cbsnews.com/video/watch/*",
    "http://www.cnn.com/video/*",
    "http://edition.cnn.com/video/*",
    "http://money.cnn.com/video/*",
    "http://today.msnbc.msn.com/id/*/vp/*",
    "http://www.msnbc.msn.com/id/*/vp/*",
    "http://www.msnbc.msn.com/id/*/ns/*",
    "http://today.msnbc.msn.com/id/*/ns/*",
    "http://multimedia.foxsports.com/m/video/*/*",
    "http://msn.foxsports.com/video*",
    "http://*amazon.*/gp/product/*",
    "http://*amazon.*/*/dp/*",
    "http://*amazon.*/dp/*",
    "http://*amazon.*/o/ASIN/*",
    "http://*amazon.*/gp/offer-listing/*",
    "http://*amazon.*/*/ASIN/*",
    "http://*amazon.*/gp/product/images/*",
    "http://www.amzn.com/*",
    "http://amzn.com/*",
    "http://www.shopstyle.com/browse*",
    "http://www.shopstyle.com/action/apiVisitRetailer*",
    "http://www.shopstyle.com/action/viewLook*",
    "http://gist.github.com/*",
    "http://twitter.com/*/status/*",
    "http://twitter.com/*/statuses/*",
    "http://www.crunchbase.com/*/*",
    "http://crunchbase.com/*/*",
    "http://www.slideshare.net/*/*",
    "http://*.scribd.com/doc/*",
    "http://screenr.com/*",
    "http://polldaddy.com/community/poll/*",
    "http://polldaddy.com/poll/*",
    "http://answers.polldaddy.com/poll/*",
    "http://www.5min.com/Video/*",
    "http://www.howcast.com/videos/*",
    "http://www.screencast.com/*/media/*",
    "http://screencast.com/*/media/*",
    "http://www.screencast.com/t/*",
    "http://screencast.com/t/*",
    "http://issuu.com/*/docs/*",
    "http://www.kickstarter.com/projects/*/*",
    "http://www.scrapblog.com/viewer/viewer.aspx*",
    "http://my.opera.com/*/albums/show.dml?id=*",
    "http://my.opera.com/*/albums/showpic.dml?album=*&picture=*",
    "http://tumblr.com/*",
    "http://*.tumblr.com/post/*",
    "http://www.polleverywhere.com/polls/*",
    "http://www.polleverywhere.com/multiple_choice_polls/*",
    "http://www.polleverywhere.com/free_text_polls/*",
    "http://www.quantcast.com/wd:*",
    "http://www.quantcast.com/*",
    "http://siteanalytics.compete.com/*",
    "http://statsheet.com/statplot/charts/*/*/*/*",
    "http://statsheet.com/statplot/charts/e/*",
    "http://statsheet.com/*/teams/*/*",
    "http://statsheet.com/tools/chartlets?chart=*",
    "http://*.status.net/notice/*",
    "http://identi.ca/notice/*",
    "http://shitmydadsays.com/notice/*"
]
{'fraction_non_alphanumeric': 0.3010079944386514, 'fraction_numerical': 0.0008342022940563086, 'mean_word_length': 2.462334536702768, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 136, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3281652', 'n_tokens_mistral': 6194, 'n_tokens_neox': 5874, 'n_words': 378}