<commit_before>module Oneboxer

  module Whitelist
    def self.entries
      [
       Entry.new(/^https?:\/\/(?:www\.)?cnn\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?washingtonpost\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?funnyordie\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?youtube\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?youtu\.be\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?500px\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?scribd\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?photobucket\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ebay\.(com|ca|co\.uk)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?nytimes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?pinterest\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?imdb\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bbc\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ask\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?huffingtonpost\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?aol\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?espn\.go\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?about\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?cnet\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ehow\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?dailymail\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?indiatimes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?answers\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?instagr\.am\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?battle\.net\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?sourceforge\.net\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?myspace\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wikia\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?etsy\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?walmart\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?reference\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?yelp\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?foxnews\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?guardian\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?digg\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?squidoo\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wsj\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?archive\.org\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?nba\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?samsung\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?mashable\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?forbes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thefreedictionary\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?groupon\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ikea\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?dell\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?mlb\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bestbuy\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bloomberg\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ign\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?twitpic\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?techcrunch\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?usatoday\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?go\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?businessinsider\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?zillow\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?tmz\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thesun\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thestar\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?theglobeandmail\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?torontosun\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?kickstarter\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wired\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?time\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?npr\.org\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?cracked\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?deadline\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thinkgeek\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?theonion\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?tumblr\.com\/.+/, false),
       Entry.new(/\/\d{4}\/\d{2}\/\d{2}\//, false),   # wordpress
      ]
    end

    def self.entry_for_url(url)
      entries.each {|e| return e if e.matches?(url) }
      nil
    end<commit_msg>add HTG to the onebox opengraph whitelist<commit_after>module Oneboxer

  module Whitelist
    def self.entries
      [
       Entry.new(/^https?:\/\/(?:www\.)?cnn\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?washingtonpost\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?funnyordie\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?youtube\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?youtu\.be\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?500px\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?scribd\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?photobucket\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ebay\.(com|ca|co\.uk)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?nytimes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?pinterest\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?imdb\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bbc\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ask\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?huffingtonpost\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?aol\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?espn\.go\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?about\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?cnet\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ehow\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?dailymail\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?indiatimes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?answers\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?instagr\.am\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?battle\.net\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?sourceforge\.net\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?myspace\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wikia\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?etsy\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?walmart\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?reference\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?yelp\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?foxnews\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?guardian\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?digg\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?squidoo\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wsj\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?archive\.org\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?nba\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?samsung\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?mashable\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?forbes\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thefreedictionary\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?groupon\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ikea\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?dell\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?mlb\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bestbuy\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?bloomberg\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?ign\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?twitpic\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?techcrunch\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?usatoday\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?go\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?businessinsider\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?zillow\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?tmz\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thesun\.co\.uk\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thestar\.(com|ca)\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?theglobeandmail\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?torontosun\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?kickstarter\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?wired\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?time\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?npr\.org\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?cracked\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?deadline\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?thinkgeek\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?theonion\.com\/.+/),
       Entry.new(/^https?:\/\/(?:www\.)?tumblr\.com\/.+/, false),
       Entry.new(/^https?:\/\/(?:www\.)?howtogeek\.com\/.+/, false),
       Entry.new(/\/\d{4}\/\d{2}\/\d{2}\//, false),   # wordpress
      ]
    end

    def self.entry_for_url(url)
      entries.each {|e| return e if e.matches?(url) }
      nil
    end
{'fraction_non_alphanumeric': 0.41726066454655114, 'fraction_numerical': 0.0013159337646671784, 'mean_word_length': 1.743682310469314, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 141, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6580388', 'n_tokens_mistral': 4150, 'n_tokens_neox': 3844, 'n_words': 189}