<commit_before>package org.lantern;

import java.util.Collection;
import java.util.HashSet;
import java.util.TreeSet;

import org.apache.commons.lang.StringUtils;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Keeps track of which domains are whitelisted.
 */
public class Whitelist {

    private final Logger log = LoggerFactory.getLogger(Whitelist.class);
    
    private final Collection<String> requiredEntries = new HashSet<String>();
    
    public static final String[] SITES = {
        // optional
        "avaaz.org",
        "bittorrent.com",
        "bloglines.com",
        "blogspot.com",
        "bloomberg.com",
        "box.com",
        "box.net",
        "change.org",
        "dailymotion.com",
        "docstoc.com",
        "dropbox.com",
        "eff.org",
        "facebook.com",
        "flickr.com",
        "friendfeed.com",
        "freedomhouse.org",
        "hrw.org", // Human Rights Watch
        "ifconfig.me",
        "igfw.net",
        "linkedin.com",
        "littleshoot.org",
        "livejournal.com",
        "myspace.com",
        "nytimes.com",
        "orkut.com",
        "paypal.com",
        "plurk.com",
        "posterous.com",
        "reddit.com",
        "rsf.org",
        "scribd.com",
        "stumbleupon.com",
        "torproject.org",
        "tumblr.com",
        "twitter.com",
        "vimeo.com",
        "whatismyip.com",
        "wikileaks.org",
        "wordpress.org",
        "wordpress.com",
        "youtube.com",
        
        // Iran-focused sites
        "30mail.net",
        "advar-news.biz",
        "balatarin.com",
        "bbc.co.uk",
        "bia2.com",
        "enghelabe-eslami.com",
        "gooya.com",
        "irangreenvoice.com",
        "iranian.com",
        "mardomak.org",
        "radiofarda.com",
        "radiozamaneh.com",
        "Roozonline.com",
        "voanews.com",
        
        
        // China (with various sub-categories below)
        "appledaily.com.tw",
        "boxun.com",
        "fc2.com",
        "hk.nextmedia.com",
        "inmediahk.net",
        "pchome.com.tw",
        "blog.idv.tw",
        "pixnet.net",
        "roodo.com",
        "wretch.cc",

        // Forums
        "canadameet.me",
        "chinasmile.net",
        "discuss.com.hk",
        //"dolc.de",
        "oursteps.com.au",
        "qoos.com",
        "sgchinese.net",
        "student.tw",
        "twbbs.tw",
        "uwants.com",
        

        // Cloud Storage (often porn, heavy load, so ignored for now).
        //https://www.rapidshare.com
        //http://www.4shared.com
        //https://www.sugarsync.com

        // News and Political
        "881903.com",
        "aboluowang.com",
        "www.am730.com.hk",
        "boxun.com",
        "bullogger.com",
        "canyu.org",
        "chinadigitaltimes.net",
        "chinainperspective.com",
        "dw.de",
        "epochtimes.com",
        "etaiwannews.com",
        "hrichina.org", 
        "globalvoicesonline.org",
        "libertytimes.com.tw",
        "mingpao.com",
        "molihua.org",
        "newcenturynews.com",
        "nextmedia.com",
        "ntdtv.com",
        "rfa.org",
        "rfi.fr",
        "rthk.hk",
        "singtao.com",
        "taiwandaily.net",
        "on.cc",
        "yzzk.com",
    };
    
    private Collection<WhitelistEntry> defaultWhitelist = 
        new HashSet<WhitelistEntry>();
    
    private Collection<WhitelistEntry> whitelist = 
        new TreeSet<WhitelistEntry>();
    
    {
        // these domains host required services and can't be removed
        addDefaultEntry("getlantern.org", true);
        addDefaultEntry("google.com", true);
        addDefaultEntry("exceptional.io", true);
        for (final String site : SITES) {
            addDefaultEntry(site);
        }
    }
    
    public boolean isWhitelisted(final String uri,
        final Collection<WhitelistEntry> wl) {
        final String toMatch = toBaseUri(uri);
        return wl.contains(new WhitelistEntry(toMatch));
    }
    
    /**
     * Decides whether or not the specified full URI matches domains for our
     * whitelist.
     * 
     * @return <code>true</code> if the specified domain matches domains for
     * our whitelist, otherwise <code>false</code>.
     */
    public boolean isWhitelisted(final String uri) {
        return isWhitelisted(uri, whitelist);
    }
    
    /**
     * Decides whether or not the specified HttpRequest is for a domain on
     * our whitelist. Note this also checks the Referer header.
     * 
     * @return <code>true</code> if the specified domain matches domains for
     * our whitelist, otherwise <code>false</code>.
     */
    public boolean isWhitelisted(final HttpRequest request) {
        log.debug("Checking whitelist for request");
        final String uri = request.getUri();
        log.debug("URI is: {}", uri);

        final String referer = request.getHeader("referer");
        
        final String uriToCheck;
        log.debug("Referer: "+referer);
        if (!StringUtils.isBlank(referer)) {
            uriToCheck = referer;
        } else {
            uriToCheck = uri;
        }

        return isWhitelisted(uriToCheck);
    }
    
    private void addDefaultEntry(final String entry) {
        addDefaultEntry(entry, false);
    }
    
    private void addDefaultEntry(final String entry, final boolean required) {
        final WhitelistEntry we = new WhitelistEntry(entry, required, true);
        whitelist.add(we);
        defaultWhitelist.add(we);
        if (required) {
            this.requiredEntries.add(entry);
        }
    }
    
    public void addEntry(final String entry) {
        whitelist.add(new WhitelistEntry(entry));
    }

    public void removeEntry(final String entry) {
        if (!this.requiredEntries.contains(entry)) {
            whitelist.remove(new WhitelistEntry(entry));
        }
    }
    
    public Collection<WhitelistEntry> getEntries() {
        return whitelist;<commit_msg>Fixed check for referer to always include both<commit_after>package org.lantern;

import java.util.Collection;
import java.util.HashSet;
import java.util.TreeSet;

import org.apache.commons.lang.StringUtils;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.jboss.netty.handler.codec.http.HttpRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Keeps track of which domains are whitelisted.
 */
public class Whitelist {

    private final Logger log = LoggerFactory.getLogger(Whitelist.class);
    
    private final Collection<String> requiredEntries = new HashSet<String>();
    
    public static final String[] SITES = {
        // optional
        "avaaz.org",
        "bittorrent.com",
        "bloglines.com",
        "blogspot.com",
        "bloomberg.com",
        "box.com",
        "box.net",
        "change.org",
        "dailymotion.com",
        "docstoc.com",
        "dropbox.com",
        "eff.org",
        "facebook.com",
        "flickr.com",
        "friendfeed.com",
        "freedomhouse.org",
        "hrw.org", // Human Rights Watch
        "ifconfig.me",
        "igfw.net",
        "linkedin.com",
        "littleshoot.org",
        "livejournal.com",
        "myspace.com",
        "nytimes.com",
        "orkut.com",
        "paypal.com",
        "plurk.com",
        "posterous.com",
        "reddit.com",
        "rsf.org",
        "scribd.com",
        "stumbleupon.com",
        "torproject.org",
        "tumblr.com",
        "twitter.com",
        "vimeo.com",
        "whatismyip.com",
        "wikileaks.org",
        "wordpress.org",
        "wordpress.com",
        "youtube.com",
        
        // Iran-focused sites
        "30mail.net",
        "advar-news.biz",
        "balatarin.com",
        "bbc.co.uk",
        "bia2.com",
        "enghelabe-eslami.com",
        "gooya.com",
        "irangreenvoice.com",
        "iranian.com",
        "mardomak.org",
        "radiofarda.com",
        "radiozamaneh.com",
        "Roozonline.com",
        "voanews.com",
        
        
        // China (with various sub-categories below)
        "appledaily.com.tw",
        "boxun.com",
        "fc2.com",
        "hk.nextmedia.com",
        "inmediahk.net",
        "pchome.com.tw",
        "blog.idv.tw",
        "pixnet.net",
        "roodo.com",
        "wretch.cc",

        // Forums
        "canadameet.me",
        "chinasmile.net",
        "discuss.com.hk",
        //"dolc.de",
        "oursteps.com.au",
        "qoos.com",
        "sgchinese.net",
        "student.tw",
        "twbbs.tw",
        "uwants.com",
        

        // Cloud Storage (often porn, heavy load, so ignored for now).
        //https://www.rapidshare.com
        //http://www.4shared.com
        //https://www.sugarsync.com

        // News and Political
        "881903.com",
        "aboluowang.com",
        "www.am730.com.hk",
        "boxun.com",
        "bullogger.com",
        "canyu.org",
        "chinadigitaltimes.net",
        "chinainperspective.com",
        "dw.de",
        "epochtimes.com",
        "etaiwannews.com",
        "hrichina.org", 
        "globalvoicesonline.org",
        "libertytimes.com.tw",
        "mingpao.com",
        "molihua.org",
        "newcenturynews.com",
        "nextmedia.com",
        "ntdtv.com",
        "rfa.org",
        "rfi.fr",
        "rthk.hk",
        "singtao.com",
        "taiwandaily.net",
        "on.cc",
        "yzzk.com",
    };
    
    private Collection<WhitelistEntry> defaultWhitelist = 
        new HashSet<WhitelistEntry>();
    
    private Collection<WhitelistEntry> whitelist = 
        new TreeSet<WhitelistEntry>();
    
    {
        // these domains host required services and can't be removed
        addDefaultEntry("getlantern.org", true);
        addDefaultEntry("google.com", true);
        addDefaultEntry("exceptional.io", true);
        for (final String site : SITES) {
            addDefaultEntry(site);
        }
    }
    
    public boolean isWhitelisted(final String uri,
        final Collection<WhitelistEntry> wl) {
        if (StringUtils.isBlank(uri)) {
            return false;
        }
        final String toMatch = toBaseUri(uri);
        return wl.contains(new WhitelistEntry(toMatch));
    }
    
    /**
     * Decides whether or not the specified full URI matches domains for our
     * whitelist.
     * 
     * @return <code>true</code> if the specified domain matches domains for
     * our whitelist, otherwise <code>false</code>.
     */
    public boolean isWhitelisted(final String uri) {
        return isWhitelisted(uri, whitelist);
    }
    
    /**
     * Decides whether or not the specified HttpRequest is for a domain on
     * our whitelist. Note this also checks the Referer header.
     * 
     * @return <code>true</code> if the specified domain matches domains for
     * our whitelist, otherwise <code>false</code>.
     */
    public boolean isWhitelisted(final HttpRequest request) {
        log.debug("Checking whitelist for request");
        final String uri = request.getUri();
        log.debug("URI is: {}", uri);
        final String referer = request.getHeader(HttpHeaders.Names.REFERER);
        return isWhitelisted(referer) || isWhitelisted(uri);
    }
    
    private void addDefaultEntry(final String entry) {
        addDefaultEntry(entry, false);
    }
    
    private void addDefaultEntry(final String entry, final boolean required) {
        final WhitelistEntry we = new WhitelistEntry(entry, required, true);
        whitelist.add(we);
        defaultWhitelist.add(we);
        if (required) {
            this.requiredEntries.add(entry);
        }
    }
    
    public void addEntry(final String entry) {
        whitelist.add(new WhitelistEntry(entry));
    }

    public void removeEntry(final String entry) {
        if (!this.requiredEntries.contains(entry)) {
            whitelist.remove(new WhitelistEntry(entry));
        }
    }
    
    public Collection<WhitelistEntry> getEntries() {
        return whitelist;
{'fraction_non_alphanumeric': 0.1276736311834173, 'fraction_numerical': 0.0026426624824510696, 'mean_word_length': 1.8819609709662066, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 4, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4541873', 'n_tokens_mistral': 3898, 'n_tokens_neox': 3777, 'n_words': 735}