<commit_before>from httplib2 import Http
from BeautifulSoup import BeautifulSoup
from HTMLParser import HTMLParseError

BEACON_REQUEST_METHOD = "HEAD"
BEACON_MIN_IMAGE_SIZE = 100
BEACON_VERIFY_ZERO_BYTES = True

_known_beacons = set()
_known_images = set()


class BeaconDetector(object):<commit_msg>Add some use case documentation.<commit_after>from httplib2 import Http
from BeautifulSoup import BeautifulSoup
from HTMLParser import HTMLParseError

BEACON_REQUEST_METHOD = "HEAD"
BEACON_MIN_IMAGE_SIZE = 100
BEACON_VERIFY_ZERO_BYTES = True

_known_beacons = set()
_known_images = set()

"""
Tracker Use Case
================

I tried all the default feeds in the overlay plus my personnal feeds
and I identified those tools:

    * Feedburner toolbar -- 4 toolbar images, 1 tracker image.
    * Pheedcontent.com toolbar -- 4 toolbar images, 1 advertisement image.
    * Digg/Reddit generic toolbar - 3 toolbar, no tracker image.
    * http://res.feedsportal.com/ -- 2 toolbar images, 1 tracker image.
    * http://a.rfihub.com/ -- associated with http://rocketfuelinc.com/,
        used for ads or tracking. Not quite sure.

feedburner toolbar and tracker
-------------------------------

  * WULFMORGENSTALLER
  * MarketWatch.com - Top Stories
  * Hollywood.com - Recent News
  * Wired: entertainement
  * Livescience.com
  * Reader Digest

Pheedcontent.com toolbar
--------------------------

  * Sports News : CBSSports.com


Digg/Reddit toolbar
-------------------

  * Abstruse goose

http://res.feedsportal.com/
------------------

  * New scientist.com


"""



class BeaconDetector(object):
{'fraction_non_alphanumeric': 0.13875, 'fraction_numerical': 0.009375, 'mean_word_length': 3.0946291560102304, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6315065', 'n_tokens_mistral': 520, 'n_tokens_neox': 475, 'n_words': 143}