<commit_before>import markdown
import logging
import traceback
import urlparse
import re
import os.path
import glob
import urllib2
import itertools
import simplejson
import twitter
import platform

from hashlib import sha1

from django.core import mail
from django.conf import settings

from zephyr.lib.avatar  import gravatar_hash
from zephyr.lib.bugdown import codehilite, fenced_code
from zephyr.lib.bugdown.fenced_code import FENCE_RE
from zephyr.lib.timeout import timeout, TimeoutExpired
from zephyr.lib.cache import cache_with_key, cache_get_many, cache_set_many
from embedly import Embedly

embedly_client = Embedly(settings.EMBEDLY_KEY, timeout=2.5)

# Format version of the bugdown rendering; stored along with rendered
# messages so that we can efficiently determine what needs to be re-rendered
version = 1

def list_of_tlds():
    # HACK we manually blacklist .py
    blacklist = ['PY\n', ]

    # tlds-alpha-by-domain.txt comes from http://data.iana.org/TLD/tlds-alpha-by-domain.txt
    tlds_file = os.path.join(os.path.dirname(__file__), 'tlds-alpha-by-domain.txt')
    tlds = [tld.lower().strip() for tld in open(tlds_file, 'r')
                if not tld in blacklist and not tld[0].startswith('#')]
    tlds.sort(key=len, reverse=True)
    return tlds

def walk_tree(root, processor, stop_after_first=False):
    results = []
    stack = [root]

    while stack:
        currElement = stack.pop()
        for child in currElement.getchildren():
            if child.getchildren():
                stack.append(child)

            result = processor(child)
            if result is not None:
                results.append(result)
                if stop_after_first:
                    return results

    return results

def add_a(root, url, link, height=None):
    div = markdown.util.etree.SubElement(root, "div")
    div.set("class", "message_inline_image");
    a = markdown.util.etree.SubElement(div, "a")
    a.set("href", link)
    a.set("target", "_blank")
    a.set("title", link)
    img = markdown.util.etree.SubElement(a, "img")
    img.set("src", url)


def mygrouper(n, iterable):
    # Adapted from http://stackoverflow.com/a/1625013/90777
    args = [iter(iterable)] * n
    return ([e for e in t if e != None] for t in itertools.izip_longest(*args))

def hash_embedly_url(link):
    return 'embedly:' + sha1(link).hexdigest()

class EmbedlyProcessor(markdown.treeprocessors.Treeprocessor):
    def run(self, root):
        # Get all URLs from the blob
        found_urls = walk_tree(root, lambda e: e.get("href") if e.tag == "a" else None)

        supported_urls = []
        for link in found_urls:
            # Don't waste our quota with unsupported links or links otherwise
            # handled by our Twitter integration
            if not embedly_client.is_supported(link) or get_tweet_id(link):
                continue
            supported_urls.append(link)

        if not supported_urls:
            return root

        # We want this to be able to easily reverse the hashing later
        keys_to_links = dict((hash_embedly_url(link), link) for link in supported_urls)
        cache_hits = cache_get_many(keys_to_links.keys(), cache_name="database")

        # Construct a dict of url => oembed_data pairs
        oembeds = dict((keys_to_links[key], cache_hits[key]) for key in cache_hits)

        to_process = [url for url in supported_urls if not url in oembeds]
        to_cache = {}

        for links in mygrouper(10, to_process):
            try:
                responses = embedly_client.oembed(links, maxwidth=250)
            except:
                # We put this in its own try-except because it requires external
                # connectivity. If embedly flakes out, we don't want to not-render
                # the entire message; we just want to not show the embedly preview.
                logging.warning(traceback.format_exc())<commit_msg>Log URLs that cause Embedly timeouts.<commit_after>import markdown
import logging
import traceback
import urlparse
import re
import os.path
import glob
import urllib2
import itertools
import simplejson
import twitter
import platform

import httplib2

from hashlib import sha1

from django.core import mail
from django.conf import settings

from zephyr.lib.avatar  import gravatar_hash
from zephyr.lib.bugdown import codehilite, fenced_code
from zephyr.lib.bugdown.fenced_code import FENCE_RE
from zephyr.lib.timeout import timeout, TimeoutExpired
from zephyr.lib.cache import cache_with_key, cache_get_many, cache_set_many
from embedly import Embedly

embedly_client = Embedly(settings.EMBEDLY_KEY, timeout=2.5)

# Format version of the bugdown rendering; stored along with rendered
# messages so that we can efficiently determine what needs to be re-rendered
version = 1

def list_of_tlds():
    # HACK we manually blacklist .py
    blacklist = ['PY\n', ]

    # tlds-alpha-by-domain.txt comes from http://data.iana.org/TLD/tlds-alpha-by-domain.txt
    tlds_file = os.path.join(os.path.dirname(__file__), 'tlds-alpha-by-domain.txt')
    tlds = [tld.lower().strip() for tld in open(tlds_file, 'r')
                if not tld in blacklist and not tld[0].startswith('#')]
    tlds.sort(key=len, reverse=True)
    return tlds

def walk_tree(root, processor, stop_after_first=False):
    results = []
    stack = [root]

    while stack:
        currElement = stack.pop()
        for child in currElement.getchildren():
            if child.getchildren():
                stack.append(child)

            result = processor(child)
            if result is not None:
                results.append(result)
                if stop_after_first:
                    return results

    return results

def add_a(root, url, link, height=None):
    div = markdown.util.etree.SubElement(root, "div")
    div.set("class", "message_inline_image");
    a = markdown.util.etree.SubElement(div, "a")
    a.set("href", link)
    a.set("target", "_blank")
    a.set("title", link)
    img = markdown.util.etree.SubElement(a, "img")
    img.set("src", url)


def mygrouper(n, iterable):
    # Adapted from http://stackoverflow.com/a/1625013/90777
    args = [iter(iterable)] * n
    return ([e for e in t if e != None] for t in itertools.izip_longest(*args))

def hash_embedly_url(link):
    return 'embedly:' + sha1(link).hexdigest()

class EmbedlyProcessor(markdown.treeprocessors.Treeprocessor):
    def run(self, root):
        # Get all URLs from the blob
        found_urls = walk_tree(root, lambda e: e.get("href") if e.tag == "a" else None)

        supported_urls = []
        for link in found_urls:
            # Don't waste our quota with unsupported links or links otherwise
            # handled by our Twitter integration
            if not embedly_client.is_supported(link) or get_tweet_id(link):
                continue
            supported_urls.append(link)

        if not supported_urls:
            return root

        # We want this to be able to easily reverse the hashing later
        keys_to_links = dict((hash_embedly_url(link), link) for link in supported_urls)
        cache_hits = cache_get_many(keys_to_links.keys(), cache_name="database")

        # Construct a dict of url => oembed_data pairs
        oembeds = dict((keys_to_links[key], cache_hits[key]) for key in cache_hits)

        to_process = [url for url in supported_urls if not url in oembeds]
        to_cache = {}

        for links in mygrouper(10, to_process):
            try:
                responses = embedly_client.oembed(links, maxwidth=250)
            except httplib2.socket.timeout:
                # We put this in its own try-except because it requires external
                # connectivity. If embedly flakes out, we don't want to not-render
                # the entire message; we just want to not show the embedly preview.
                logging.warning("Embedly Embed timeout for URLs: %s" % (" ".join(links)))
                logging.warning(traceback.format_exc())
                return root
            except Exception:
                # If things break for any other reason, don't make things sad.
                logging.warning(traceback.format_exc())
{'fraction_non_alphanumeric': 0.09116495349975526, 'fraction_numerical': 0.006118453255017132, 'mean_word_length': 2.7802960222016653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4645143', 'n_tokens_mistral': 2555, 'n_tokens_neox': 2475, 'n_words': 738}