<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""
Print a markdown-formatted bullet point with a link to the URL in your
clipboard.
"""
from HTMLParser import HTMLParser
from glob import glob
from itertools import chain
from os import path
from subprocess import Popen, PIPE
from urllib2 import urlopen, URLError
from urlparse import urlparse
import json

from BeautifulSoup import BeautifulSoup


def main():
    """Returns the .md formatted bullet point for the URL in the clipboard."""
    url = get_url_from_clipboard()
    site = urlparse(url).netloc.replace('www.', '')
    title = get_title(url)
    title = transform_title(title, site)
    return u'  * [{title} [{site}]]({url})'.format(
        title=title, site=site, url=url)


def get_url_from_clipboard():
    """Returns the contents of your clipboard, hopefully a url."""
    process = Popen(['xclip', '-out'], stdout=PIPE)
    clipboard_contents = process.communicate()[0]
    url = clipboard_contents.strip()
    return url


def get_title(url):
    """Get the title of the page matching the `url`."""
    title = get_title_from_firefox_session(url)
    if not title:
        title = get_title_from_internet(url)
    return title


def transform_title(title, site):
    """Miscellaneous title transformations.

    Handle some unicode, unescape HTML, simplify hierarchical titles, ...

    """
    title = HTMLParser().unescape(title)
    substitutions = {
        u'\u2013': '-', u'\u2014': '-', u'\u2019': "'",  u'\xb7': '-'}
    for letter, replacement in substitutions.iteritems():
        title = title.replace(letter, replacement)
    title = ''.join([x if ord(x) < 128 else '?' for x in title])
    title = parse_fancy_titles(title, site)
    return title


def get_title_from_firefox_session(url):
    """Get the title for the `url` from Firefox, if it's recently opened."""
    def get_firefox_data():
        try:
            session = get_session_file()
            with open(session, 'r') as json_session_file:
                firefox_data = json.loads(json_session_file.read())
        except IOError:
            return ''
        return firefox_data

    def walk(structure, criteria=lambda x: True):
        """Yield each substructure in the nested `structure`.

        Only yields things that match the `criteria` passed in.

        """
        if criteria(structure):
            yield structure
        if isinstance(structure, dict):
            for value in structure.itervalues():
                for structure in walk(value, criteria=criteria):
                    yield structure
        elif isinstance(structure, list) or isinstance(structure, tuple):
            for item in structure:
                for structure in walk(item, criteria=criteria):
                    yield structure

    def search(structure, criteria):
        """Search the `structure` for a substruct that matches `criteria`."""
        for match in walk(structure, criteria):
            return match
        return {}

    def has_matching_url(element):
        return isinstance(element, dict) and element.get('url') == url

    firefox_data = get_firefox_data()
    matching_tab = search(firefox_data, has_matching_url)
    return matching_tab.get('title')


def get_session_file():
    """Get the path of the place where firefox stores its session info."""
    home = path.expanduser('~')
    glob_path = path.join(home, '.mozilla/firefox/*.default/sessionstore.js')
    session_file = glob(glob_path)
    return session_file[0]


def get_title_from_internet(url):
    """Get the title of the `url` from the internet."""
    try:
        html = urlopen(url).read()
        soup = BeautifulSoup(html)
        title = soup.find('title').text
    except ValueError:
        print u'Bad URL: ' + repr(url)
        title = ''
    except URLError:
        title = ''

    return title
<commit_msg>Handle images / PDFs / ... better<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-
"""
Print a markdown-formatted bullet point with a link to the URL in your
clipboard.
"""
from HTMLParser import HTMLParser
from glob import glob
from itertools import chain
from os import path
from subprocess import Popen, PIPE
from urllib2 import urlopen, URLError
from urlparse import urlparse
import json

from BeautifulSoup import BeautifulSoup


def main():
    """Returns the .md formatted bullet point for the URL in the clipboard."""
    url = get_url_from_clipboard()
    site = urlparse(url).netloc.replace('www.', '')
    title = get_title(url)
    title = transform_title(title, site)
    return u'  * [{title} [{site}]]({url})'.format(
        title=title, site=site, url=url)


def get_url_from_clipboard():
    """Returns the contents of your clipboard, hopefully a url."""
    process = Popen(['xclip', '-out'], stdout=PIPE)
    clipboard_contents = process.communicate()[0]
    url = clipboard_contents.strip()
    return url


def get_title(url):
    """Get the title of the page matching the `url`."""
    title = get_title_from_firefox_session(url)
    if not title:
        title = get_title_from_internet(url)
    return title


def transform_title(title, site):
    """Miscellaneous title transformations.

    Handle some unicode, unescape HTML, simplify hierarchical titles, ...

    """
    title = HTMLParser().unescape(title)
    substitutions = {
        u'\u2013': '-', u'\u2014': '-', u'\u2019': "'",  u'\xb7': '-'}
    for letter, replacement in substitutions.iteritems():
        title = title.replace(letter, replacement)
    title = ''.join([x if ord(x) < 128 else '?' for x in title])
    title = parse_fancy_titles(title, site)
    return title


def get_title_from_firefox_session(url):
    """Get the title for the `url` from Firefox, if it's recently opened."""
    def get_firefox_data():
        try:
            session = get_session_file()
            with open(session, 'r') as json_session_file:
                firefox_data = json.loads(json_session_file.read())
        except IOError:
            return ''
        return firefox_data

    def walk(structure, criteria=lambda x: True):
        """Yield each substructure in the nested `structure`.

        Only yields things that match the `criteria` passed in.

        """
        if criteria(structure):
            yield structure
        if isinstance(structure, dict):
            for value in structure.itervalues():
                for structure in walk(value, criteria=criteria):
                    yield structure
        elif isinstance(structure, list) or isinstance(structure, tuple):
            for item in structure:
                for structure in walk(item, criteria=criteria):
                    yield structure

    def search(structure, criteria):
        """Search the `structure` for a substruct that matches `criteria`."""
        for match in walk(structure, criteria):
            return match
        return {}

    def has_matching_url(element):
        return isinstance(element, dict) and element.get('url') == url

    firefox_data = get_firefox_data()
    matching_tab = search(firefox_data, has_matching_url)
    return matching_tab.get('title')


def get_session_file():
    """Get the path of the place where firefox stores its session info."""
    home = path.expanduser('~')
    glob_path = path.join(home, '.mozilla/firefox/*.default/sessionstore.js')
    session_file = glob(glob_path)
    return session_file[0]


def get_title_from_internet(url):
    """Get the title of the `url` from the internet."""
    try:
        html = urlopen(url).read()
        soup = BeautifulSoup(html)
        title = soup.find('title').text
    except ValueError:
        print u'Bad URL: ' + repr(url)
        title = ''
    except AttributeError:
        # Usually happens when soup = None; e.g. when URL is an image or PDF.
        title = ''
    except URLError:
        title = ''

    return title

{'fraction_non_alphanumeric': 0.11106889564336372, 'fraction_numerical': 0.005065856129685917, 'mean_word_length': 2.8353569694026226, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827895', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2212, 'n_words': 707}