<commit_before># -*- coding: utf-8 -*-
"""
:copyright: (c) 2013-2016 by <NAME> and <NAME>
:license: MIT, see LICENSE for more details.

IndieWeb Webmention Tools
"""

import os
import re
import json
import shlex
from urlparse import urlsplit, urlunsplit

import requests


cfgFilenames = ('ronkyuu.cfg', '.ronkyuu.cfg')
cfgFilepaths = (os.getcwd(), '~/', '~/.ronkyuu/')

possibleConfigFiles = []

for p in cfgFilepaths:
    for f in cfgFilenames:
        possibleConfigFiles.append(os.path.join(p, f))


def discoverConfig(cfgFilename=None):
    result = {}

    if cfgFilename is None:
        for possibleFile in possibleConfigFiles:
            possibleFile = os.path.expanduser(possibleFile)
            if os.path.exists(possibleFile):
                result = json.load(open(possibleFile, 'r'))
                break
    else:
        possibleFile = os.path.expanduser(cfgFilename)
        if os.path.exists(possibleFile):
            result = json.load(open(possibleFile, 'r'))

    return result


def getURLChain(targetURL):
    ok = False
    chain = []
    try:
        r = requests.head(targetURL, allow_redirects=True)
        ok = r.status_code == requests.codes.ok
        if ok:
            for resp in r.history:
                chain.append(r.url)
    except:
        ok = False
    return (ok, chain)

def cleanURL(targetURL):
    scheme, netloc, path, query, fragment = urlsplit(targetURL)
    if len(path) == 0:
        path = '/'
    return urlunsplit((scheme, netloc, path, query, fragment))

def normalizeURL(targetURL):
    result = None
    ok, chain = getURLChain(targetURL)
    if ok:
        if len(chain) > 0:
            result = chain[-1]
        else:
            result = targetURL
    return result

def quoted_split(text, boundary):
    o = shlex.shlex(text)
    o.whitespace = boundary
    o.whitespace_split = True
    return list(o)

def parse_link_header(link):
    """takes the link header as a string and returns a dictionary with rel values as keys and urls as values
    :param link: link header as a string
    :rtype: dictionary {rel_name: rel_value}
    """
    rel_dict = {}
    for rels in link.split(','):
        rel_break = quoted_split(rels, ';')
        try:
            rel_url = re.search('<(.+?)>', rel_break[0]).group(1)
            rel_names = quoted_split(rel_break[1], '=')[-1]
            for name in rel_names.split():
                if name.startswith('"') and name.endswith('"'):
                    name = name[1:-1]
                rel_dict[name] = rel_url
        except (AttributeError, IndexError):
            pass

    return rel_dict
<commit_msg>silly mistake, remove quotes outside of the loop<commit_after># -*- coding: utf-8 -*-
"""
:copyright: (c) 2013-2016 by <NAME> and <NAME>
:license: MIT, see LICENSE for more details.

IndieWeb Webmention Tools
"""

import os
import re
import json
import shlex
from urlparse import urlsplit, urlunsplit

import requests


cfgFilenames = ('ronkyuu.cfg', '.ronkyuu.cfg')
cfgFilepaths = (os.getcwd(), '~/', '~/.ronkyuu/')

possibleConfigFiles = []

for p in cfgFilepaths:
    for f in cfgFilenames:
        possibleConfigFiles.append(os.path.join(p, f))


def discoverConfig(cfgFilename=None):
    result = {}

    if cfgFilename is None:
        for possibleFile in possibleConfigFiles:
            possibleFile = os.path.expanduser(possibleFile)
            if os.path.exists(possibleFile):
                result = json.load(open(possibleFile, 'r'))
                break
    else:
        possibleFile = os.path.expanduser(cfgFilename)
        if os.path.exists(possibleFile):
            result = json.load(open(possibleFile, 'r'))

    return result


def getURLChain(targetURL):
    ok = False
    chain = []
    try:
        r = requests.head(targetURL, allow_redirects=True)
        ok = r.status_code == requests.codes.ok
        if ok:
            for resp in r.history:
                chain.append(r.url)
    except:
        ok = False
    return (ok, chain)

def cleanURL(targetURL):
    scheme, netloc, path, query, fragment = urlsplit(targetURL)
    if len(path) == 0:
        path = '/'
    return urlunsplit((scheme, netloc, path, query, fragment))

def normalizeURL(targetURL):
    result = None
    ok, chain = getURLChain(targetURL)
    if ok:
        if len(chain) > 0:
            result = chain[-1]
        else:
            result = targetURL
    return result

def quoted_split(text, boundary):
    o = shlex.shlex(text)
    o.whitespace = boundary
    o.whitespace_split = True
    return list(o)

def parse_link_header(link):
    """takes the link header as a string and returns a dictionary with rel values as keys and urls as values
    :param link: link header as a string
    :rtype: dictionary {rel_name: rel_value}
    """
    rel_dict = {}
    for rels in link.split(','):
        rel_break = quoted_split(rels, ';')
        try:
            rel_url = re.search('<(.+?)>', rel_break[0]).group(1)
            rel_names = quoted_split(rel_break[1], '=')[-1]
            if rel_names.startswith('"') and rel_names.endswith('"'):
                    rel_names = rel_names[1:-1]
            for name in rel_names.split():
                rel_dict[name] = rel_url
        except (AttributeError, IndexError):
            pass

    return rel_dict

{'fraction_non_alphanumeric': 0.11661311661311662, 'fraction_numerical': 0.006804006804006804, 'mean_word_length': 2.4141935483870967, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7612291', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1629, 'n_words': 445}