<commit_before># 
# FILE: download.py
#
# AUTHOR: <NAME> <<EMAIL>>
#
# DESCRIPTION: Retrieve bookmarks from Delicious.
#
# LICENSE: BSD v3.
#
# TOUCHED: 2016.03.25
#

import json
import os
import os.path
import urllib.request


# urllib.error.HTTPError


def fetchDataAtUri(uri):
    """
    Return data in JSON format as bytes for 'uri'.
    """
    data = None
    with urllib.request.urlopen(uri) as f:
        data = f.read()
    return data


def fetchTags(username):
    """
    Return tags for 'username'.
    """
    uri = 'http://feeds.delicious.com/v2/json/tags/%s' % (username)
    return fetchDataAtUri(uri).decode('utf-8')


def fetchBookmarksForTag(tag, username):
    """
    Return bookmarks for 'tag' for 'username'.
    """
    uri = 'http://feeds.delicious.com/v2/json/%s/%s' % (username, tag)
    return fetchDataAtUri(uri).decode('utf-8')


def loadTagsFromFile(path):
    """
    Return tags loaded from file at 'path'.
    """
    # This can be used if you've already retrieved the tags.
    tags = None
    with open(path, 'r') as f:
        tags = json.loads(f.read())
    return tags


def loadTags(username, path):
    """
    Load tags from either file at 'path' or network using 'username'.
    """
    tags = None
    if path:
        tags = loadTagsFromFile(path)
    else:
        tags = json.loads(fetchTags(username)
    return tags


def main(username, tagFilePath=None):
    """
    Retrieve bookmarks for 'username': max 100 per tag, per Delicious restrictions.
    """
    directory = 'data'
    if not os.path.exists(directory):
        os.makedirs(directory)
    with open(os.path.join(directory, 'bookmarks.log'), 'w') as log:
        tags = loadTags(username, tagFilePath)
        # Doing everything in 1 thread reduces load on the Delicious server.
        for aKey in tags.keys():
            bookmarks = fetchBookmarksForTag(aKey, username)
            with open(os.path.join(directory, '%s.json' % (aKey)), 'w') as f:
                f.write(bookmarks)
            print(aKey)
            if tags[aKey] > 100:
                # We want to know if we're missing bookmarks for a tag.
                log.write('%s: %d\n' % (aKey, tags[aKey]))


if __name__ == '__main__':
    # Set username for account.
    username = 'mario.d'<commit_msg>Command line argument added for username.<commit_after># 
# FILE: download.py
#
# AUTHOR: <NAME> <<EMAIL>>
#
# DESCRIPTION: Retrieve bookmarks from Delicious.
#
# LICENSE: BSD v3.
#
# TOUCHED: 2016.03.25
#

import json
import os
import os.path
import sys
import urllib.request


# urllib.error.HTTPError


def fetchDataAtUri(uri):
    """
    Return data in JSON format as bytes for 'uri'.
    """
    data = None
    with urllib.request.urlopen(uri) as f:
        data = f.read()
    return data


def fetchTags(username):
    """
    Return tags for 'username'.
    """
    uri = 'http://feeds.delicious.com/v2/json/tags/%s' % (username)
    return fetchDataAtUri(uri).decode('utf-8')


def fetchBookmarksForTag(tag, username):
    """
    Return bookmarks for 'tag' for 'username'.
    """
    uri = 'http://feeds.delicious.com/v2/json/%s/%s' % (username, tag)
    return fetchDataAtUri(uri).decode('utf-8')


def loadTagsFromFile(path):
    """
    Return tags loaded from file at 'path'.
    """
    # This can be used if you've already retrieved the tags.
    tags = None
    with open(path, 'r') as f:
        tags = json.loads(f.read())
    return tags


def loadTags(username, path):
    """
    Load tags from either file at 'path' or network using 'username'.
    """
    tags = None
    if path:
        tags = loadTagsFromFile(path)
    else:
        tags = json.loads(fetchTags(username)
    return tags


def main(username, tagFilePath=None):
    """
    Retrieve bookmarks for 'username': max 100 per tag, per Delicious restrictions.
    """
    directory = 'data'
    if not os.path.exists(directory):
        os.makedirs(directory)
    with open(os.path.join(directory, 'bookmarks.log'), 'w') as log:
        tags = loadTags(username, tagFilePath)
        # Doing everything in 1 thread reduces load on the Delicious server.
        for aKey in tags.keys():
            bookmarks = fetchBookmarksForTag(aKey, username)
            with open(os.path.join(directory, '%s.json' % (aKey)), 'w') as f:
                f.write(bookmarks)
            print(aKey)
            if tags[aKey] > 100:
                # We want to know if we're missing bookmarks for a tag.
                log.write('%s: %d\n' % (aKey, tags[aKey]))


if __name__ == '__main__':
    # Set username for account.
    username = ''
    try:
        username = sys.argv[1]
    except:
        print("Usage: python3 download.py <username>")
        sys.exit(1)
{'fraction_non_alphanumeric': 0.13052676115929765, 'fraction_numerical': 0.009096678654537761, 'mean_word_length': 2.5257270693512304, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5577072', 'n_tokens_mistral': 1568, 'n_tokens_neox': 1516, 'n_words': 434}