<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import argparse
import os
import os.path
import random
import readline
import shutil
import subprocess
import sys
import time
import urllib.request

UA = "BetterPonymotes stylesheet update checker (1req/2.5secs; pm Typhos)"
SESS = "reddit_session=9958622%2C2012-06-14T22%3A56%3A05%2C432b711c2f42ca0748d224c12c54ec2ed514cd7d"

def cmd_help(args):
    parser = argparse.ArgumentParser(description="Print help text", prog="help")
    args = parser.parse_args(args)

    print(" ".join(sorted(Commands)))

def cmd_list(args):
    parser = argparse.ArgumentParser(description="List updated files", prog="list")
    args = parser.parse_args(args)

    print("Stylesheet updates:")
    subprocess.call(["bzr", "status", "stylesheets"])
    print("Emote updates:")
    subprocess.call(["bzr", "status", "emotes"])

def update_css(num, total, subreddit):
    url = "http://reddit.com/r/%s/stylesheet?nocache=%s" % (subreddit, random.randrange(1000000))

    try:
        old_ss = open("stylesheets/%s.css" % (subreddit), "rb").read()
    except IOError:
        # Assume file doesn't exist; new subreddit
        print("NOTICE: stylesheets/%s.css does not exist; new subreddit?" % (subreddit))
        old_ss = ""

    print("[%s/%s]: %s" % (num+1, total, url))
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Cookie": SESS})
    with urllib.request.urlopen(req) as stream:
        new_ss = stream.read()

    if old_ss != new_ss:
        print("NOTICE: Stylesheet changed in r/%s" % (subreddit))
        with open("stylesheets/%s.css" % (subreddit), "wb") as file:
            file.write(new_ss)
        return subreddit

def cmd_update(args):
    parser = argparse.ArgumentParser(description="Update stylesheet cache", prog="update")
    parser.add_argument("subreddits", nargs="*")
    args = parser.parse_args(args)

    if not args.subreddits:
        filenames = [fn for fn in sorted(os.listdir("stylesheets")) if fn.endswith(".css")]
        filenames.sort()
        subreddits = [fn.split(".")[0] for fn in filenames]
    else:
        subreddits = args.subreddits

    updates = []
    for (i, sr) in enumerate(subreddits[:-1]):
        updates.append(update_css(i, len(subreddits), sr))
        time.sleep(2.5)
    updates.append(update_css(len(subreddits) - 1, len(subreddits), subreddits[-1]))

    updates = list(filter(None, updates))<commit_msg>Shell no longer requires login cookie<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import argparse
import os
import os.path
import random
import readline
import shutil
import subprocess
import sys
import time
import urllib.request

UA = "BetterPonymotes stylesheet update checker (1req/2.5secs; pm Typhos)"

def cmd_help(args):
    parser = argparse.ArgumentParser(description="Print help text", prog="help")
    args = parser.parse_args(args)

    print(" ".join(sorted(Commands)))

def cmd_list(args):
    parser = argparse.ArgumentParser(description="List updated files", prog="list")
    args = parser.parse_args(args)

    print("Stylesheet updates:")
    subprocess.call(["bzr", "status", "stylesheets"])
    print("Emote updates:")
    subprocess.call(["bzr", "status", "emotes"])

def update_css(num, total, subreddit):
    url = "http://reddit.com/r/%s/stylesheet.css?nocache=%s" % (subreddit, random.randrange(1000000))

    try:
        old_ss = open("stylesheets/%s.css" % (subreddit), "rb").read()
    except IOError:
        # Assume file doesn't exist; new subreddit
        print("NOTICE: stylesheets/%s.css does not exist; new subreddit?" % (subreddit))
        old_ss = ""

    print("[%s/%s]: %s" % (num+1, total, url))
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req) as stream:
        new_ss = stream.read()

    if old_ss != new_ss:
        print("NOTICE: Stylesheet changed in r/%s" % (subreddit))
        with open("stylesheets/%s.css" % (subreddit), "wb") as file:
            file.write(new_ss)
        return subreddit

def cmd_update(args):
    parser = argparse.ArgumentParser(description="Update stylesheet cache", prog="update")
    parser.add_argument("subreddits", nargs="*")
    args = parser.parse_args(args)

    if not args.subreddits:
        filenames = [fn for fn in sorted(os.listdir("stylesheets")) if fn.endswith(".css")]
        filenames.sort()
        subreddits = [fn.split(".")[0] for fn in filenames]
    else:
        subreddits = args.subreddits

    updates = []
    for (i, sr) in enumerate(subreddits[:-1]):
        updates.append(update_css(i, len(subreddits), sr))
        time.sleep(2.5)
    updates.append(update_css(len(subreddits) - 1, len(subreddits), subreddits[-1]))

    updates = list(filter(None, updates))
{'fraction_non_alphanumeric': 0.1963421194190425, 'fraction_numerical': 0.018648018648018648, 'mean_word_length': 3.9275618374558303, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4386104', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1795, 'n_words': 432}