<commit_before>from bs4 import BeautifulSoup
import markovify
import sys
import requests

inputgiven = False
intitles = ""
headers = {
    'User-Agent': 'python:reddit-title-generator/v0.1',
    'content-type': 'text'
}

# Check if the user has given input, if not, prompt them for it
if len(sys.argv) == 1:
    subreddit = input("Generate post title from which subreddit? (Just the name, no /r/): ")
    url = requests.get("http://reddit.com/r/" + subreddit, headers=headers)
elif len(sys.argv) > 2:
    print("Please only enter a single argument, the subreddit name!")
    exit()
else:
    url = requests.get("http://reddit.com/r/" + sys.argv[1], headers=headers)

soup = BeautifulSoup(url.text, 'lxml')

# Find all the titles on a single reddit page
titles = soup('a', {'class': 'title may-blank '})

# Get each title and make it more like a long paragraph (better for markovify)
for title in titles:
    intitles += title.text + '. '

markov = markovify.Text(intitles, state_size=1)

print(markov.make_short_sentence(300, tries=50))
<commit_msg>moved everything into functions, better input checking/handling<commit_after>from bs4 import BeautifulSoup
import markovify
import sys
import requests

headers = {
    'User-Agent': 'python:reddit-title-generator/v0.1',
    'content-type': 'text'
}


def crawl_url(input_url):
    intitles = ""
    soup = BeautifulSoup(input_url.text, 'lxml')

    # Find all the titles on a single reddit page
    titles = soup('a', {'class': 'title may-blank '})

    # Get each title and make it more like a long paragraph (better for markovify)
    for title in titles:
        intitles += title.text + '. '
    if intitles == '':
        print("Crawl did not seem to work, please try a different subreddit!")
        exit()
    return intitles


def generate_text(text):
    markov = markovify.Text(text, state_size=1)
    print(markov.make_short_sentence(300, tries=40))


if __name__ == "__main__":
    url = ""
    if len(sys.argv) == 1:
        subreddit = ""
        while not subreddit:
            subreddit = input("Generate post title from which subreddit? (Just the name, no /r/):\n")
        url = requests.get("http://reddit.com/r/" + subreddit, headers=headers, timeout=10)
    elif len(sys.argv) > 2:
        print("Please only enter a single argument, the subreddit name!")
        exit()
    else:
        url = requests.get("http://reddit.com/r/" + sys.argv[1], headers=headers, timeout=10)

    generate_text(crawl_url(url))

{'fraction_non_alphanumeric': 0.12978637646110439, 'fraction_numerical': 0.01128577186618299, 'mean_word_length': 3.4480286738351253, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4249006', 'n_tokens_mistral': 818, 'n_tokens_neox': 810, 'n_words': 242}