<commit_before>import praw, HTMLParser, json, requests
from config import username, password, subreddit, wiki_stream_config, wiki_streams
from PIL import Image
from StringIO import StringIO

def chunker(seq, size):
	# Chunks up the stream wikipage into chunks to send to twitch.tv 
	# because otherwise it'd be wasteful to send a bunch of single requests.
	# Shamelessly taken from a SO answer. 
    return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))

#Checks reddit inbox for streams that have been messaged to the bot. 
def check_inbox():
	streamlist = []
	inbox = reddit.get_inbox()
	for message in inbox:
		if message.new:
			msg = message.body.split()[0]

			# This is why I should learn regexp 
			if "twitch.tv/" in msg \
			and len((msg[(msg.index(".tv/")+4):len(msg)])) <=25 \
			and message.subject == "Twitch.tv request /r/" + str(subreddit):
				streamlist.append(msg[(msg.index(".tv/")+4):len(msg)])
				message.reply(
					"Your stream has been added to the list of livestreams in the sidebar, \
					it will display the next time you are live on twitch.tv. \n \n Problems? Contact \
					the moderators [here](http://www.reddit.com/message/compose?to=%2Fr%2F"+ str(subreddit) + "). Do not reply to this message."
					)
				message.mark_as_read()
			else:
				pass
		message.mark_as_read()
	return streamlist

def get_stream_list(subreddit):
	# This tries to get the config settings for meta_games from wiki_stream_config, and the list of streams from wiki/streams/
	# It will then check the bot's inbox for any new livestreams it's been messaged, and add those to the /wiki/streams/ page.
	# after that, it chunks up the streams and requests info from twitch.tv.
	stream_strings = ""
	try:
		config_list = reddit.get_wiki_page(subreddit, wiki_stream_config).content_md.splitlines() 
		# Gets the wikipage wiki_stream_config, splits it up into a list seperated by each new line.
		for meta_game in config_list[:]:
		# Loop through each meta_game that is listed in wiki_stream_config
			if meta_game == '': 
		# If it's an empty string (as in a blank line), remove it from the config list
				config_list.remove(meta_game)
	except: 
		config_list = [] 

	# Same as above, but append the streams to a string called stream_strings. 
	stream_wikipage = reddit.get_wiki_page(subreddit, wiki_streams).content_md.splitlines()
	for stream in stream_wikipage:
		if stream == '':
			stream_wikipage.remove(stream)
		stream_strings += stream + "\n"

	# Check inbox for new streams sent via a message. 
	streamlist = check_inbox()
	# Add any new streams to the wiki page, 
	if len(streamlist):
		for stream in streamlist:
			if stream not in stream_wikipage:
				stream_strings += "\n" + stream
				stream_wikipage.append(stream)
		subreddit.edit_wiki_page(wiki_streams, stream_strings, reason="Adding streams: " + ", ".join(streamlist))

	livestreams = []
	for chunk in chunker(stream_wikipage, 100): 
	# For chunk of 100 (or less, if the list is not 100 long.)
	# Reset the api link
		api_link = "http://api.justin.tv/api/stream/list.json?channel="
		for stream in chunk:
			api_link += stream + ","
		livestreams = get_stream_info(api_link, livestreams)
	results, preview_images = parse_stream_info(livestreams, config_list)
	return results, preview_images

def get_stream_info(api_link,livestreams):
	# This checks that there was any data received in the chunk, and if there was, it appends it to livestreams list.
	data = requests.get(api_link).json()
	# Get the JSON data from our api request.
	if len(data):
		livestreams.append(data)
	return livestreams


def parse_stream_info(livestreams, config_list):
	# Formats the stream info (if there was any received) to be posted to the reddit sidebar. 
	# Also prepares the thumbnail urls to be put into a spritesheet. 
	results, n, thumbnail_urls, preview_images= [], 0, [], []

	# If it doesn't contain anything (meaning there are no current live streams), 
	# append results with the no streams are live string. 
	if not len(livestreams):
		results.append("**No streams are currently live.**\n") 
	else:
		for streamer_list in livestreams: 
		# Looping through the JSON structure 
			for streamer_info_dict in streamer_list:
				if not len(config_list) or streamer_info_dict["channel"]["meta_game"] in config_list:
				# If the meta_game is in config_list, do the following:<commit_msg>fixing problem if no json object is received from twitch.tv<commit_after>import praw, HTMLParser, json, requests
from config import username, password, subreddit, wiki_stream_config, wiki_streams
from PIL import Image
from StringIO import StringIO

def chunker(seq, size):
	# Chunks up the stream wikipage into chunks to send to twitch.tv 
	# because otherwise it'd be wasteful to send a bunch of single requests.
	# Shamelessly taken from a SO answer. 
    return (seq[pos:pos + size] for pos in xrange(0, len(seq), size))

#Checks reddit inbox for streams that have been messaged to the bot. 
def check_inbox():
	streamlist = []
	inbox = reddit.get_inbox()
	for message in inbox:
		if message.new:
			msg = message.body.split()[0]

			# This is why I should learn regexp 
			if "twitch.tv/" in msg \
			and len((msg[(msg.index(".tv/")+4):len(msg)])) <=25 \
			and message.subject == "Twitch.tv request /r/" + str(subreddit):
				streamlist.append(msg[(msg.index(".tv/")+4):len(msg)])
				message.reply(
					"Your stream has been added to the list of livestreams in the sidebar, \
					it will display the next time you are live on twitch.tv. \n \n Problems? Contact \
					the moderators [here](http://www.reddit.com/message/compose?to=%2Fr%2F"+ str(subreddit) + "). Do not reply to this message."
					)
				message.mark_as_read()
			else:
				pass
		message.mark_as_read()
	return streamlist

def get_stream_list(subreddit):
	# This tries to get the config settings for meta_games from wiki_stream_config, and the list of streams from wiki/streams/
	# It will then check the bot's inbox for any new livestreams it's been messaged, and add those to the /wiki/streams/ page.
	# after that, it chunks up the streams and requests info from twitch.tv.
	stream_strings = ""
	try:
		config_list = reddit.get_wiki_page(subreddit, wiki_stream_config).content_md.splitlines() 
		# Gets the wikipage wiki_stream_config, splits it up into a list seperated by each new line.
		for meta_game in config_list[:]:
		# Loop through each meta_game that is listed in wiki_stream_config
			if meta_game == '': 
		# If it's an empty string (as in a blank line), remove it from the config list
				config_list.remove(meta_game)
	except: 
		config_list = [] 

	# Same as above, but append the streams to a string called stream_strings. 
	stream_wikipage = reddit.get_wiki_page(subreddit, wiki_streams).content_md.splitlines()
	for stream in stream_wikipage:
		if stream == '':
			stream_wikipage.remove(stream)
		stream_strings += stream + "\n"

	# Check inbox for new streams sent via a message. 
	streamlist = check_inbox()
	# Add any new streams to the wiki page, 
	if len(streamlist):
		for stream in streamlist:
			if stream not in stream_wikipage:
				stream_strings += "\n" + stream
				stream_wikipage.append(stream)
		subreddit.edit_wiki_page(wiki_streams, stream_strings, reason="Adding streams: " + ", ".join(streamlist))

	livestreams = []
	for chunk in chunker(stream_wikipage, 100): 
	# For chunk of 100 (or less, if the list is not 100 long.)
	# Reset the api link
		api_link = "http://api.justin.tv/api/stream/list.json?channel="
		for stream in chunk:
			api_link += stream + ","
		livestreams = get_stream_info(api_link, livestreams)
	results, preview_images = parse_stream_info(livestreams, config_list)
	return results, preview_images

def get_stream_info(api_link,livestreams):
	# This checks that there was any data received in the chunk, and if there was, it appends it to livestreams list.
	try:
		data = requests.get(api_link).json()
		if len(data):
			livestreams.append(data)
	except ValueError:
		pass
	# Get the JSON data from our api request.
	return livestreams


def parse_stream_info(livestreams, config_list):
	# Formats the stream info (if there was any received) to be posted to the reddit sidebar. 
	# Also prepares the thumbnail urls to be put into a spritesheet. 
	results, n, thumbnail_urls, preview_images= [], 0, [], []

	# If it doesn't contain anything (meaning there are no current live streams), 
	# append results with the no streams are live string. 
	if not len(livestreams):
		results.append("**No streams are currently live.**\n") 
	else:
		for streamer_list in livestreams: 
		# Looping through the JSON structure 
			for streamer_info_dict in streamer_list:
				if not len(config_list) or streamer_info_dict["channel"]["meta_game"] in config_list:
				# If the meta_game is in config_list, do the following:
{'fraction_non_alphanumeric': 0.09261363636363637, 'fraction_numerical': 0.004090909090909091, 'mean_word_length': 4.134772462077013, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2671308', 'n_tokens_mistral': 3058, 'n_tokens_neox': 2774, 'n_words': 1006}