<commit_before>import json
import random
import socket
import dateutil.parser
import sqlalchemy
from sqlalchemy.dialects.postgresql import insert
import collections

import common.http
from common import utils
from common import postgres
from common.config import config

GAME_CHECK_INTERVAL = 5*60

User = collections.namedtuple('User', ['id', 'name', 'display_name', 'token'])
def get_user(id=None, name=None, get_missing=True):
	"""
	Get the details for a given user, specified by either name or id.

	Returns a named tuple of (id, name, display_name, token)

	If get_missing is true, get the details for this user from Twitch if the user
	is not already in the database. Otherwise if the userisn't in the database,
	this returns None.
	"""
	engine, metadata = postgres.get_engine_and_metadata()
	users = metadata.tables["users"]
	with engine.begin() as conn:
		query = sqlalchemy.select([users.c.id, users.c.name, users.c.display_name, users.c.twitch_oauth])
		if id is not None:
			query = query.where(users.c.id == id)
			url = "https://api.twitch.tv/kraken/users/%d" % id
			data = {}
			by_name = False
		elif name is not None:
			query = query.where(users.c.name == name)
			url = "https://api.twitch.tv/kraken/users"
			data = {'login': name}
			by_name = True
		else:
			raise ValueError("Pass at least one of name or id")

		row = conn.execute(query).first()
		if row:
			return User(*row)

		if get_missing:
			headers = {
				'Client-ID': config['twitch_clientid'],
				'Accept': 'application/vnd.twitchtv.v5+json',
			}
			res = common.http.request(url, data=data, headers=headers)
			user = json.loads(res)
			if by_name:
				user = user['users'][0]
			insert_query = insert(users)
			insert_query = insert_query.on_conflict_do_update(
				index_elements=[users.c.id],
				set_={
					'name': insert_query.excluded.name,
					'display_name': insert_query.excluded.display_name,
				},
			)
			conn.execute(insert_query, {
				"id": user["_id"],
				"name": user["name"],
				"display_name": user["display_name"],
			})

			return User(user["_id"], user["name"], user['display_name'], None)

class get_paginated_by_offset:
	"""
	Collect all the results from a paginated query that uses the offset/limit
	method of pagination.

	See:
	https://dev.twitch.tv/docs/v5/guides/using-the-twitch-api#paging-through-results-cursor-vs-offset

	Provide the url/data/headers without the offset/limit arguments, and the name
	of the attribute in the result which contains the list to be collected.

	Optionally can provide a limit to stop the process before the end of the list.
	The returned list may be longer than limit, but once the limit is reached, no
	further pages will be requested.
	"""
	PAGE_SIZE = 25

	def __init__(self, url, attr, data=None, headers=None, limit=None):
		self.url = url
		self.attr = attr
		if data:
			self.data = dict(data)
		else:
			self.data = {}
		self.headers = headers
		self.limit = limit

		self.count = 0
		self.total = 1
		self.offset = 0
		self.buffer = None

	async def __aiter__(self):
		return self

	async def __anext__(self):
		while True:
			if self.buffer:
				self.count += 1
				return self.buffer.pop(0)

			if self.offset > self.total:
				raise StopAsyncIteration
			if self.limit is not None and self.count > self.limit:
				raise StopAsyncIteration

			self.data['offset'] = self.offset
			self.data['limit'] = self.PAGE_SIZE
			res = await common.http.request_coro(self.url, data=self.data, headers=self.headers)
			res = json.loads(res)
			self.buffer = res[self.attr]
			self.total = res['_total']
			self.offset += self.PAGE_SIZE

class get_paginated_by_cursor:
	"""
	Collect all the results from a paginated query that uses the cursor
	method of pagination.

	See:
	https://dev.twitch.tv/docs/v5/guides/using-the-twitch-api#paging-through-results-cursor-vs-offset

	Provide the url/data/headers without the cursor argument, and the name
	of the attribute in the result which contains the list to be collected.

	Optionally can provide a limit to stop the process before the end of the list.
	The returned list may be longer than limit, but once the limit is reached, no
	further pages will be requested.
	"""
	def __init__(self, url, attr, data=None, headers=None, limit=None):
		self.url = url
		self.attr = attr
		if data:
			self.data = dict(data)
		else:
			self.data = {}
		self.headers = headers
		self.limit = limit

		self.count = 0
		self.cursor = True
		self.buffer = None

	async def __aiter__(self):
		return self

	async def __anext__(self):
		while True:
			if self.buffer:
				self.count += 1
				return self.buffer.pop(0)

			if not self.cursor:
				raise StopAsyncIteration
			if self.limit is not None and self.count > self.limit:
				raise StopAsyncIteration

			res = await common.http.request_coro(self.url, data=self.data, headers=self.headers)
			res = json.loads(res)
			self.buffer = res[self.attr]
			self.cursor = self.data['cursor'] = res.get('_cursor')

def get_info_uncached(username=None, use_fallback=True):
	"""
	Get the Twitch info for a particular user or channel.

	Defaults to the stream channel if not otherwise specified.

	For response object structure, see:
	https://dev.twitch.tv/docs/v5/reference/channels/#get-channel-by-id

	May throw exceptions on network/Twitch error.
	"""
	if username is None:
		username = config['channel']<commit_msg>fix async generators<commit_after>import json
import random
import socket
import dateutil.parser
import sqlalchemy
from sqlalchemy.dialects.postgresql import insert
import collections

import common.http
from common import utils
from common import postgres
from common.config import config

GAME_CHECK_INTERVAL = 5*60

User = collections.namedtuple('User', ['id', 'name', 'display_name', 'token'])
def get_user(id=None, name=None, get_missing=True):
	"""
	Get the details for a given user, specified by either name or id.

	Returns a named tuple of (id, name, display_name, token)

	If get_missing is true, get the details for this user from Twitch if the user
	is not already in the database. Otherwise if the userisn't in the database,
	this returns None.
	"""
	engine, metadata = postgres.get_engine_and_metadata()
	users = metadata.tables["users"]
	with engine.begin() as conn:
		query = sqlalchemy.select([users.c.id, users.c.name, users.c.display_name, users.c.twitch_oauth])
		if id is not None:
			query = query.where(users.c.id == id)
			url = "https://api.twitch.tv/kraken/users/%d" % id
			data = {}
			by_name = False
		elif name is not None:
			query = query.where(users.c.name == name)
			url = "https://api.twitch.tv/kraken/users"
			data = {'login': name}
			by_name = True
		else:
			raise ValueError("Pass at least one of name or id")

		row = conn.execute(query).first()
		if row:
			return User(*row)

		if get_missing:
			headers = {
				'Client-ID': config['twitch_clientid'],
				'Accept': 'application/vnd.twitchtv.v5+json',
			}
			res = common.http.request(url, data=data, headers=headers)
			user = json.loads(res)
			if by_name:
				user = user['users'][0]
			insert_query = insert(users)
			insert_query = insert_query.on_conflict_do_update(
				index_elements=[users.c.id],
				set_={
					'name': insert_query.excluded.name,
					'display_name': insert_query.excluded.display_name,
				},
			)
			conn.execute(insert_query, {
				"id": user["_id"],
				"name": user["name"],
				"display_name": user["display_name"],
			})

			return User(user["_id"], user["name"], user['display_name'], None)

class get_paginated_by_offset:
	"""
	Collect all the results from a paginated query that uses the offset/limit
	method of pagination.

	See:
	https://dev.twitch.tv/docs/v5/guides/using-the-twitch-api#paging-through-results-cursor-vs-offset

	Provide the url/data/headers without the offset/limit arguments, and the name
	of the attribute in the result which contains the list to be collected.

	Optionally can provide a limit to stop the process before the end of the list.
	The returned list may be longer than limit, but once the limit is reached, no
	further pages will be requested.
	"""
	PAGE_SIZE = 25

	def __init__(self, url, attr, data=None, headers=None, limit=None):
		self.url = url
		self.attr = attr
		if data:
			self.data = dict(data)
		else:
			self.data = {}
		self.headers = headers
		self.limit = limit

		self.count = 0
		self.total = 1
		self.offset = 0
		self.buffer = None

	def __aiter__(self):
		return self

	async def __anext__(self):
		while True:
			if self.buffer:
				self.count += 1
				return self.buffer.pop(0)

			if self.offset > self.total:
				raise StopAsyncIteration
			if self.limit is not None and self.count > self.limit:
				raise StopAsyncIteration

			self.data['offset'] = self.offset
			self.data['limit'] = self.PAGE_SIZE
			res = await common.http.request_coro(self.url, data=self.data, headers=self.headers)
			res = json.loads(res)
			self.buffer = res[self.attr]
			self.total = res['_total']
			self.offset += self.PAGE_SIZE

class get_paginated_by_cursor:
	"""
	Collect all the results from a paginated query that uses the cursor
	method of pagination.

	See:
	https://dev.twitch.tv/docs/v5/guides/using-the-twitch-api#paging-through-results-cursor-vs-offset

	Provide the url/data/headers without the cursor argument, and the name
	of the attribute in the result which contains the list to be collected.

	Optionally can provide a limit to stop the process before the end of the list.
	The returned list may be longer than limit, but once the limit is reached, no
	further pages will be requested.
	"""
	def __init__(self, url, attr, data=None, headers=None, limit=None):
		self.url = url
		self.attr = attr
		if data:
			self.data = dict(data)
		else:
			self.data = {}
		self.headers = headers
		self.limit = limit

		self.count = 0
		self.cursor = True
		self.buffer = None

	def __aiter__(self):
		return self

	async def __anext__(self):
		while True:
			if self.buffer:
				self.count += 1
				return self.buffer.pop(0)

			if not self.cursor:
				raise StopAsyncIteration
			if self.limit is not None and self.count > self.limit:
				raise StopAsyncIteration

			res = await common.http.request_coro(self.url, data=self.data, headers=self.headers)
			res = json.loads(res)
			self.buffer = res[self.attr]
			self.cursor = self.data['cursor'] = res.get('_cursor')

def get_info_uncached(username=None, use_fallback=True):
	"""
	Get the Twitch info for a particular user or channel.

	Defaults to the stream channel if not otherwise specified.

	For response object structure, see:
	https://dev.twitch.tv/docs/v5/reference/channels/#get-channel-by-id

	May throw exceptions on network/Twitch error.
	"""
	if username is None:
		username = config['channel']
{'fraction_non_alphanumeric': 0.109604414102684, 'fraction_numerical': 0.0033666884877957544, 'mean_word_length': 3.696530522617479, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6369057', 'n_tokens_mistral': 3884, 'n_tokens_neox': 3464, 'n_words': 1071}