<commit_before>#!usr/bin/python
#
# (C) Legoktm 2009
# Released under the MIT License
#
# All pages should be in the wiki.Page() format

import wiki
#set up api
API = wiki.API()

"""Gets all articles in a certain category and returns a list"""
def category(page, excludens = False):
	if not page.isCategory():
		raise wiki.NotCategory(page.title())
	params = {
		'action':'query',
		'list':'categorymembers',
		'cmtitle':page.title(),
		'cmlimit':'500',
	}
	result = API.query(params)
	#format the list
	list = []
	res = result['query']['categorymembers']
	for page in res:
		try:
			if excludens:
				if page['ns'] != int(excludens):
					list.append(wiki.Page(page['title']))
			else:
				list.append(wiki.Page(page['title']))
		except UnicodeEncodeError:
			pass
	return list

def prefixindex(page):
	ns = page.namespace()
	prefix = page.titlewonamespace() + '/'
	params = {
		'action':'query',
		'list':'alllinks',
		'alprefix':prefix,
		'alnamespace':ns,
		'allimit':'500',
	}
	res = API.query(params)['query']['alllinks']
	list = []
	for page in res:
		wikipage = wiki.Page(page['title'])
		if not (wikipage in list):
			list.append(wikipage)
	return list<commit_msg>try new pagegen option to reduce server load<commit_after>#!usr/bin/python
#
# (C) Legoktm 2009
# Released under the MIT License
#
# All pages should be in the wiki.Page() format

import wiki
#set up api
API = wiki.API()

"""Gets all articles in a certain category and returns a list"""
def category(page, excludens = False):
	if not page.isCategory():
		raise wiki.NotCategory(page.title())
	params = {
		'action':'query',
		'list':'categorymembers',
		'cmtitle':page.title(),
		'cmlimit':'500',
	}
	result = API.query(params)
	#format the list
	list = []
	res = result['query']['categorymembers']
	for page in res:
		try:
			if excludens:
				if page['ns'] != int(excludens):
					list.append(page['title'])
			else:
				list.append(page['title'])
		except UnicodeEncodeError:
			pass
	return list

def prefixindex(page):
	ns = page.namespace()
	prefix = page.titlewonamespace() + '/'
	params = {
		'action':'query',
		'list':'alllinks',
		'alprefix':prefix,
		'alnamespace':ns,
		'allimit':'500',
	}
	res = API.query(params)['query']['alllinks']
	list = []
	for page in res:
		wikipage = wiki.Page(page['title'])
		if not (wikipage in list):
			list.append(wikipage)
	return list
{'fraction_non_alphanumeric': 0.16390658174097664, 'fraction_numerical': 0.008492569002123142, 'mean_word_length': 4.258928571428571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1745063', 'n_tokens_mistral': 923, 'n_tokens_neox': 820, 'n_words': 214}