<commit_before># vim:ts=4:sw=4:noexpandtab

from __future__ import absolute_import, print_function

import sys
import datetime
import traceback
import time
import random
import textwrap
import functools
import argparse
import logging
import itertools
import pprint
import re

import irc.bot
import irc.client
import pkg_resources

import pmxbot.itertools
import pmxbot.dictlib

log = logging.getLogger('pmxbot')

class WarnHistory(dict):
	warn_every = datetime.timedelta(seconds=60)
	warn_message = textwrap.dedent("""
		PRIVACY INFORMATION: LOGGING IS ENABLED!!

		The following channels are logged are being logged to provide a
		convenient, searchable archive of conversation histories:
		{logged_channels_string}
		""").lstrip()

	def needs_warning(self, key):
		now = datetime.datetime.utcnow()
		if not key in self or self._expired(self[key], now):
			self[key] = now
			return True
		return False

	def _expired(self, last, now):
		return now - last > self.warn_every

class NoLog(object):
	"""
	A sentinel indicating that subsequent items should not be logged.
	"""

	@classmethod
	def secret_items(cls, items):
		"""
		Iterate over the items, and yield each item with an indicator of
		whether it should be secret or not.

		>>> tuple(NoLog.secret_items(['a', 'b', NoLog, 'c']))
		((False, 'a'), (False, 'b'), (True, 'c'))
		"""
		secret = False
		for item in items:
			if item is cls:
				secret = True
				continue
			yield secret, item

class LoggingCommandBot(irc.bot.SingleServerIRCBot):
	def __init__(self, server, port, nickname, channels, password=<PASSWORD>):
		server_list = [(server, port, password)]
		irc.bot.SingleServerIRCBot.__init__(self, server_list, nickname,
			nickname)
		self.nickname = nickname
		self._channels = channels
		self._nickname = nickname
		self.warn_history = WarnHistory()

	def connect(self, *args, **kwargs):
		factory = irc.connection.Factory()
		factory.from_legacy_params(ssl=pmxbot.config.use_ssl)
		return irc.bot.SingleServerIRCBot.connect(self,
			connect_factory = factory, *args, **kwargs)

	def out(self, channel, s, log=True):
		log &= (channel in self._channels
			and channel in pmxbot.config.log_channels)
		self._out(self.connection, channel, s, self._nickname, log)

	@staticmethod
	def _out(client, channel, msg, nick, log=True):
		func = client.privmsg
		if msg.startswith(u'/me '):
			func = client.action
			msg = msg.split(' ', 1)[-1].lstrip()
			log = False
		try:
			func(channel, msg)
		except irc.client.MessageTooLong:
			# some messages will fail because they're too long
			globals()['log'].warning("Long message could not be "
				"transmitted: %s", msg)
			return
		except irc.client.InvalidCharacters:
			globals()['log'].warning("Message contains carriage returns, "
				"which aren't allowed in IRC messages: %r", msg)
			return
		except Exception:
			globals()['log'].exception("Unhandled exception transmitting "
				"message: %r", msg)
			return
		if log:
			pmxbot.logging.Logger.store.message(channel, nick, msg)

	def _schedule_at(self, name, channel, when, func, args, doc):
		arguments = self.c, channel, func, args
		if isinstance(when, datetime.date):
			midnight = datetime.time(0,0)
			when = datetime.datetime.combine(when, midnight)
		if isinstance(when, datetime.datetime):
			cmd = irc.client.DelayedCommand.at_time(
				when, self.background_runner, arguments)
			self.c.irclibobj._schedule_command(cmd)
			return
		if not isinstance(when, datetime.time):
			raise ValueError("when must be datetime, date, or time")
		daily = datetime.timedelta(days=1)
		# convert when to the next datetime matching this time
		when = datetime.datetime.combine(datetime.date.today(), when)
		if when < datetime.datetime.now():
			when += daily
		cmd = irc.client.PeriodicCommandFixedDelay.at_time(
			when, daily, self.background_runner, arguments)
		self.c.irclibobj._schedule_command(cmd)

	def on_welcome(self, connection, event):
		for channel in self._channels:
			if not channel.startswith('#'):
				channel = '#' + channel
			connection.join(channel)
		for name, channel, howlong, func, args, doc, repeat in _delay_registry:
			arguments = connection, channel, func, args
			executor = (
				connection.execute_every if repeat
				else connection.execute_delayed
			)
			executor(howlong, self.background_runner, arguments)
		for action in _at_registry:
			self._schedule_at(*action)
<commit_msg>Added comments<commit_after># vim:ts=4:sw=4:noexpandtab

from __future__ import absolute_import, print_function

import sys
import datetime
import traceback
import time
import random
import textwrap
import functools
import argparse
import logging
import itertools
import pprint
import re

import irc.bot
import irc.client
import pkg_resources

import pmxbot.itertools
import pmxbot.dictlib

log = logging.getLogger('pmxbot')

class WarnHistory(dict):
	warn_every = datetime.timedelta(seconds=60)
	warn_message = textwrap.dedent("""
		PRIVACY INFORMATION: LOGGING IS ENABLED!!

		The following channels are logged are being logged to provide a
		convenient, searchable archive of conversation histories:
		{logged_channels_string}
		""").lstrip()

	def needs_warning(self, key):
		now = datetime.datetime.utcnow()
		if not key in self or self._expired(self[key], now):
			self[key] = now
			return True
		return False

	def _expired(self, last, now):
		return now - last > self.warn_every

class NoLog(object):
	"""
	A sentinel indicating that subsequent items should not be logged.
	"""

	@classmethod
	def secret_items(cls, items):
		"""
		Iterate over the items, and yield each item with an indicator of
		whether it should be secret or not.

		>>> tuple(NoLog.secret_items(['a', 'b', NoLog, 'c']))
		((False, 'a'), (False, 'b'), (True, 'c'))
		"""
		secret = False
		for item in items:
			if item is cls:
				secret = True
				continue
			yield secret, item

class LoggingCommandBot(irc.bot.SingleServerIRCBot):
	def __init__(self, server, port, nickname, channels, password=None):
		server_list = [(server, port, password)]
		irc.bot.SingleServerIRCBot.__init__(self, server_list, nickname,
			nickname)
		self.nickname = nickname
		self._channels = channels
		self._nickname = nickname
		self.warn_history = WarnHistory()

	def connect(self, *args, **kwargs):
		factory = irc.connection.Factory()
		factory.from_legacy_params(ssl=pmxbot.config.use_ssl)
		return irc.bot.SingleServerIRCBot.connect(self,
			connect_factory = factory, *args, **kwargs)

	def out(self, channel, s, log=True):
		log &= (channel in self._channels
			and channel in pmxbot.config.log_channels)
		self._out(self.connection, channel, s, self._nickname, log)

	@staticmethod
	def _out(client, channel, msg, nick, log=True):
		func = client.privmsg
		if msg.startswith(u'/me '):
			func = client.action
			msg = msg.split(' ', 1)[-1].lstrip()
			log = False
		try:
			func(channel, msg)
		except irc.client.MessageTooLong:
			# some messages will fail because they're too long
			globals()['log'].warning("Long message could not be "
				"transmitted: %s", msg)
			return
		except irc.client.InvalidCharacters:
			globals()['log'].warning("Message contains carriage returns, "
				"which aren't allowed in IRC messages: %r", msg)
			return
		except Exception:
			globals()['log'].exception("Unhandled exception transmitting "
				"message: %r", msg)
			return
		if log:
			pmxbot.logging.Logger.store.message(channel, nick, msg)

	def _schedule_at(self, name, channel, when, func, args, doc):
		arguments = self.c, channel, func, args
		if isinstance(when, datetime.date):
			midnight = datetime.time(0,0)
			when = datetime.datetime.combine(when, midnight)
		if isinstance(when, datetime.datetime):
			cmd = irc.client.DelayedCommand.at_time(
				when, self.background_runner, arguments)
			self.c.irclibobj._schedule_command(cmd)
			return
		if not isinstance(when, datetime.time):
			raise ValueError("when must be datetime, date, or time")
		daily = datetime.timedelta(days=1)
		# convert when to the next datetime matching this time
		when = datetime.datetime.combine(datetime.date.today(), when)
		if when < datetime.datetime.now():
			when += daily
		cmd = irc.client.PeriodicCommandFixedDelay.at_time(
			when, daily, self.background_runner, arguments)
		self.c.irclibobj._schedule_command(cmd)

	def on_welcome(self, connection, event):
		# join channels
		for channel in self._channels:
			if not channel.startswith('#'):
				channel = '#' + channel
			connection.join(channel)

		# set up delayed tasks
		for name, channel, howlong, func, args, doc, repeat in _delay_registry:
			arguments = connection, channel, func, args
			executor = (
				connection.execute_every if repeat
				else connection.execute_delayed
			)
			executor(howlong, self.background_runner, arguments)
		for action in _at_registry:
			self._schedule_at(*action)

{'fraction_non_alphanumeric': 0.11099761661559414, 'fraction_numerical': 0.0020429009193054137, 'mean_word_length': 4.409453652547575, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '711955', 'n_tokens_mistral': 3130, 'n_tokens_neox': 2788, 'n_words': 803}