<commit_before>import auth, random, re, socket, sys, time
import sqlite3 as sql
from markov import Markov


"""
Helper functions.
"""

# Average output generator
def avg (list_):
   x = 0
   for i in list_:
      x += i
   return int(x / len(list_))


"""
Markov cleaning functions.
"""

# Characters that define what lines to ignore for the output
WEECHAT = ['>>', '<<', '--', '-->', '<--', '@Satsuki']
IGNORE = ['.luser', 'Raw', '!user', '.s']

# Counter of how many words are in a message to be averaged later
chat_average = {}

def clean_file (file_):
   global chat_average
   word_count = []
   avg_words = 0

   """
   This will create a file of just user submitted messages.

   Weechat logs in the following format:

   YYYY-MM-DD HH:MM:SS USERNAME	MESSAGE
   YYYY-MM-DD HH:MM:SS << USERNAME (ADDRESS) has quit (REASON)
   """
   chat_name = file_.split('/')[-1].split('.')[2]
   send_message(CHAN, "Just one second while refresh the logs.")
   print('[LOG] Creating clean output file for {0}.'.format(chat_name))

   file_i = open(file_, encoding="utf8").read().splitlines()
   file_o = open('clean_log_{0}'.format(chat_name), 'w', encoding="utf8")

   for line in file_i:
      try:
         msg = line.split('	') # Splits to [time / name / message] etc
         words = msg[2].split()  # Splits the message into words
         try:
            if msg[1] not in WEECHAT:
               if len(msg[2].split('::')) == 1:
                  if words[0] not in IGNORE:
                     if words[0].split(':')[0] not in ['http', 'https']:
                        word_count.append(len(words))
                        words.append('\n')
                        file_o.write(' '.join(words))
         except:
            pass
      except:
         pass
   chat_average[chat_name] = avg(word_count)
   file_o.close()


def refresh (chat_name):
   global chat_average

   clean_file('../../.weechat/logs/irc.animebytes.{0}.weechatlog'.format(chat_name))
   print('[LOG] Chat log file for {0} refreshed.'.format(chat_name))


def gen_markov (chat_name):
   global chat_average

   while True:
      try:
         file_ = open('clean_log_{0}'.format(chat_name), encoding="utf8")
         markov = Markov(file_)
         return markov.generate_markov_text(chat_average[chat_name] + random.randint(0, int(chat_average[chat_name]/2)))
         break
      except FileNotFoundError:
         return "Sorry, either that log does not exist or you forgot to enter a parameter."
      except:
         refresh(chat_name)
   file_.close()


"""
Database functions.
"""

# Initializes ops.
def get_ops ():
   global OPS

   con = sql.connect('living_in_the_.db')

   with con:
      con.row_factory = sql.Row
      db = con.cursor()

      OPS = []
      db.execute('SELECT * FROM Ops;')
      rows = db.fetchall()

      for row in rows:
         OPS.append(row['Name'])


# Adds a new op.
def add_op (user):
   global OPS

   con = sql.connect('living_in_the_.db')

   with con:
      con.row_factory = sql.Row
      db = con.cursor()

      try:
         db.execute("INSERT INTO Ops(Name) IF NOT EXISTS VALUES('{0}');".format(user))
      except sql.Error as e:
         send_message(CHAN, "Unable to add user: {0}".format(e.args[0]))

   get_ops()<commit_msg>testing<commit_after>import auth, random, re, socket, sys, time
import sqlite3 as sql
from markov import Markov


"""
Helper functions.
"""

# Average output generator
def avg (list_):
   x = 0
   for i in list_:
      x += i
   return int(x / len(list_))


"""
Markov cleaning functions.
"""

# Characters that define what lines to ignore for the output
WEECHAT = ['>>', '<<', '--', '-->', '<--', '@Satsuki']
IGNORE = ['.luser', 'Raw', '!user', '.s']

# Counter of how many words are in a message to be averaged later
chat_average = {}

def clean_file (file_):
   global chat_average
   word_count = []
   avg_words = 0

   """
   This will create a file of just user submitted messages.

   Weechat logs in the following format:

   YYYY-MM-DD HH:MM:SS USERNAME	MESSAGE
   YYYY-MM-DD HH:MM:SS << USERNAME (ADDRESS) has quit (REASON)
   """
   chat_name = file_.split('/')[-1].split('.')[2]
   send_message(CHAN, "Just one second while refresh the logs.")
   print('[LOG] Creating clean output file for {0}.'.format(chat_name))

   file_i = open(file_, encoding="utf8").read().splitlines()
   file_o = open('clean_log_{0}'.format(chat_name), 'w', encoding="utf8")

   for line in file_i:
      try:
         msg = line.split('	') # Splits to [time / name / message] etc
         words = msg[2].split()  # Splits the message into words
         try:
            if msg[1] not in WEECHAT:
               if len(msg[2].split('::')) == 1:
                  if words[0] not in IGNORE:
                     if words[0].split(':')[0] not in ['http', 'https']:
                        word_count.append(len(words))
                        words.append('\n')
                        file_o.write(' '.join(words))
         except:
            pass
      except:
         pass
   chat_average[chat_name] = avg(word_count)
   file_o.close()


def refresh (chat_name):
   global chat_average

   clean_file('../../.weechat/logs/irc.animebytes.{0}.weechatlog'.format(chat_name))
   print('[LOG] Chat log file for {0} refreshed.'.format(chat_name))


def gen_markov (chat_name):
   global chat_average

   while True:
      try:
         file_ = open('clean_log_{0}'.format(chat_name), encoding="utf8")
         markov = Markov(file_)
         return markov.generate_markov_text(chat_average[chat_name] + random.randint(0, int(chat_average[chat_name]/2)))
         break
      except FileNotFoundError:
         return "Sorry, either that log does not exist or you forgot to enter a parameter."
      except:
         refresh(chat_name)
   file_.close()


"""
Database functions.
"""

# Initializes ops.
def get_ops ():
   global OPS

   con = sql.connect('living_in_the_.db')

   with con:
      con.row_factory = sql.Row
      db = con.cursor()

      OPS = []
      db.execute('SELECT * FROM Ops;')
      rows = db.fetchall()

      for row in rows:
         OPS.append(row['Name'])


# Adds a new op.
def add_op (user):
   global OPS

   con = sql.connect('living_in_the_.db')

   with con:
      con.row_factory = sql.Row
      db = con.cursor()

      try:
         db.execute("INSERT INTO Ops(Name) WHERE NOT EXISTS VALUES('{0}');".format(user))
      except sql.Error as e:
         send_message(CHAN, "Unable to add user: {0}".format(e.args[0]))

   get_ops()
{'fraction_non_alphanumeric': 0.14191368453386577, 'fraction_numerical': 0.007679311933650745, 'mean_word_length': 2.495437466451959, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '756109', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2118, 'n_words': 583}