<commit_before>from __future__ import division
import codecs
import date
import re
import operator
import sys
import json


class Chat():
    def __init__(self, filename):
        self.filename = filename
        self.raw_messages = []

        self.datelist = []
        self.timelist = []
        self.senderlist = []
        self.messagelist = []

    def open_file(self):
        arq = codecs.open(self.filename, "r", "utf-8-sig")
        content = arq.read()
        arq.close()
        lines = content.split("\n")
        lines = [l for l in lines if len(l) != 1]
        for l in lines:
            self.raw_messages.append(l.encode("utf-8"))

    def feed_lists(self):
        for l in self.raw_messages:
            search_obj = re.search(r'[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}', l)
            if search_obj:
                columns = l.split(": ")
                date, time = columns[0].split(" ")
                sender = columns[1]
                message = columns[2]
            else:
                message = l
            self.datelist.append(date)
            self.timelist.append(time)
            self.senderlist.append(sender)
            self.messagelist.append(message)
<commit_msg>Simplify feed_lists() method<commit_after>from __future__ import division
import codecs
import date
import re
import operator
import sys
import json


class Chat():
    def __init__(self, filename):
        self.filename = filename
        self.raw_messages = []

        self.datelist = []
        self.timelist = []
        self.senderlist = []
        self.messagelist = []

    def open_file(self):
        arq = codecs.open(self.filename, "r", "utf-8-sig")
        content = arq.read()
        arq.close()
        lines = content.split("\n")
        lines = [l for l in lines if len(l) != 1]
        for l in lines:
            self.raw_messages.append(l.encode("utf-8"))

    def feed_lists(self):
        for l in self.raw_messages:
            msg_date, sep, msg = l.partition(": ")
            raw_date, sep, time = msg_date.partition(" ")
            sender, sep, message = msg.partition(": ")

            if message:
                self.datelist.append(raw_date)
                self.timelist.append(time)
                self.senderlist.append(sender)
                self.messagelist.append(message)
            else:
                self.messagelist.append(l)

{'fraction_non_alphanumeric': 0.12074042911232646, 'fraction_numerical': 0.008413967185527976, 'mean_word_length': 1.8108747044917257, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3235980', 'n_tokens_mistral': 741, 'n_tokens_neox': 700, 'n_words': 161}