<commit_before>import re
import pwd
import logging
import time
import os


class LogReader(object):
    output_directory = '/tmp'

    @classmethod
    def set_output_directory(cls, path):
        cls.output_directory = path

    @classmethod
    def get_info(cls, user):
        """
        >>> pwd.getpwnam('root') is not None
        True
        """
        try:
            return pwd.getpwnam(user)
        except KeyError:
            return None

    @classmethod
    def check_line(cls, line):
        raise NotImplementedError


class CombinedLogReader(LogReader):
    """Parse Apache logs in the combined format.
    "%h %l %u %t \\"%r\\" %>s %b \\"%{Referer}i\\" \\"%{User-agent}i\\"
    """

    # 1 = Remote host
    # 2 = remote ident
    # 3 = ?
    # 4 = date
    # 5 = HTTP Request, (GET, POST, etc.)
    # 6 = URI
    # 7 = HTTP version
    # 8 = Rest
    REGEX = (r'^([^ ]+) ([^ ]+) ([^ ]+) \[([^]]+)\] '
             r'"([^ ]+) ([^ ]+) ([^ ]+)" ?(.*)$')
    REGEX_C = re.compile(REGEX)
    prefixes = ['/~']

    @classmethod
    def add_prefix(cls, *args):
        """
        >>> cls = CombinedLogReader
        >>> '/~' in cls.prefixes
        True
        >>> cls.add_prefix('abc', 'xyz')
        >>> 'abc' in cls.prefixes
        True
        >>> 'xyz' in cls.prefixes
        True
        """
        cls.prefixes += args

    @classmethod
    def check_line(cls, line):
        uri = cls.parse_uri(line)
        if uri is not None:
            user = cls.parse_user(uri)
            if user is not None:
                return cls.get_info(user)

    @classmethod
    def parse_uri(cls, line):
        m = cls.REGEX_C.match(line)
        if m is not None:
            uri = m.group(6)
            return uri

    @classmethod
    def parse_user(cls, uri):
        """
        >>> clr = CombinedLogReader()
        >>> clr.parse_user('')
        >>> clr.parse_user('/')
        >>> clr.parse_user('/otherstuff')
        >>> clr.parse_user('/~abc')
        'abc'
        >>> clr.parse_user('/~abc/index.html')
        'abc'
        >>> clr.parse_user('/home/abc')
        >>> clr.parse_user('/home/abc/index.html')
        >>> clr.add_prefix('/home/')
        >>> clr.parse_user('/home/abc')
        'abc'
        >>> clr.parse_user('/home/abc/index.html')
        'abc'
        """
        for p in cls.prefixes:
            if uri.startswith(p):
                s = uri[len(p):]
                return s.split('/', 1)[0]


class ErrorLogReader(LogReader):
    HOMEDIR_ROOT = '/home'
    HOMEDIR_ROOT_LEN = len(HOMEDIR_ROOT)

    @classmethod
    def configure(cls, **kwargs):
        """
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/home'
        >>> ErrorLogReader.configure()
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/home'
        >>> ErrorLogReader.configure(homedir_root='/other/home')
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/other/home'
        """
        try:
            cls.HOMEDIR_ROOT = kwargs['homedir_root']
            cls.HOMEDIR_ROOT_LEN = len(cls.HOMEDIR_ROOT)
        except KeyError:
            pass

    @classmethod
    def check_line(cls, line):
        index = line.find(HOMEDIR_ROOT)
        if index < 0:
            return

        index += HOMEDIR_ROOT_LEN
        user = line[index:].split('/', 1)[0].split(' ')[0]
        return cls.get_info(user)


if __name__ == '__main__':
    import doctest
    doctest.testmod()<commit_msg>Add more doctests, fix typo<commit_after>import re
import pwd
import logging
import time
import os


class LogReader(object):
    output_directory = '/tmp'

    @classmethod
    def set_output_directory(cls, path):
        cls.output_directory = path

    @classmethod
    def get_info(cls, user):
        """
        >>> pwd.getpwnam('root') is not None
        True
        """
        try:
            return pwd.getpwnam(user)
        except KeyError:
            return None

    @classmethod
    def check_line(cls, line):
        raise NotImplementedError


class CombinedLogReader(LogReader):
    """Parse Apache logs in the combined format.
    "%h %l %u %t \\"%r\\" %>s %b \\"%{Referer}i\\" \\"%{User-agent}i\\"
    """

    # 1 = Remote host
    # 2 = remote ident
    # 3 = ?
    # 4 = date
    # 5 = HTTP Request, (GET, POST, etc.)
    # 6 = URI
    # 7 = HTTP version
    # 8 = Rest
    REGEX = (r'^([^ ]+) ([^ ]+) ([^ ]+) \[([^]]+)\] '
             r'"([^ ]+) ([^ ]+) ([^ ]+)" ?(.*)$')
    REGEX_C = re.compile(REGEX)
    prefixes = ['/~']

    @classmethod
    def add_prefix(cls, *args):
        """
        >>> cls = CombinedLogReader
        >>> '/~' in cls.prefixes
        True
        >>> cls.add_prefix('abc', 'xyz')
        >>> 'abc' in cls.prefixes
        True
        >>> 'xyz' in cls.prefixes
        True
        """
        cls.prefixes += args

    @classmethod
    def check_line(cls, line):
        uri = cls.parse_uri(line)
        if uri is not None:
            user = cls.parse_user(uri)
            if user is not None:
                return cls.get_info(user)

    @classmethod
    def parse_uri(cls, line):
        m = cls.REGEX_C.match(line)
        if m is not None:
            uri = m.group(6)
            return uri

    @classmethod
    def parse_user(cls, uri):
        """
        >>> clr = CombinedLogReader()
        >>> clr.parse_user('')
        >>> clr.parse_user('/')
        >>> clr.parse_user('/otherstuff')
        >>> clr.parse_user('/~abc')
        'abc'
        >>> clr.parse_user('/~abc/index.html')
        'abc'
        >>> clr.parse_user('/home/abc')
        >>> clr.parse_user('/home/abc/index.html')
        >>> clr.add_prefix('/home/')
        >>> clr.parse_user('/home/abc')
        'abc'
        >>> clr.parse_user('/home/abc/index.html')
        'abc'
        """
        for p in cls.prefixes:
            if uri.startswith(p):
                s = uri[len(p):]
                return s.split('/', 1)[0]


class ErrorLogReader(LogReader):
    HOMEDIR_ROOT = '/home/'
    HOMEDIR_ROOT_LEN = len(HOMEDIR_ROOT)

    @classmethod
    def configure(cls, **kwargs):
        """
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/home/'
        >>> ErrorLogReader.configure()
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/home/'
        >>> ErrorLogReader.configure(homedir_root='/other/home')
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/other/home/'
        >>> ErrorLogReader.configure(homedir_root='/the/other/home/')
        >>> ErrorLogReader.HOMEDIR_ROOT
        '/the/other/home/'
        """
        try:
            abs = os.path.abspath(kwargs['homedir_root'])
            cls.HOMEDIR_ROOT = '%s/' % abs
            cls.HOMEDIR_ROOT_LEN = len(cls.HOMEDIR_ROOT)
        except KeyError:
            pass

    @classmethod
    def check_line(cls, line):
        """
        >>> ErrorLogReader.check_line('blah /home/root/blah') is not None
        True
        >>> ErrorLogReader.check_line('blah /home/root blah') is not None
        True
        """
        index = line.find(cls.HOMEDIR_ROOT)
        if index < 0:
            return

        index += cls.HOMEDIR_ROOT_LEN
        user = line[index:].split(' ', 1)[0].split('/', 1)[0]
        return cls.get_info(user)


if __name__ == '__main__':
    import doctest
    doctest.testmod()
{'fraction_non_alphanumeric': 0.1578874602516245, 'fraction_numerical': 0.004285911793170193, 'mean_word_length': 1.7919722115013508, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 149, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4001388', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2406, 'n_words': 472}