<commit_before>#!/usr/bin/env python3

import datetime
import http
import io
import os
import pickle
import re

import dateutil.parser
import requests


API_ENDPOINT = 'https://p99tunnel.com/upload_log'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROCESSED_LINES_PATH = os.path.join(SCRIPT_DIR, '.processed-lines')
CACHED_LOG_DIR_PATH = os.path.join(SCRIPT_DIR, '.log-dir')

LOG_NAMES_REGEX = re.compile(r'eqlog_(.*)_project1999.txt')
TIMESTAMP_REGEX = re.compile(r'^\[[^ ]+ ([^]]+)]')
MY_AUCTION_REGEX = re.compile(r"^\[[^ ]+ [^]]+] You auction, '.+'$")
OTHER_AUCTION_REGEX = re.compile(r"^\[[^ ]+ [^]]+] [^ ]+ auctions, '.+'$")


class NamedStream(object):

  def __init__(self, name, stream):
    self.name = name
    self.stream = stream

  def __repr__(self):
    return self.name

  def readline(self):
    return self.stream.readline()


class ProcessedLines(object):

  def __init__(self, path=PROCESSED_LINES_PATH):
    self.already_processed = {}
    if os.path.isfile(path):
      with open(path, 'r') as f:
        self.already_processed = pickle.load(f)

  def update_in_memory(self, stream, last_line):
    self.already_processed[stream.name] = last_line

  def save_to_disk(self, path=PROCESSED_LINES_PATH):
    with open(path, 'w') as f:
      pickle.dump(self.already_processed, f)

  def get(self, stream_name):
    return self.already_processed.get(stream_name)


def get_log_directory():
  print('Getting log directory...')
  if os.path.isfile(CACHED_LOG_DIR_PATH):
    with open(CACHED_LOG_DIR_PATH, 'r') as f:
      path = f.read()
      print('  Log directory: ' + path)
      return path
  path = input('Please paste in the path to your log directory: ').strip()
  path = os.path.expanduser(path)
  assert os.path.isdir(path)
  with open(CACHED_LOG_DIR_PATH, 'w') as f:
    print('  Caching log directory...')
    f.write(path)
    return path


def get_log_streams(log_dir):
  print('Opening up log streams...')
  names = os.listdir(log_dir)
  streams = []
  for name in names:
    match = LOG_NAMES_REGEX.match(name)
    if match:
      print('  Opening a stream for: ' + name)
      character_name = match.group(1)
      path = os.path.join(log_dir, name)
      streams.append(NamedStream(name=character_name, stream=io.open(path)))
    else:
      print('  Not a p99 log: ' + name)
  return streams


def get_time(log_line):
  match = TIMESTAMP_REGEX.match(log_line)
  timestamp_str = log_line.group(1)
  return dateutil.parser.parse(timestamp_str)


def consume_up_to(stream, last_line):
  print('Getting to the new stuff in stream: ' + stream.name)
  if last_line is None:
    return
  last_time = get_time(last_line)
  last_old_offset = stream.tell()
  while True:
    line = stream.readline()
    # We got to the end of the file without finding a time after the last line
    if not line:
      return
    time = get_time(line)
    if time > last_time:
      stream.seek(last_old_offset)
      return
    last_old_offset = stream.tell()


def get_local_time_str():
  return datetime.datetime.now().isoformat()


def upload_auction(auction):
  now_str = get_local_time_str()
  post_body = '{} {}'.format(now_str, auction)
  response = requests.post(API_ENDPOINT, data=post_body)
  if response.status_code != 200:
    print('Bad response: ', response)


def consume_log_output(stream):
  while True:
    line = stream.readline()
    # The stream doesn't have any more output for us to consume
    if not line:
      return
    # I auctioned something, which means I need to reformat the line to look
    # like a generic auction
    match = MY_AUCTION_REGEX.match(line)
    if match:
      line = line.replace(
        ' You auction, ', ' {} auctions, '.format(stream.name))
    # The line is a valid auction
    match = OTHER_AUCTION_REGEX.match(line)
    if match:
      upload_auction(line)


def main():
  log_dir = get_log_directory()
  log_streams = get_log_streams(log_dir)
  processed_lines = ProcessedLines()
  for stream in log_streams:
    consume_up_to(stream, processed_lines.get(stream.name))
  print('Streaming log updates...')
  while True:
    for stream in log_streams:
      last_line = consume_log_output(stream)
      processed_lines.update_in_memory(stream, last_line)
    processed_lines.save_to_disk()
    time.sleep(10)
<commit_msg>Add missing imports; add missing methods; pickle in binary; pickle with human readable protocol; fix typo; return the last line of consumed log output; don't clobber the list of processed lines<commit_after>#!/usr/bin/env python3

import datetime
import http
import io
import os
import pickle
import re
import time

import dateutil.parser
import requests


API_ENDPOINT = 'https://p99tunnel.com/upload_log'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PROCESSED_LINES_PATH = os.path.join(SCRIPT_DIR, '.processed-lines')
CACHED_LOG_DIR_PATH = os.path.join(SCRIPT_DIR, '.log-dir')

LOG_NAMES_REGEX = re.compile(r'eqlog_(.*)_project1999.txt')
TIMESTAMP_REGEX = re.compile(r'^\[[^ ]+ ([^]]+)]')
MY_AUCTION_REGEX = re.compile(r"^\[[^ ]+ [^]]+] You auction, '.+'$")
OTHER_AUCTION_REGEX = re.compile(r"^\[[^ ]+ [^]]+] [^ ]+ auctions, '.+'$")


class NamedStream(object):

  def __init__(self, name, stream):
    self.name = name
    self.stream = stream

  def __repr__(self):
    return self.name

  def readline(self):
    return self.stream.readline()

  def tell(self):
    return self.stream.tell()

  def seek(self, offset):
    return self.stream.seek(offset)


class ProcessedLines(object):

  def __init__(self, path=PROCESSED_LINES_PATH):
    self.already_processed = {}
    if os.path.isfile(path):
      with open(path, 'rb') as f:
        self.already_processed = pickle.load(f)
        print('Starting at these lines for these characters:')
        print(str(self.already_processed))

  def update_in_memory(self, stream, last_line):
    self.already_processed[stream.name] = last_line

  def save_to_disk(self, path=PROCESSED_LINES_PATH):
    with open(path, 'wb') as f:
      pickle.dump(self.already_processed, f, protocol=0)

  def get(self, stream_name):
    return self.already_processed.get(stream_name)


def get_log_directory():
  print('Getting log directory...')
  if os.path.isfile(CACHED_LOG_DIR_PATH):
    with open(CACHED_LOG_DIR_PATH, 'r') as f:
      path = f.read()
      print('  Log directory: ' + path)
      return path
  path = input('Please paste in the path to your log directory: ').strip()
  path = os.path.expanduser(path)
  assert os.path.isdir(path)
  with open(CACHED_LOG_DIR_PATH, 'w') as f:
    print('  Caching log directory...')
    f.write(path)
    return path


def get_log_streams(log_dir):
  print('Opening up log streams...')
  names = os.listdir(log_dir)
  streams = []
  for name in names:
    match = LOG_NAMES_REGEX.match(name)
    if match:
      print('  Opening a stream for: ' + name)
      character_name = match.group(1)
      path = os.path.join(log_dir, name)
      streams.append(NamedStream(name=character_name, stream=io.open(path)))
    else:
      print('  Not a p99 log: ' + name)
  return streams


def get_time(log_line):
  match = TIMESTAMP_REGEX.match(log_line)
  timestamp_str = match.group(1)
  return dateutil.parser.parse(timestamp_str)


def consume_up_to(stream, last_line):
  print('Getting to the new stuff in stream: ' + stream.name)
  if last_line is None:
    return
  last_time = get_time(last_line)
  last_old_offset = stream.tell()
  while True:
    line = stream.readline()
    # We got to the end of the file without finding a time after the last line
    if not line:
      return
    time = get_time(line)
    if time > last_time:
      stream.seek(last_old_offset)
      return
    last_old_offset = stream.tell()


def get_local_time_str():
  return datetime.datetime.now().isoformat()


def upload_auction(auction):
  now_str = get_local_time_str()
  post_body = '{} {}'.format(now_str, auction)
  response = requests.post(API_ENDPOINT, data=post_body)
  if response.status_code != 200:
    print('Bad response: ', response)


def consume_log_output(stream, last_line):
  while True:
    line = stream.readline()
    # The stream doesn't have any more output for us to consume
    if not line:
      return last_line
    last_line = line
    # I auctioned something, which means I need to reformat the line to look
    # like a generic auction
    match = MY_AUCTION_REGEX.match(line)
    if match:
      line = line.replace(
        ' You auction, ', ' {} auctions, '.format(stream.name))
    # The line is a valid auction
    match = OTHER_AUCTION_REGEX.match(line)
    if match:
      upload_auction(line)


def main():
  log_dir = get_log_directory()
  log_streams = get_log_streams(log_dir)
  processed_lines = ProcessedLines()
  for stream in log_streams:
    consume_up_to(stream, processed_lines.get(stream.name))
  print('Streaming log updates...')
  while True:
    for stream in log_streams:
      last_line = consume_log_output(stream, processed_lines.get(stream.name))
      processed_lines.update_in_memory(stream, last_line)
    processed_lines.save_to_disk()
    time.sleep(10)

{'fraction_non_alphanumeric': 0.11907641561297416, 'fraction_numerical': 0.003628367234744365, 'mean_word_length': 3.7473903966597075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3714265', 'n_tokens_mistral': 3203, 'n_tokens_neox': 3102, 'n_words': 751}