<commit_before>#!/usr/bin/python
#
## Dear hackers!
##
## It would be great to have more mailbox classes.  They should be derived
## from or implement the same interfaces as Python's native mailboxes, with
## the additional constraint that they support pickling and unpickling using
## cPickle.  The mailbox class is also responsible for generating and parsing
## a "pointer" which should be a short as possible while still encoding the
## info required to locate this message and this message only within the
## larger mailbox.
#
###############################################################################
import cPickle
import email.parser
import errno
import mailbox
import os
import traceback

from mailpile.util import *
from lxml.html.clean import Cleaner

try:
  from GnuPGInterface import GnuPG
  from mailpile.pgpmime import PGPMimeParser
except ImportError:
  GnuPG = PGPMimeParser = None


def ParseMessage(fd, pgpmime=True):
  pos = fd.tell()
  header = [fd.readline()]
  while header[-1] not in ('', '\n', '\r\n'):
    line = fd.readline()
    if line.startswith(' ') or line.startswith('\t'):
      header[-1] += line
    else:
      header.append(line)

  fd.seek(pos)
  if GnuPG and pgpmime:
    message = PGPMimeParser().parse(fd)
  else:
    message = email.parser.Parser().parse(fd)

  message.raw_header = header
  return message


MUA_HEADERS = ('date', 'from', 'to', 'cc', 'subject', 'message-id', 'reply-to',
               'mime-version','content-disposition', 'content-type',
               'user-agent', 'list-id', 'list-subscribe', 'list-unsubscribe',
               'x-ms-tnef-correlator', 'x-ms-has-attach')
DULL_HEADERS = ('in-reply-to', 'references')
def HeaderPrint(message):
  """Generate a fingerprint from message headers which identifies the MUA."""
  headers = [x.split(':', 1)[0] for x in message.raw_header]

  while headers and headers[0].lower() not in MUA_HEADERS:
    headers.pop(0)
  while headers and headers[-1].lower() not in MUA_HEADERS:
    headers.pop(-1)
  headers = [h for h in headers if h.lower() not in DULL_HEADERS]

  return b64w(sha1b64('\n'.join(headers))).lower()


class NoSuchMailboxError(OSError):
  pass


def OpenMailbox(fn):
  if os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'cur')):
    return IncrementalMaildir(fn)
  else:
    return IncrementalMbox(fn)


class IncrementalMaildir(mailbox.Maildir):
  """A Maildir class that supports pickling and a few mailpile specifics."""

  save_to = None
  parsed = {}

  def __setstate__(self, dict):
    self.__dict__.update(dict)
    self.update_toc()

  def save(self, session=None, to=None):
    if to:
      self.save_to = to
    if self.save_to and len(self) > 0:
      if session: session.ui.mark('Saving state to %s' % self.save_to)
      fd = open(self.save_to, 'w')
      cPickle.dump(self, fd)
      fd.close()

  def unparsed(self):
    return [i for i in self.keys() if i not in self.parsed]

  def mark_parsed(self, i):
    self.parsed[i] = True

  def update_toc(self):
    self._refresh()

  def get_msg_size(self, toc_id):
    fd = self.get_file(toc_id)
    fd.seek(0, 2)
    return fd.tell()

  def get_msg_ptr(self, idx, toc_id):
    return '%s%s' % (idx, toc_id)

  def get_file_by_ptr(self, msg_ptr):
    return self.get_file(msg_ptr[3:])


class IncrementalMbox(mailbox.mbox):
  """A mbox class that supports pickling and a few mailpile specifics."""

  last_parsed = 0
  save_to = None

  def __getstate__(self):<commit_msg>initial support for gmvault mailbox format<commit_after>#!/usr/bin/python
#
## Dear hackers!
##
## It would be great to have more mailbox classes.  They should be derived
## from or implement the same interfaces as Python's native mailboxes, with
## the additional constraint that they support pickling and unpickling using
## cPickle.  The mailbox class is also responsible for generating and parsing
## a "pointer" which should be a short as possible while still encoding the
## info required to locate this message and this message only within the
## larger mailbox.
#
###############################################################################
import cPickle
import email.parser
import errno
import mailbox
import os
import traceback
import rfc822
import gzip

from mailpile.util import *
from lxml.html.clean import Cleaner

try:
  from GnuPGInterface import GnuPG
  from mailpile.pgpmime import PGPMimeParser
except ImportError:
  GnuPG = PGPMimeParser = None


def ParseMessage(fd, pgpmime=True):
  pos = fd.tell()
  header = [fd.readline()]
  while header[-1] not in ('', '\n', '\r\n'):
    line = fd.readline()
    if line.startswith(' ') or line.startswith('\t'):
      header[-1] += line
    else:
      header.append(line)

  fd.seek(pos)
  if GnuPG and pgpmime:
    message = PGPMimeParser().parse(fd)
  else:
    message = email.parser.Parser().parse(fd)

  message.raw_header = header
  return message


MUA_HEADERS = ('date', 'from', 'to', 'cc', 'subject', 'message-id', 'reply-to',
               'mime-version','content-disposition', 'content-type',
               'user-agent', 'list-id', 'list-subscribe', 'list-unsubscribe',
               'x-ms-tnef-correlator', 'x-ms-has-attach')
DULL_HEADERS = ('in-reply-to', 'references')
def HeaderPrint(message):
  """Generate a fingerprint from message headers which identifies the MUA."""
  headers = [x.split(':', 1)[0] for x in message.raw_header]

  while headers and headers[0].lower() not in MUA_HEADERS:
    headers.pop(0)
  while headers and headers[-1].lower() not in MUA_HEADERS:
    headers.pop(-1)
  headers = [h for h in headers if h.lower() not in DULL_HEADERS]

  return b64w(sha1b64('\n'.join(headers))).lower()


class NoSuchMailboxError(OSError):
  pass


def OpenMailbox(fn):
  if os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'cur')):
    return IncrementalMaildir(fn)
  elif os.path.isdir(fn) and os.path.exists(os.path.join(fn, 'db')):
    return IncrementalGmvault(fn)
  else:
    return IncrementalMbox(fn)


class IncrementalMaildir(mailbox.Maildir):
  """A Maildir class that supports pickling and a few mailpile specifics."""

  save_to = None
  parsed = {}

  def __setstate__(self, dict):
    self.__dict__.update(dict)
    self.update_toc()

  def save(self, session=None, to=None):
    if to:
      self.save_to = to
    if self.save_to and len(self) > 0:
      if session: session.ui.mark('Saving state to %s' % self.save_to)
      fd = open(self.save_to, 'w')
      cPickle.dump(self, fd)
      fd.close()

  def unparsed(self):
    return [i for i in self.keys() if i not in self.parsed]

  def mark_parsed(self, i):
    self.parsed[i] = True

  def update_toc(self):
    self._refresh()

  def get_msg_size(self, toc_id):
    fd = self.get_file(toc_id)
    fd.seek(0, 2)
    return fd.tell()

  def get_msg_ptr(self, idx, toc_id):
    return '%s%s' % (idx, toc_id)

  def get_file_by_ptr(self, msg_ptr):
    return self.get_file(msg_ptr[3:])

class IncrementalGmvault(IncrementalMaildir):
  """A Gmvault class that supports pickling and a few mailpile specifics."""
  
  def __init__(self, dirname, factory=rfc822.Message, create=True):
    IncrementalMaildir.__init__(self, dirname, factory, create)

    dbpath = os.path.join(self._path, 'db')
    self._paths = dict((os.path.join('db', entry), os.path.join(dbpath, entry)) for entry in os.listdir(dbpath))
    self._toc_mtimes = dict((i, 0) for i in self._paths)

  def get_file(self, key):
    """Return a file-like representation or raise a KeyError."""
    f = gzip.open(os.path.join(self._path, self._lookup(key)), 'rb')
    return mailbox._ProxyFile(f)

  def _refresh(self):
    """Update table of contents mapping."""
    # Refresh toc
    self._toc = {}
    for subdir in self._toc_mtimes:
      path = self._paths[subdir]
      for entry in os.listdir(path):
	if not entry.endswith('.gz'):
	  continue
        p = os.path.join(path, entry)
        if os.path.isdir(p):
          continue
        self._toc[entry] = os.path.join(subdir, entry)
    self._last_read = time.time()

class IncrementalMbox(mailbox.mbox):
  """A mbox class that supports pickling and a few mailpile specifics."""

  last_parsed = 0
  save_to = None

  def __getstate__(self):
{'fraction_non_alphanumeric': 0.14501953125, 'fraction_numerical': 0.0052490234375, 'mean_word_length': 3.6737022247575584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1032669', 'n_tokens_mistral': 2828, 'n_tokens_neox': 2710, 'n_words': 780}