<commit_before>#!/usr/bin/python

"""
Forward messages sent to @streams.zulip.com to Zulip.

Messages to that address go to the Inbox of <EMAIL>.

Messages meant for Zulip have a special recipient form of

<stream name>+<regenerable stream token>@<EMAIL>

We extract and validate the target stream from information in the
recipient address and retrieve, forward, and archive the message.

Run this management command out of a cron job.
"""

import email
from os import path
from email.header import decode_header
import logging
import re
import sys

from django.conf import settings
from django.core.management.base import BaseCommand

from zerver.lib.actions import decode_email_address
from zerver.lib.upload import upload_message_image
from zerver.models import Stream, get_user_profile_by_email

from twisted.internet import protocol, reactor, ssl
from twisted.mail import imap4

import html2text

sys.path.insert(0, path.join(path.dirname(__file__), "../../../api"))
import zulip

GATEWAY_EMAIL = "<EMAIL>"
# Application-specific password.
PASSWORD = "<PASSWORD>"

SERVER = "imap.gmail.com"
PORT = 993

## Setup ##

log_file = "/var/log/humbug/email-mirror.log"
log_format = "%(asctime)s: %(message)s"
logging.basicConfig(format=log_format)

formatter = logging.Formatter(log_format)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(file_handler)

email_gateway_user = get_user_profile_by_email(GATEWAY_EMAIL)
api_key = email_gateway_user.api_key
if settings.DEPLOYED:
    staging_client = zulip.Client(
        site="https://staging.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
    prod_client = zulip.Client(
        site="https://api.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
    inbox = "INBOX"
else:
    staging_client = prod_client = zulip.Client(
        site="http://localhost:9991/api", email=GATEWAY_EMAIL, api_key=api_key)
    inbox = "Test"

def redact_stream(error_message):
    stream_match = re.search("(\S+?)\+(\S+?)@streams.<EMAIL>", error_message)
    if stream_match:
        stream_name = stream_match.groups()[0]
        return error_message.replace(stream_name, "X" * len(stream_name))
    return error_message

def report_to_zulip(error_message):
    error_stream = Stream.objects.get(name="errors", realm__domain="zulip.com")
    send_zulip(error_stream, "email mirror error",
               """~~~\n%s\n~~~""" % (error_message,))

def log_and_report(email_message, error_message, debug_info):
    scrubbed_error = "Sender: %s\n%s" % (email_message.get("From"),
                                         redact_stream(error_message))

    if "to" in debug_info:
        scrubbed_error = "Stream: %s\n%s" % (redact_stream(debug_info["to"]),
                                             scrubbed_error)

    if "stream" in debug_info:
        scrubbed_error = "Realm: %s\n%s" % (debug_info["stream"].realm.domain,
                                            scrubbed_error)

    logger.error(scrubbed_error)
    report_to_zulip(scrubbed_error)

## Sending the Zulip ##

class ZulipEmailForwardError(Exception):
    pass

def send_zulip(stream, topic, content):
    if stream.realm.domain in ["zulip.com", "wdaher.com"]:
        api_client = staging_client
    else:
        api_client = prod_client

    # TODO: restrictions on who can send? Consider: cross-realm
    # messages, private streams.
    message_data = {
        "type": "stream",
        # TODO: handle rich formatting.
        "content": content[:2000],
        "subject": topic[:60],
        "to": stream.name,
        "domain": stream.realm.domain
        }

    response = api_client.send_message(message_data)
    if response["result"] != "success":
        raise ZulipEmailForwardError(response["msg"])

def valid_stream(stream_name, token):
    try:
        stream = Stream.objects.get(email_token=token)
        return stream.name.lower() == stream_name.lower()
    except Stream.DoesNotExist:
        return False

def get_message_part_by_type(message, content_type):
    charset = message.get_content_charset()

    for part in message.walk():
        if part.get_content_type() == content_type:
            content = part.get_payload(decode=True)
            if charset:
                content = content.decode(charset)
            return content

def extract_body(message):
    # If the message contains a plaintext version of the body, use
    # that.
    plaintext_content = get_message_part_by_type(message, "text/plain")
    if plaintext_content:
        return plaintext_content

    # If we only have an HTML version, try to make that look nice.
    html_content = get_message_part_by_type(message, "text/html")
    if html_content:
        converter = html2text.HTML2Text()
        converter.ignore_links = True
        converter.ignore_images = True
        return converter.handle(html_content)<commit_msg>email mirror: use a message part's specific charset rather than the message charset.<commit_after>#!/usr/bin/python

"""
Forward messages sent to @streams.zulip.com to Zulip.

Messages to that address go to the Inbox of <EMAIL>.

Messages meant for Zulip have a special recipient form of

<stream name>+<regenerable stream token>@streams.zulip.com

We extract and validate the target stream from information in the
recipient address and retrieve, forward, and archive the message.

Run this management command out of a cron job.
"""

import email
from os import path
from email.header import decode_header
import logging
import re
import sys

from django.conf import settings
from django.core.management.base import BaseCommand

from zerver.lib.actions import decode_email_address
from zerver.lib.upload import upload_message_image
from zerver.models import Stream, get_user_profile_by_email

from twisted.internet import protocol, reactor, ssl
from twisted.mail import imap4

import html2text

sys.path.insert(0, path.join(path.dirname(__file__), "../../../api"))
import zulip

GATEWAY_EMAIL = "<EMAIL>"
# Application-specific password.
PASSWORD = "<PASSWORD>"

SERVER = "imap.gmail.com"
PORT = 993

## Setup ##

log_file = "/var/log/humbug/email-mirror.log"
log_format = "%(asctime)s: %(message)s"
logging.basicConfig(format=log_format)

formatter = logging.Formatter(log_format)
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logger.addHandler(file_handler)

email_gateway_user = get_user_profile_by_email(GATEWAY_EMAIL)
api_key = email_gateway_user.api_key
if settings.DEPLOYED:
    staging_client = zulip.Client(
        site="https://staging.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
    prod_client = zulip.Client(
        site="https://api.zulip.com", email=GATEWAY_EMAIL, api_key=api_key)
    inbox = "INBOX"
else:
    staging_client = prod_client = zulip.Client(
        site="http://localhost:9991/api", email=GATEWAY_EMAIL, api_key=api_key)
    inbox = "Test"

def redact_stream(error_message):
    stream_match = re.search("(\S+?)\+(\S+?)@<EMAIL>", error_message)
    if stream_match:
        stream_name = stream_match.groups()[0]
        return error_message.replace(stream_name, "X" * len(stream_name))
    return error_message

def report_to_zulip(error_message):
    error_stream = Stream.objects.get(name="errors", realm__domain="zulip.com")
    send_zulip(error_stream, "email mirror error",
               """~~~\n%s\n~~~""" % (error_message,))

def log_and_report(email_message, error_message, debug_info):
    scrubbed_error = "Sender: %s\n%s" % (email_message.get("From"),
                                         redact_stream(error_message))

    if "to" in debug_info:
        scrubbed_error = "Stream: %s\n%s" % (redact_stream(debug_info["to"]),
                                             scrubbed_error)

    if "stream" in debug_info:
        scrubbed_error = "Realm: %s\n%s" % (debug_info["stream"].realm.domain,
                                            scrubbed_error)

    logger.error(scrubbed_error)
    report_to_zulip(scrubbed_error)

## Sending the Zulip ##

class ZulipEmailForwardError(Exception):
    pass

def send_zulip(stream, topic, content):
    if stream.realm.domain in ["zulip.com", "wdaher.com"]:
        api_client = staging_client
    else:
        api_client = prod_client

    # TODO: restrictions on who can send? Consider: cross-realm
    # messages, private streams.
    message_data = {
        "type": "stream",
        # TODO: handle rich formatting.
        "content": content[:2000],
        "subject": topic[:60],
        "to": stream.name,
        "domain": stream.realm.domain
        }

    response = api_client.send_message(message_data)
    if response["result"] != "success":
        raise ZulipEmailForwardError(response["msg"])

def valid_stream(stream_name, token):
    try:
        stream = Stream.objects.get(email_token=token)
        return stream.name.lower() == stream_name.lower()
    except Stream.DoesNotExist:
        return False

def get_message_part_by_type(message, content_type):
    charsets = message.get_charsets()

    for idx, part in enumerate(message.walk()):
        if part.get_content_type() == content_type:
            content = part.get_payload(decode=True)
            if charsets[idx]:
                content = content.decode(charsets[idx])
            return content

def extract_body(message):
    # If the message contains a plaintext version of the body, use
    # that.
    plaintext_content = get_message_part_by_type(message, "text/plain")
    if plaintext_content:
        return plaintext_content

    # If we only have an HTML version, try to make that look nice.
    html_content = get_message_part_by_type(message, "text/html")
    if html_content:
        converter = html2text.HTML2Text()
        converter.ignore_links = True
        converter.ignore_images = True
        return converter.handle(html_content)
{'fraction_non_alphanumeric': 0.10693207962388716, 'fraction_numerical': 0.0038011403421026307, 'mean_word_length': 3.483408071748879, 'pattern_counts': {'":': 12, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5552626', 'n_tokens_mistral': 3170, 'n_tokens_neox': 3120, 'n_words': 724}