Python email.message.EmailMessage() Examples

def handle(self, **options: Optional[str]) -> None:
        if options['fixture'] is None:
            self.print_help('./manage.py', 'send_to_email_mirror')
            raise CommandError

        if options['stream'] is None:
            stream = "Denmark"
        else:
            stream = options['stream']

        realm = self.get_realm(options)
        if realm is None:
            realm = get_realm("zulip")

        full_fixture_path = os.path.join(settings.DEPLOY_ROOT, options['fixture'])

        # parse the input email into EmailMessage type and prepare to process_message() it
        message = self._parse_email_fixture(full_fixture_path)
        self._prepare_message(message, realm, stream)

        mirror_email_message(
            message['To'].addresses[0].addr_spec,
            base64.b64encode(message.as_bytes()).decode(),
        ) 

def process_message(message: EmailMessage, rcpt_to: Optional[str]=None) -> None:
    to: Optional[str] = None

    try:
        if rcpt_to is not None:
            to = rcpt_to
        else:
            to = find_emailgateway_recipient(message)

        if is_missed_message_address(to):
            process_missed_message(to, message)
        else:
            process_stream_message(to, message)
    except ZulipEmailForwardUserError as e:
        # TODO: notify sender of error, retry if appropriate.
        logging.warning(e.args[0])
    except ZulipEmailForwardError as e:
        log_and_report(message, e.args[0], to) 

def put(self, device_id):
        self._authenticate(device_id)
        payload = json.loads(self.request.data)
        send_email = SendEmail(payload)
        send_email.validate()

        account = AccountRepository(self.db).get_account_by_device_id(device_id)

        if account:
            message = EmailMessage()
            message['Subject'] = str(send_email.title)
            message['From'] = str(send_email.sender)
            message.set_content(str(send_email.body))
            message['To'] = account.email_address
            self._send_email(message)
            response = '', HTTPStatus.OK
        else:
            response = '', HTTPStatus.NO_CONTENT
        return response 

def test_flatten_message():
    message = EmailMessage()
    message["To"] = "bob@example.com"
    message["Subject"] = "Hello, World."
    message["From"] = "alice@example.com"
    message.set_content("This is a test")

    flat_message = flatten_message(message)

    expected_message = b"""To: bob@example.com\r
Subject: Hello, World.\r
From: alice@example.com\r
Content-Type: text/plain; charset="utf-8"\r
Content-Transfer-Encoding: 7bit\r
MIME-Version: 1.0\r
\r
This is a test\r
"""
    assert flat_message == expected_message 

def test_extract_recipients_resent_message():
    message = EmailMessage()
    message["To"] = "bob@example.com"
    message["Cc"] = "claire@example.com"
    message["Bcc"] = "dustin@example.com"

    message["Resent-Date"] = "Mon, 20 Nov 2017 21:04:27 -0000"
    message["Resent-To"] = "eliza@example.com"
    message["Resent-Cc"] = "fred@example.com"
    message["Resent-Bcc"] = "gina@example.com"

    recipients = extract_recipients(message)

    assert message["Resent-To"] in recipients
    assert message["Resent-Cc"] in recipients
    assert message["Resent-Bcc"] in recipients
    assert message["To"] not in recipients
    assert message["Cc"] not in recipients
    assert message["Bcc"] not in recipients 

def test_receive_stream_email_messages_blank_subject_success(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)

        stream_to_address = encode_email_address(stream)

        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('TestStreamEmailMessages Body')

        incoming_valid_message['Subject'] = ''
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)

        # Hamlet is subscribed to this stream so should see the email message from Othello.
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "TestStreamEmailMessages Body")
        self.assertEqual(get_display_recipient(message.recipient), stream.name)
        self.assertEqual(message.topic_name(), "(no topic)") 

def test_office365_auth_send():
    message = EmailMessage()
    message["From"] = "user@mydomain.com"
    message["To"] = "somebody@example.com"
    message["Subject"] = "Hello World!"
    message.set_content("Sent via aiosmtplib")

    with pytest.raises(SMTPAuthenticationError):
        await send(
            message,
            hostname="smtp.office365.com",
            port=587,
            start_tls=True,
            password="test",
            username="test",
        ) 

def test_receive_private_stream_email_messages_success(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.make_stream("private_stream", invite_only=True)
        self.subscribe(user_profile, "private_stream")
        stream = get_stream("private_stream", user_profile.realm)

        stream_to_address = encode_email_address(stream)

        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('TestStreamEmailMessages Body')

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)

        # Hamlet is subscribed to this stream so should see the email message from Othello.
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "TestStreamEmailMessages Body")
        self.assertEqual(get_display_recipient(message.recipient), stream.name)
        self.assertEqual(message.topic_name(), incoming_valid_message['Subject']) 

def test_receive_stream_email_show_sender_success(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)

        stream_to_address = encode_email_address(stream)
        parts = stream_to_address.split('@')
        parts[0] += "+show-sender"
        stream_to_address = '@'.join(parts)

        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('TestStreamEmailMessages Body')
        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "From: {}\n{}".format(self.example_email('hamlet'),
                                                                "TestStreamEmailMessages Body"))
        self.assertEqual(get_display_recipient(message.recipient), stream.name)
        self.assertEqual(message.topic_name(), incoming_valid_message['Subject']) 

def test_receive_stream_email_show_sender_utf8_encoded_sender(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)

        stream_to_address = encode_email_address(stream)
        parts = stream_to_address.split('@')
        parts[0] += "+show-sender"
        stream_to_address = '@'.join(parts)

        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('TestStreamEmailMessages Body')
        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = 'Test =?utf-8?b?VXNlcsOzxIXEmQ==?= <=?utf-8?q?hamlet=5F=C4=99?=@zulip.com>'
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "From: {}\n{}".format('Test Useróąę <hamlet_ę@zulip.com>',
                                                                "TestStreamEmailMessages Body"))
        self.assertEqual(get_display_recipient(message.recipient), stream.name)
        self.assertEqual(message.topic_name(), incoming_valid_message['Subject']) 

def test_message_with_invalid_attachment(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_to_address = encode_email_address(stream)

        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content("Test body")
        # Create an invalid attachment:
        attachment_msg = MIMEPart()
        attachment_msg.add_header('Content-Disposition', 'attachment', filename="some_attachment")
        incoming_valid_message.add_attachment(attachment_msg)

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        with mock.patch('zerver.lib.email_mirror.logger.warning') as mock_warn:
            process_message(incoming_valid_message)
            mock_warn.assert_called_with(
                "Payload is not bytes (invalid attachment %s in message from %s).",
                'some_attachment', self.example_email('hamlet'),
            ) 

def test_receive_stream_email_messages_empty_body(self) -> None:
        # build dummy messages for stream
        # test message with empty body is not sent
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_to_address = encode_email_address(stream)

        # empty body
        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('')

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        with mock.patch('zerver.lib.email_mirror.logging.warning') as mock_warn:
            process_message(incoming_valid_message)
            mock_warn.assert_called_with("Email has no nonempty body sections; ignoring.") 

def test_receive_stream_email_messages_empty_body_after_stripping(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)

        stream_to_address = encode_email_address(stream)
        headers = {}
        headers['Reply-To'] = self.example_email('othello')

        # empty body
        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('-- \nFooter')

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "(No email body)") 

def test_smtputf8_policy(self):
        msg = EmailMessage()
        msg['From'] = "Páolo <főo@bar.com>"
        msg['To'] = 'Dinsdale'
        msg['Subject'] = 'Nudge nudge, wink, wink \u1F609'
        msg.set_content("oh là là, know what I mean, know what I mean?")
        expected = textwrap.dedent("""\
            From: Páolo <főo@bar.com>
            To: Dinsdale
            Subject: Nudge nudge, wink, wink \u1F609
            Content-Type: text/plain; charset="utf-8"
            Content-Transfer-Encoding: 8bit
            MIME-Version: 1.0

            oh là là, know what I mean, know what I mean?
            """).encode('utf-8').replace(b'\n', b'\r\n')
        s = io.BytesIO()
        g = BytesGenerator(s, policy=policy.SMTPUTF8)
        g.flatten(msg)
        self.assertEqual(s.getvalue(), expected) 

def test_process_message_strips_subject(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_to_address = encode_email_address(stream)
        incoming_valid_message = EmailMessage()
        incoming_valid_message.set_content('TestStreamEmailMessages Body')
        incoming_valid_message['Subject'] = "Re: Fwd: Re: Test"
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)
        self.assertEqual("Test", message.topic_name())

        # If after stripping we get an empty subject, it should get set to (no topic)
        del incoming_valid_message['Subject']
        incoming_valid_message['Subject'] = "Re: Fwd: Re: "
        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)
        self.assertEqual("(no topic)", message.topic_name()) 

def test_charset_not_specified(self) -> None:
        message_as_string = self.fixture_data('1.txt', type='email')
        message_as_string = message_as_string.replace("Content-Type: text/plain; charset=\"us-ascii\"",
                                                      "Content-Type: text/plain")
        incoming_message = message_from_string(message_as_string, policy=email.policy.default)
        assert isinstance(incoming_message, EmailMessage)  # https://github.com/python/typeshed/issues/2417

        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_to_address = encode_email_address(stream)

        del incoming_message['To']
        incoming_message['To'] = stream_to_address
        process_message(incoming_message)
        message = most_recent_message(user_profile)

        self.assertEqual(message.content, "Email fixture 1.txt body") 

def test_receive_stream_email_messages_no_textual_body(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_to_address = encode_email_address(stream)
        # No textual body
        incoming_valid_message = EmailMessage()
        with open(os.path.join(settings.DEPLOY_ROOT, "static/images/default-avatar.png"), 'rb') as f:
            incoming_valid_message.add_attachment(
                f.read(),
                maintype="image",
                subtype="png",
            )

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_to_address
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        with mock.patch('zerver.lib.email_mirror.logging.warning') as mock_warn:
            process_message(incoming_valid_message)
            mock_warn.assert_called_with("Unable to find plaintext or HTML message body") 

def construct_zulip_body(message: EmailMessage, realm: Realm, show_sender: bool=False,
                         include_quotes: bool=False, include_footer: bool=False,
                         prefer_text: bool=True) -> str:
    body = extract_body(message, include_quotes, prefer_text)
    # Remove null characters, since Zulip will reject
    body = body.replace("\x00", "")
    if not include_footer:
        body = filter_footer(body)

    if not body.endswith('\n'):
        body += '\n'
    body += extract_and_upload_attachments(message, realm)
    body = body.strip()
    if not body:
        body = '(No email body)'

    if show_sender:
        sender = str(message.get("From", ""))
        body = f"From: {sender}\n{body}"

    return body

## Sending the Zulip ## 

def extract_body(message: EmailMessage, include_quotes: bool=False, prefer_text: bool=True) -> str:
    plaintext_content = extract_plaintext_body(message, include_quotes)
    html_content = extract_html_body(message, include_quotes)

    if plaintext_content is None and html_content is None:
        logging.warning("Content types: %s", [part.get_content_type() for part in message.walk()])
        raise ZulipEmailForwardUserError("Unable to find plaintext or HTML message body")
    if not plaintext_content and not html_content:
        raise ZulipEmailForwardUserError("Email has no nonempty body sections; ignoring.")

    if prefer_text:
        if plaintext_content:
            return plaintext_content
        else:
            assert html_content  # Needed for mypy. Ensured by the validating block above.
            return html_content
    else:
        if html_content:
            return html_content
        else:
            assert plaintext_content  # Needed for mypy. Ensured by the validating block above.
            return plaintext_content 

def test_receive_only_plaintext_with_prefer_html_option(self) -> None:
        user_profile = self.example_user('hamlet')
        self.login_user(user_profile)
        self.subscribe(user_profile, "Denmark")
        stream = get_stream("Denmark", user_profile.realm)
        stream_address_prefer_html = f"Denmark.{stream.email_token}.prefer-html@testserver"

        text = "Test message"
        # This should be correctly identified as empty html body:
        html = "<html><body></body></html>"

        incoming_valid_message = EmailMessage()
        incoming_valid_message.add_alternative(text)
        incoming_valid_message.add_alternative(html, subtype="html")

        incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
        incoming_valid_message['From'] = self.example_email('hamlet')
        incoming_valid_message['To'] = stream_address_prefer_html
        incoming_valid_message['Reply-to'] = self.example_email('othello')

        process_message(incoming_valid_message)
        message = most_recent_message(user_profile)

        # HTML body is empty, so the plaintext content should be picked, despite prefer-html option.
        self.assertEqual(message.content, "Test message") 

def extract_and_upload_attachments(message: EmailMessage, realm: Realm) -> str:
    user_profile = get_system_bot(settings.EMAIL_GATEWAY_BOT)

    attachment_links = []
    for part in message.walk():
        content_type = part.get_content_type()
        filename = part.get_filename()
        if filename:
            attachment = part.get_payload(decode=True)
            if isinstance(attachment, bytes):
                s3_url = upload_message_file(filename, len(attachment), content_type,
                                             attachment,
                                             user_profile,
                                             target_realm=realm)
                formatted_link = f"[{filename}]({s3_url})"
                attachment_links.append(formatted_link)
            else:
                logger.warning("Payload is not bytes (invalid attachment %s in message from %s).",
                               filename, message.get("From"))

    return '\n'.join(attachment_links) 

def find_emailgateway_recipient(message: EmailMessage) -> str:
    # We can't use Delivered-To; if there is a X-Gm-Original-To
    # it is more accurate, so try to find the most-accurate
    # recipient list in descending priority order
    recipient_headers = ["X-Gm-Original-To", "Delivered-To", "Envelope-To",
                         "Resent-To", "Resent-CC", "To", "CC"]

    pattern_parts = [re.escape(part) for part in settings.EMAIL_GATEWAY_PATTERN.split('%s')]
    match_email_re = re.compile(".*?".join(pattern_parts))

    for header_name in recipient_headers:
        for header_value in message.get_all(header_name, []):
            for addr in header_value.addresses:
                if match_email_re.match(addr.addr_spec):
                    return addr.addr_spec

    raise ZulipEmailForwardError("Missing recipient in mirror email") 

def process_stream_message(to: str, message: EmailMessage) -> None:
    subject_header = message.get("Subject", "")
    subject = strip_from_subject(subject_header) or "(no topic)"

    stream, options = decode_stream_email_address(to)
    # Don't remove quotations if message is forwarded, unless otherwise specified:
    if 'include_quotes' not in options:
        options['include_quotes'] = is_forwarded(subject_header)

    body = construct_zulip_body(message, stream.realm, **options)
    send_zulip(
        get_system_bot(settings.EMAIL_GATEWAY_BOT),
        stream, subject, body)
    logger.info(
        "Successfully processed email to %s (%s)",
        stream.name, stream.realm.string_id,
    ) 

def test_extract_recipients_valueerror_on_multiple_resent_message():
    message = EmailMessage()
    message["Resent-Date"] = "Mon, 20 Nov 2016 21:04:27 -0000"
    message["Resent-Date"] = "Mon, 20 Nov 2017 21:04:27 -0000"

    with pytest.raises(ValueError):
        extract_recipients(message) 

def __init__(self, _factory=None, *, policy=compat32):
        """_factory is called with no arguments to create a new message obj

        The policy keyword specifies a policy object that controls a number of
        aspects of the parser's operation.  The default policy maintains
        backward compatibility.

        """
        self.policy = policy
        self._factory_kwds = lambda: {'policy': self.policy}
        if _factory is None:
            # What this should be:
            #self._factory = policy.default_message_factory
            # but, because we are post 3.4 feature freeze, fix with temp hack:
            if self.policy is compat32:
                self._factory = message.Message
            else:
                self._factory = message.EmailMessage
        else:
            self._factory = _factory
            try:
                _factory(policy=self.policy)
            except TypeError:
                # Assume this is an old-style factory
                self._factory_kwds = lambda: {}
        self._input = BufferedSubFile()
        self._msgstack = []
        self._parse = self._parsegen().__next__
        self._cur = None
        self._last = None
        self._headersonly = False

    # Non-public interface for supporting Parser's headersonly flag 

def emit(self, record):
        """
        Emit a record.

        Format the record and send it to the specified addressees.
        """
        try:
            import smtplib
            from email.message import EmailMessage
            import email.utils

            port = self.mailport
            if not port:
                port = smtplib.SMTP_PORT
            smtp = smtplib.SMTP(self.mailhost, port, timeout=self.timeout)
            msg = EmailMessage()
            msg['From'] = self.fromaddr
            msg['To'] = ','.join(self.toaddrs)
            msg['Subject'] = self.getSubject(record)
            msg['Date'] = email.utils.localtime()
            msg.set_content(self.format(record))
            if self.username:
                if self.secure is not None:
                    smtp.ehlo()
                    smtp.starttls(*self.secure)
                    smtp.ehlo()
                smtp.login(self.username, self.password)
            smtp.send_message(msg)
            smtp.quit()
        except Exception:
            self.handleError(record) 

def test_rfc2043_auto_decoded_and_emailmessage_used(self):
        m = message_from_string(textwrap.dedent("""\
            Subject: Ayons asperges pour le =?utf-8?q?d=C3=A9jeuner?=
            From: =?utf-8?q?Pep=C3=A9?= Le Pew <pepe@example.com>
            To: "Penelope Pussycat" <"penelope@example.com">
            MIME-Version: 1.0
            Content-Type: text/plain; charset="utf-8"

            sample text
            """), policy=policy.default)
        self.assertEqual(m['subject'], "Ayons asperges pour le déjeuner")
        self.assertEqual(m['from'], "Pepé Le Pew <pepe@example.com>")
        self.assertIsInstance(m, EmailMessage) 

def test_extract_recipients_invalid_email():
    message = EmailMessage()
    message["Cc"] = "me"

    recipients = extract_recipients(message)

    assert recipients == ["me"] 

def test_extract_sender_resent_message():
    message = EmailMessage()
    message["From"] = "alice@example.com"

    message["Resent-Date"] = "Mon, 20 Nov 2017 21:04:27 -0000"
    message["Resent-From"] = "hubert@example.com"

    sender = extract_sender(message)

    assert sender == message["Resent-From"]
    assert sender != message["From"] 

def test_extract_recipients_includes_bcc():
    message = EmailMessage()
    message["Bcc"] = "alice@example.com"

    recipients = extract_recipients(message)

    assert recipients == [message["Bcc"]]