1 # Copyright (C) 2001-2010 Python Software Foundation 2 # Author: Barry Warsaw 3 # Contact: email-sig@python.org 4 5 """Classes to generate plain text from a message object tree.""" 6 7 __all__ = ['Generator', 'DecodedGenerator', 'BytesGenerator'] 8 9 import re 10 import sys 11 import time 12 import random 13 import warnings 14 15 from io import StringIO, BytesIO 16 from email._policybase import compat32 17 from email.header import Header 18 from email.utils import _has_surrogates 19 import email.charset as _charset 20 21 UNDERSCORE = '_' 22 NL = '\n' # XXX: no longer used by the code below. 23 24 fcre = re.compile(r'^From ', re.MULTILINE) 25 26 27 28 class Generator: 29 """Generates output from a Message object tree. 30 31 This basic generator writes the message to the given file object as plain 32 text. 33 """ 34 # 35 # Public interface 36 # 37 38 def __init__(self, outfp, mangle_from_=True, maxheaderlen=None, *, 39 policy=None): 40 """Create the generator for message flattening. 41 42 outfp is the output file-like object for writing the message to. It 43 must have a write() method. 44 45 Optional mangle_from_ is a flag that, when True (the default), escapes 46 From_ lines in the body of the message by putting a `>' in front of 47 them. 48 49 Optional maxheaderlen specifies the longest length for a non-continued 50 header. When a header line is longer (in characters, with tabs 51 expanded to 8 spaces) than maxheaderlen, the header will split as 52 defined in the Header class. Set maxheaderlen to zero to disable 53 header wrapping. The default is 78, as recommended (but not required) 54 by RFC 2822. 55 56 The policy keyword specifies a policy object that controls a number of 57 aspects of the generator's operation. The default policy maintains 58 backward compatibility. 59 60 """ 61 self._fp = outfp 62 self._mangle_from_ = mangle_from_ 63 self.maxheaderlen = maxheaderlen 64 self.policy = policy 65 66 def write(self, s): 67 # Just delegate to the file object 68 self._fp.write(s) 69 70 def flatten(self, msg, unixfrom=False, linesep=None): 71 r"""Print the message object tree rooted at msg to the output file 72 specified when the Generator instance was created. 73 74 unixfrom is a flag that forces the printing of a Unix From_ delimiter 75 before the first object in the message tree. If the original message 76 has no From_ delimiter, a `standard' one is crafted. By default, this 77 is False to inhibit the printing of any From_ delimiter. 78 79 Note that for subobjects, no From_ line is printed. 80 81 linesep specifies the characters used to indicate a new line in 82 the output. The default value is determined by the policy. 83 84 """ 85 # We use the _XXX constants for operating on data that comes directly 86 # from the msg, and _encoded_XXX constants for operating on data that 87 # has already been converted (to bytes in the BytesGenerator) and 88 # inserted into a temporary buffer. 89 policy = msg.policy if self.policy is None else self.policy 90 if linesep is not None: 91 policy = policy.clone(linesep=linesep) 92 if self.maxheaderlen is not None: 93 policy = policy.clone(max_line_length=self.maxheaderlen) 94 self._NL = policy.linesep 95 self._encoded_NL = self._encode(self._NL) 96 self._EMPTY = '' 97 self._encoded_EMTPY = self._encode('') 98 # Because we use clone (below) when we recursively process message 99 # subparts, and because clone uses the computed policy (not None), 100 # submessages will automatically get set to the computed policy when 101 # they are processed by this code. 102 old_gen_policy = self.policy 103 old_msg_policy = msg.policy 104 try: 105 self.policy = policy 106 msg.policy = policy 107 if unixfrom: 108 ufrom = msg.get_unixfrom() 109 if not ufrom: 110 ufrom = 'From nobody ' + time.ctime(time.time()) 111 self.write(ufrom + self._NL) 112 self._write(msg) 113 finally: 114 self.policy = old_gen_policy 115 msg.policy = old_msg_policy 116 117 def clone(self, fp): 118 """Clone this generator with the exact same options.""" 119 return self.__class__(fp, 120 self._mangle_from_, 121 None, # Use policy setting, which we've adjusted 122 policy=self.policy) 123 124 # 125 # Protected interface - undocumented ;/ 126 # 127 128 # Note that we use 'self.write' when what we are writing is coming from 129 # the source, and self._fp.write when what we are writing is coming from a 130 # buffer (because the Bytes subclass has already had a chance to transform 131 # the data in its write method in that case). This is an entirely 132 # pragmatic split determined by experiment; we could be more general by 133 # always using write and having the Bytes subclass write method detect when 134 # it has already transformed the input; but, since this whole thing is a 135 # hack anyway this seems good enough. 136 137 # Similarly, we have _XXX and _encoded_XXX attributes that are used on 138 # source and buffer data, respectively. 139 _encoded_EMPTY = '' 140 141 def _new_buffer(self): 142 # BytesGenerator overrides this to return BytesIO. 143 return StringIO() 144 145 def _encode(self, s): 146 # BytesGenerator overrides this to encode strings to bytes. 147 return s 148 149 def _write_lines(self, lines): 150 # We have to transform the line endings. 151 if not lines: 152 return 153 lines = lines.splitlines(True) 154 for line in lines[:-1]: 155 self.write(line.rstrip('\r\n')) 156 self.write(self._NL) 157 laststripped = lines[-1].rstrip('\r\n') 158 self.write(laststripped) 159 if len(lines[-1]) != len(laststripped): 160 self.write(self._NL) 161 162 def _write(self, msg): 163 # We can't write the headers yet because of the following scenario: 164 # say a multipart message includes the boundary string somewhere in 165 # its body. We'd have to calculate the new boundary /before/ we write 166 # the headers so that we can write the correct Content-Type: 167 # parameter. 168 # 169 # The way we do this, so as to make the _handle_*() methods simpler, 170 # is to cache any subpart writes into a buffer. The we write the 171 # headers and the buffer contents. That way, subpart handlers can 172 # Do The Right Thing, and can still modify the Content-Type: header if 173 # necessary. 174 oldfp = self._fp 175 try: 176 self._fp = sfp = self._new_buffer() 177 self._dispatch(msg) 178 finally: 179 self._fp = oldfp 180 # Write the headers. First we see if the message object wants to 181 # handle that itself. If not, we'll do it generically. 182 meth = getattr(msg, '_write_headers', None) 183 if meth is None: 184 self._write_headers(msg) 185 else: 186 meth(self) 187 self._fp.write(sfp.getvalue()) 188 189 def _dispatch(self, msg): 190 # Get the Content-Type: for the message, then try to dispatch to 191 # self._handle_<maintype>_<subtype>(). If there's no handler for the 192 # full MIME type, then dispatch to self._handle_<maintype>(). If 193 # that's missing too, then dispatch to self._writeBody(). 194 main = msg.get_content_maintype() 195 sub = msg.get_content_subtype() 196 specific = UNDERSCORE.join((main, sub)).replace('-', '_') 197 meth = getattr(self, '_handle_' + specific, None) 198 if meth is None: 199 generic = main.replace('-', '_') 200 meth = getattr(self, '_handle_' + generic, None) 201 if meth is None: 202 meth = self._writeBody 203 meth(msg) 204 205 # 206 # Default handlers 207 # 208 209 def _write_headers(self, msg): 210 for h, v in msg.raw_items(): 211 self.write(self.policy.fold(h, v)) 212 # A blank line always separates headers from body 213 self.write(self._NL) 214 215 # 216 # Handlers for writing types and subtypes 217 # 218 219 def _handle_text(self, msg): 220 payload = msg.get_payload() 221 if payload is None: 222 return 223 if not isinstance(payload, str): 224 raise TypeError('string payload expected: %s' % type(payload)) 225 if _has_surrogates(msg._payload): 226 charset = msg.get_param('charset') 227 if charset is not None: 228 del msg['content-transfer-encoding'] 229 msg.set_payload(payload, charset) 230 payload = msg.get_payload() 231 if self._mangle_from_: 232 payload = fcre.sub('>From ', payload) 233 self._write_lines(payload) 234 235 # Default body handler 236 _writeBody = _handle_text 237 238 def _handle_multipart(self, msg): 239 # The trick here is to write out each part separately, merge them all 240 # together, and then make sure that the boundary we've chosen isn't 241 # present in the payload. 242 msgtexts = [] 243 subparts = msg.get_payload() 244 if subparts is None: 245 subparts = [] 246 elif isinstance(subparts, str): 247 # e.g. a non-strict parse of a message with no starting boundary. 248 self.write(subparts) 249 return 250 elif not isinstance(subparts, list): 251 # Scalar payload 252 subparts = [subparts] 253 for part in subparts: 254 s = self._new_buffer() 255 g = self.clone(s) 256 g.flatten(part, unixfrom=False, linesep=self._NL) 257 msgtexts.append(s.getvalue()) 258 # BAW: What about boundaries that are wrapped in double-quotes? 259 boundary = msg.get_boundary() 260 if not boundary: 261 # Create a boundary that doesn't appear in any of the 262 # message texts. 263 alltext = self._encoded_NL.join(msgtexts) 264 boundary = self._make_boundary(alltext) 265 msg.set_boundary(boundary) 266 # If there's a preamble, write it out, with a trailing CRLF 267 if msg.preamble is not None: 268 if self._mangle_from_: 269 preamble = fcre.sub('>From ', msg.preamble) 270 else: 271 preamble = msg.preamble 272 self._write_lines(preamble) 273 self.write(self._NL) 274 # dash-boundary transport-padding CRLF 275 self.write('--' + boundary + self._NL) 276 # body-part 277 if msgtexts: 278 self._fp.write(msgtexts.pop(0)) 279 # *encapsulation 280 # --> delimiter transport-padding 281 # --> CRLF body-part 282 for body_part in msgtexts: 283 # delimiter transport-padding CRLF 284 self.write(self._NL + '--' + boundary + self._NL) 285 # body-part 286 self._fp.write(body_part) 287 # close-delimiter transport-padding 288 self.write(self._NL + '--' + boundary + '--') 289 if msg.epilogue is not None: 290 self.write(self._NL) 291 if self._mangle_from_: 292 epilogue = fcre.sub('>From ', msg.epilogue) 293 else: 294 epilogue = msg.epilogue 295 self._write_lines(epilogue) 296 297 def _handle_multipart_signed(self, msg): 298 # The contents of signed parts has to stay unmodified in order to keep 299 # the signature intact per RFC1847 2.1, so we disable header wrapping. 300 # RDM: This isn't enough to completely preserve the part, but it helps. 301 p = self.policy 302 self.policy = p.clone(max_line_length=0) 303 try: 304 self._handle_multipart(msg) 305 finally: 306 self.policy = p 307 308 def _handle_message_delivery_status(self, msg): 309 # We can't just write the headers directly to self's file object 310 # because this will leave an extra newline between the last header 311 # block and the boundary. Sigh. 312 blocks = [] 313 for part in msg.get_payload(): 314 s = self._new_buffer() 315 g = self.clone(s) 316 g.flatten(part, unixfrom=False, linesep=self._NL) 317 text = s.getvalue() 318 lines = text.split(self._encoded_NL) 319 # Strip off the unnecessary trailing empty line 320 if lines and lines[-1] == self._encoded_EMPTY: 321 blocks.append(self._encoded_NL.join(lines[:-1])) 322 else: 323 blocks.append(text) 324 # Now join all the blocks with an empty line. This has the lovely 325 # effect of separating each block with an empty line, but not adding 326 # an extra one after the last one. 327 self._fp.write(self._encoded_NL.join(blocks)) 328 329 def _handle_message(self, msg): 330 s = self._new_buffer() 331 g = self.clone(s) 332 # The payload of a message/rfc822 part should be a multipart sequence 333 # of length 1. The zeroth element of the list should be the Message 334 # object for the subpart. Extract that object, stringify it, and 335 # write it out. 336 # Except, it turns out, when it's a string instead, which happens when 337 # and only when HeaderParser is used on a message of mime type 338 # message/rfc822. Such messages are generated by, for example, 339 # Groupwise when forwarding unadorned messages. (Issue 7970.) So 340 # in that case we just emit the string body. 341 payload = msg._payload 342 if isinstance(payload, list): 343 g.flatten(msg.get_payload(0), unixfrom=False, linesep=self._NL) 344 payload = s.getvalue() 345 else: 346 payload = self._encode(payload) 347 self._fp.write(payload) 348 349 # This used to be a module level function; we use a classmethod for this 350 # and _compile_re so we can continue to provide the module level function 351 # for backward compatibility by doing 352 # _make_boundary = Generator._make_boundary 353 # at the end of the module. It *is* internal, so we could drop that... 354 @classmethod 355 def _make_boundary(cls, text=None): 356 # Craft a random boundary. If text is given, ensure that the chosen 357 # boundary doesn't appear in the text. 358 token = random.randrange(sys.maxsize) 359 boundary = ('=' * 15) + (_fmt % token) + '==' 360 if text is None: 361 return boundary 362 b = boundary 363 counter = 0 364 while True: 365 cre = cls._compile_re('^--' + re.escape(b) + '(--)?$', re.MULTILINE) 366 if not cre.search(text): 367 break 368 b = boundary + '.' + str(counter) 369 counter += 1 370 return b 371 372 @classmethod 373 def _compile_re(cls, s, flags): 374 return re.compile(s, flags) 375 376 377 class BytesGenerator(Generator): 378 """Generates a bytes version of a Message object tree. 379 380 Functionally identical to the base Generator except that the output is 381 bytes and not string. When surrogates were used in the input to encode 382 bytes, these are decoded back to bytes for output. If the policy has 383 cte_type set to 7bit, then the message is transformed such that the 384 non-ASCII bytes are properly content transfer encoded, using the charset 385 unknown-8bit. 386 387 The outfp object must accept bytes in its write method. 388 """ 389 390 # Bytes versions of this constant for use in manipulating data from 391 # the BytesIO buffer. 392 _encoded_EMPTY = b'' 393 394 def write(self, s): 395 self._fp.write(s.encode('ascii', 'surrogateescape')) 396 397 def _new_buffer(self): 398 return BytesIO() 399 400 def _encode(self, s): 401 return s.encode('ascii') 402 403 def _write_headers(self, msg): 404 # This is almost the same as the string version, except for handling 405 # strings with 8bit bytes. 406 for h, v in msg.raw_items(): 407 self._fp.write(self.policy.fold_binary(h, v)) 408 # A blank line always separates headers from body 409 self.write(self._NL) 410 411 def _handle_text(self, msg): 412 # If the string has surrogates the original source was bytes, so 413 # just write it back out. 414 if msg._payload is None: 415 return 416 if _has_surrogates(msg._payload) and not self.policy.cte_type=='7bit': 417 if self._mangle_from_: 418 msg._payload = fcre.sub(">From ", msg._payload) 419 self._write_lines(msg._payload) 420 else: 421 super(BytesGenerator,self)._handle_text(msg) 422 423 # Default body handler 424 _writeBody = _handle_text 425 426 @classmethod 427 def _compile_re(cls, s, flags): 428 return re.compile(s.encode('ascii'), flags) 429 430 431 432 _FMT = '[Non-text (%(type)s) part of message omitted, filename %(filename)s]' 433 434 class DecodedGenerator(Generator): 435 """Generates a text representation of a message. 436 437 Like the Generator base class, except that non-text parts are substituted 438 with a format string representing the part. 439 """ 440 def __init__(self, outfp, mangle_from_=True, maxheaderlen=78, fmt=None): 441 """Like Generator.__init__() except that an additional optional 442 argument is allowed. 443 444 Walks through all subparts of a message. If the subpart is of main 445 type `text', then it prints the decoded payload of the subpart. 446 447 Otherwise, fmt is a format string that is used instead of the message 448 payload. fmt is expanded with the following keywords (in 449 %(keyword)s format): 450 451 type : Full MIME type of the non-text part 452 maintype : Main MIME type of the non-text part 453 subtype : Sub-MIME type of the non-text part 454 filename : Filename of the non-text part 455 description: Description associated with the non-text part 456 encoding : Content transfer encoding of the non-text part 457 458 The default value for fmt is None, meaning 459 460 [Non-text (%(type)s) part of message omitted, filename %(filename)s] 461 """ 462 Generator.__init__(self, outfp, mangle_from_, maxheaderlen) 463 if fmt is None: 464 self._fmt = _FMT 465 else: 466 self._fmt = fmt 467 468 def _dispatch(self, msg): 469 for part in msg.walk(): 470 maintype = part.get_content_maintype() 471 if maintype == 'text': 472 print(part.get_payload(decode=False), file=self) 473 elif maintype == 'multipart': 474 # Just skip this 475 pass 476 else: 477 print(self._fmt % { 478 'type' : part.get_content_type(), 479 'maintype' : part.get_content_maintype(), 480 'subtype' : part.get_content_subtype(), 481 'filename' : part.get_filename('[no filename]'), 482 'description': part.get('Content-Description', 483 '[no description]'), 484 'encoding' : part.get('Content-Transfer-Encoding', 485 '[no encoding]'), 486 }, file=self) 487 488 489 490 # Helper used by Generator._make_boundary 491 _width = len(repr(sys.maxsize-1)) 492 _fmt = '%%0%dd' % _width 493 494 # Backward compatibility 495 _make_boundary = Generator._make_boundary |