Changeset 115:7aa870b75d11 for imapclient

Timestamp:

01/11/10 19:11:54 (2 years ago)

Author:

Menno Smits <menno@…>

Branch:

default

Message:

Parse imaplib's fetch data structures instead of doing it all ourselves

This makes the code messier but means imaplib does a bit more of the
heavy lifting - this is probably safer from a bug standpoint.

From Mark Hammond.

Location:

imapclient

Files:

• imapclient.py (modified) (4 diffs)
• response_parser.py (modified) (7 diffs)
• test/test_response_parser.py (modified) (9 diffs)

imapclient/imapclient.py

@@ -14 +14 @@
 __all__ = ['IMAPClient', 'DELETED', 'SEEN', 'ANSWERED', 'FLAGGED', 'DRAFT',
     'RECENT']
+
+from response_parser import parse_fetch_response
 
 # System flags
@@ -327 +329 @@
 
         self._checkok('search', typ, data)
+        if data == [None]: # no untagged responses...
+            return []
 
         return [ long(i) for i in data[0].split() ]
@@ -436 +440 @@
         return parser(data)
 
-
     def altfetch(self, messages, parts):
         if not messages:
@@ -448 +451 @@
         else:
             tag = self._imap._command('FETCH', msg_list, parts_list)
-
-        print tag
-        lines = []
-        while True:
-            line = self._imap._get_line()
-            if line.startswith(tag):
-                break
-            lines.append(line)
-        return lines
-    
-        #self._checkok('fetch', typ, data)
-
-        #parser = FetchParser()
-        #return parser(data)
-
+        typ, data = self._imap._command_complete('FETCH', tag)
+        self._checkok('fetch', typ, data)
+        typ, data = self._imap._untagged_response(typ, data, 'FETCH')
+        # appears to be a special case - no 'untagged' responses (ie, no
+        # folders) results in [None]
+        if data == [None]:
+          return {}
+
+        return parse_fetch_response(data)
 
     def append(self, folder, msg, flags=(), msg_time=None):

imapclient/response_parser.py

@@ -22 +22 @@
 
 
-
 def parse_response(text):
     #XXX doc
@@ -38 +37 @@
     response = iter(parse_response(text))
 
-    def expect(expected_value):
-        next_value = response.next().upper()
-        if next_value != expected_value:
-            raise ParseError('expected %r, got %r' % (expected_value, next_value))
-
     parsed_response = {}
     while True:
         try:
-            expect('*')
+            msg_id = _int_or_error(response.next(), 'invalid message ID')
         except StopIteration:
             break
 
-        msg_id = _int_or_error(response.next(), 'invalid message ID')
-        expect('FETCH')
+        try:
+            msg_response = response.next()
+        except StopIteration:
+            raise ParseError('unexpected EOF')
 
-        msg_response = response.next()
         if not isinstance(msg_response, tuple):
             raise ParseError('bad response type: %s' % repr(msg_response))
@@ -59 +54 @@
             raise ParseError('uneven number of response items: %s' % repr(msg_response))
 
-        #XXX extract this
-        msg_data = {}
+        # always return the 'sequence' of the message, so it is available
+        # even if we return keyed by UID.
+        msg_data = {'SEQ': msg_id}
         for i in xrange(0, len(msg_response), 2):
             word = msg_response[i].upper()
@@ -81 +77 @@
         raise ParseError('%s: %s' % (error_text, repr(value)))
 
+EOF = object()
 
+# imaplib has poor handling of 'literals' - it both fails to remove the
+# {size} marker, and fails to keep responses grouped into the same logical
+# 'line'.  What we end up with is a list of response 'records', where each
+# record is either a simple string, or tuple of (str_with_lit, literal) -
+# where str_with_lit is a string with the {xxx} marker at its end.  Note
+# that each elt of this list does *not* correspond 1:1 with the untagged
+# responses.
+# (http://bugs.python.org/issue5045 also has comments about this)
+# So: we have a special file-like object for each of these records.  When
+# a string literal is finally processed, we peek into this file-like object
+# to grab the literal.
+class LiteralHandlingReader:
+    def __init__(self, lexer, resp_record):
+        self.pushed = None
+        self.lexer = lexer
+        if isinstance(resp_record, tuple):
+            # A 'record' with a string which includes a literal marker, and
+            # the literal itself.
+            src_text, self.literal = resp_record
+            assert src_text.endswith("}"), src_text
+            # add a token-sep after the text.
+            self.src = StringIO(src_text + " ")
+        else:
+            # just a line with no literals.
+            self.src = StringIO(resp_record)
+            self.literal = None
 
-EOF = object()
+    def read(self, n):
+        # We also hack into the lexer so we get special treatment for '\\'
+        # chars - they are only special inside a quoted string.
+        assert n==1
+        if self.pushed is not None:
+            ret = self.pushed
+            self.pushed = None
+        else:
+            ret = self.src.read(n)
+            if ret=="\\" and self.lexer.state not in '"\\':
+                self.pushed = "\\"
+        return ret
+
+    def close(self):
+        self.src.close()
+        self.src = None
+        self.literal = None
+
 
 class ResponseTokeniser(object):
 
     CTRL_CHARS = ''.join([chr(ch) for ch in range(32)])
-    ATOM_SPECIALS = r'()%*"]' + CTRL_CHARS
+    ATOM_SPECIALS = r'()%*"' + CTRL_CHARS
     ALL_CHARS = [chr(ch) for ch in range(256)]
     ATOM_NON_SPECIALS = [ch for ch in ALL_CHARS if ch not in ATOM_SPECIALS]
 
-    def __init__(self, text):
-        self.lex = shlex.shlex(text)
+    def __init__(self, resp_chunks):
+        # initialize the lexer with all the chunks we read.
+        self.lex = shlex.shlex('', posix=True)
+        for chunk in reversed(resp_chunks):
+            self.lex.push_source(LiteralHandlingReader(self.lex, chunk))
+
         self.lex.quotes = '"'
         self.lex.commenters = ''
@@ -106 +150 @@
         except StopIteration:
             return EOF
-
-    def read(self, bytes):
-        return self.lex.instream.read(bytes)
 
 
@@ -126 +167 @@
     elif token.startswith('{'):
         literal_len = int(token[1:-1])
-        if src.read(1) != '\n':
-           raise ParseError('No CRLF after %s' % token)
-        return src.read(literal_len)
+        literal_text = src.lex.instream.literal
+        if literal_text is None:
+           raise ParseError('No literal corresponds to %r' % token)
+        if len(literal_text) != literal_len:
+            raise ParseError('Expecting literal of size %d, got %d' % (
+                                literal_len, len(literal_text)))
+        return literal_text
     elif token.startswith('"'):
         return token[1:-1]
@@ -135 +180 @@
     else:
         return token
-
-
-

imapclient/test/test_response_parser.py

@@ -74 +74 @@
                     '<hi.there>', 'foo', 'BASE64', 4554, 73), 'MIXED'))
 
+    def test_envelopey(self):
+        self._test('(UID 5 ENVELOPE ("internal_date" "subject" '
+                   '(("name" NIL "address1" "domain1.com")) '
+                   '((NIL NIL "address2" "domain2.com")) '
+                   '(("name" NIL "address3" "domain3.com")) '
+                   '((NIL NIL "address4" "domain4.com")) '
+                   'NIL NIL "<reply-to-id>" "<msg_id>"))',
+                   ('UID',
+                    5,
+                    'ENVELOPE',
+                    ('internal_date',
+                     'subject',
+                     (('name', None, 'address1', 'domain1.com'),),
+                     ((None, None, 'address2', 'domain2.com'),),
+                     (('name', None, 'address3', 'domain3.com'),),
+                     ((None, None, 'address4', 'domain4.com'),),
+                     None,
+                     None,
+                     '<reply-to-id>',
+                     '<msg_id>')))
+
+    def test_envelopey_quoted(self):
+        self._test('(UID 5 ENVELOPE ("internal_date" "subject with \\"quotes\\"" '
+                   '(("name" NIL "address1" "domain1.com")) '
+                   '((NIL NIL "address2" "domain2.com")) '
+                   '(("name" NIL "address3" "domain3.com")) '
+                   '((NIL NIL "address4" "domain4.com")) '
+                   'NIL NIL "<reply-to-id>" "<msg_id>"))',
+                   ('UID',
+                    5,
+                    'ENVELOPE',
+                    ('internal_date',
+                     'subject with "quotes"',
+                     (('name', None, 'address1', 'domain1.com'),),
+                     ((None, None, 'address2', 'domain2.com'),),
+                     (('name', None, 'address3', 'domain3.com'),),
+                     ((None, None, 'address4', 'domain4.com'),),
+                     None,
+                     None,
+                     '<reply-to-id>',
+                     '<msg_id>')))
 
     def test_literal(self):
@@ -80 +121 @@
             abc def XYZ
             """))
-        self._test('{18}' + CRLF + literal_text, literal_text)
+        self._test([('{18}', literal_text)], literal_text)
 
 
@@ -88 +129 @@
             abc def XYZ
             """))
-        response = add_crlf(dedent("""\
-            (12 "foo" {18}
-            %s)
-            """) % literal_text)
+        response = [('(12 "foo" {18}', literal_text), ")"]
         self._test(response, (12, 'foo', literal_text))
 
 
     def test_quoted_specials(self):
-        self._test(r'"foo \"bar\""', ('foo "bar"',))
-        self._test(r'"foo\\bar"', (r'foo\bar',))
+        self._test(r'"foo \"bar\""', 'foo "bar"')
+        self._test(r'"foo\\bar"', r'foo\bar')
 
 
@@ -105 +143 @@
 
     def test_bad_literal(self):
-        self._test_parse_error('{99} abc', 'No CRLF after {99}')
+        self._test_parse_error([('{99}', 'abc')],
+                               'Expecting literal of size 99, got 3')
 
 
     def test_bad_quoting(self):
-        self._test_parse_error('"abc next', 'No closing quotation: "abc next')
-
-
+        self._test_parse_error('"abc next', 'No closing quotation:')
 
 
@@ -118 +155 @@
             # convenience - expected value should be wrapped in another tuple
             expected = (expected,)
+        if not isinstance(to_parse, list):
+            to_parse = [to_parse]
         output = parse_response(to_parse)
         self.assert_(
@@ -129 +168 @@
             self.fail("didn't raise an exception")
         except ParseError, err:
-            self.assert_(expected_msg == str(err),
+            self.assert_(expected_msg == str(err)[:len(expected_msg)],
                          'got ParseError with wrong msg: %r' % str(err))
 
@@ -141 +180 @@
 
     def test_basic(self):
-        self.assertEquals(parse_fetch_response('* 4 FETCH ()'), {4: {}})
-        self.assertEquals(parse_fetch_response('* 4 fEtCh ()'), {4: {}})
-
-
-    def test_non_fetch(self):
-        self.assertRaises(ParseError, parse_fetch_response, '* 4 OTHER ()')
+        self.assertEquals(parse_fetch_response('4 ()'), {4: {'SEQ': 4}})
+
+
+#    def test_non_fetch(self):
+#        self.assertRaises(ParseError, parse_fetch_response, ['4 ()'])
 
 
     def test_bad_msgid(self):
-        self.assertRaises(ParseError, parse_fetch_response, '* abc FETCH ()')
+        self.assertRaises(ParseError, parse_fetch_response, ['abc ()'])
 
 
     def test_bad_data(self):
-        self.assertRaises(ParseError, parse_fetch_response, '* 2 FETCH WHAT')
+        self.assertRaises(ParseError, parse_fetch_response, ['2 WHAT'])
 
 
     def test_missing_data(self):
-        self.assertRaises(ParseError, parse_fetch_response, '* 2 FETCH')
+        self.assertRaises(ParseError, parse_fetch_response, ['2'])
 
 
     def test_simple_pairs(self):
-        self.assertEquals(parse_fetch_response('* 23 FETCH (ABC 123 StUfF "hello")'),
+        self.assertEquals(parse_fetch_response(['23 (ABC 123 StUfF "hello")']),
                           {23: {'ABC': 123,
-                                'STUFF': 'hello'}})
+                                'STUFF': 'hello',
+                                'SEQ': 23}})
 
 
     def test_odd_pairs(self):
-        self.assertRaises(ParseError, parse_fetch_response, '* 2 FETCH (ONE)')
-        self.assertRaises(ParseError, parse_fetch_response, '* 2 FETCH (ONE TWO THREE)')
+        self.assertRaises(ParseError, parse_fetch_response, ['* 2 FETCH (ONE)'])
+        self.assertRaises(ParseError, parse_fetch_response, ['* 2 FETCH (ONE TWO THREE)'])
 
 
     def test_UID(self):
-        self.assertEquals(parse_fetch_response('* 23 FETCH (UID 76)'),
-                          {76: {}})
-        self.assertEquals(parse_fetch_response('* 23 FETCH (uiD 76)'),
-                          {76: {}})
+        self.assertEquals(parse_fetch_response(['23 (UID 76)']),
+                          {76: {'SEQ': 23}})
+        self.assertEquals(parse_fetch_response(['23 (uiD 76)']),
+                          {76: {'SEQ': 23}})
 
 
@@ -184 +223 @@
 
     def test_FLAGS(self):
-        self.assertEquals(parse_fetch_response('* 23 FETCH (FLAGS (\Seen Stuff))'),
-                          {23: {'FLAGS': (r'\Seen', 'Stuff')}})
+        self.assertEquals(parse_fetch_response(['23 (FLAGS (\Seen Stuff))']),
+                          {23: {'SEQ': 23, 'FLAGS': (r'\Seen', 'Stuff')}})
 
 
     def test_multiple_messages(self):
-        self.fail()
+        self.assertEquals(parse_fetch_response(
+                                    ["2 (FLAGS (Foo Bar)) ",
+                                     "7 (FLAGS (Baz Sneeve))"]),
+                         {
+                            2: {'FLAGS': ('Foo', 'Bar'), 'SEQ': 2},
+                            7: {'FLAGS': ('Baz', 'Sneeve'), 'SEQ': 7},
+                         })
 
 
@@ -217 +262 @@
 #                    datetime.datetime(2007, 12, 9, 17, 8, 8, 0, FixedOffset(0)))
 
-
-#     def testMultipleMessages(self):
-#         '''Test with multple messages in the response
-#         '''
-#         self._parse_test(
-#             [
-#                 r'2 (FLAGS (Foo Bar))',
-#                 r'7 (FLAGS (Baz Sneeve))',
-#                 ],
-#             {
-#                 2: {'FLAGS': ['Foo', 'Bar']},
-#                 7: {'FLAGS': ['Baz', 'Sneeve']},
-#                 }
-#             )
 
 #     def testLiteral(self):