Changeset 151:8b42bc92613a for imapclient

Timestamp:

05/07/10 13:37:02 (2 years ago)

Author:

Menno Smits <menno@…>

Branch:

default

Message:

refactored Lexer classes. Not quite working yet.

Location:

imapclient

Files:

• response_lexer.py (modified) (2 diffs)
• response_parser.py (modified) (4 diffs)

imapclient/response_lexer.py

@@ -20 +20 @@
         self.wordchars = set(self.NON_SPECIALS)
         self.whitespace = set((' \t\r\n'))
-        self.sources = sources
+        self.sources = (LiteralHandlingIter(self, chunk) for chunk in sources)
         self.current_source = None
 
@@ -100 +100 @@
 
     def __iter__(self):
-        "Generate a token"
+        "Generate tokens"
         for source in self.sources:
             self.current_source = source
             for tok in self.read_token_stream(iter(source)):
                 yield tok
+
+    @classmethod
+    def create_token_source(cls, text):
+        lex = cls(text)
+        return TokenIterator(lex)
+
+
+class TokenIterator(object):
+
+    def __init__(self, lex):
+        self.lex = lex
+        self.src = iter(lex)
+
+    @property
+    def current_literal(self):
+        return self.lex.current_source.literal
+
+    def __iter__(self):
+        return self.src
+    
+
+# imaplib has poor handling of 'literals' - it both fails to remove the
+# {size} marker, and fails to keep responses grouped into the same logical
+# 'line'.  What we end up with is a list of response 'records', where each
+# record is either a simple string, or tuple of (str_with_lit, literal) -
+# where str_with_lit is a string with the {xxx} marker at its end.  Note
+# that each elt of this list does *not* correspond 1:1 with the untagged
+# responses.
+# (http://bugs.python.org/issue5045 also has comments about this)
+# So: we have a special file-like object for each of these records.  When
+# a string literal is finally processed, we peek into this file-like object
+# to grab the literal.
+class LiteralHandlingIter:
+    def __init__(self, lexer, resp_record):
+        self.pushed = None
+        self.lexer = lexer
+        if isinstance(resp_record, tuple):
+            # A 'record' with a string which includes a literal marker, and
+            # the literal itself.
+            src_text, self.literal = resp_record
+            assert src_text.endswith("}"), src_text
+            self.src_text = self.literal
+        else:
+            # just a line with no literals.
+            self.src_text = resp_record
+            self.literal = None
+
+    def __iter__(self):
+        return iter(self.src_text)
+

imapclient/response_parser.py

@@ -9 +9 @@
 
 import imaplib
-import response_lexer
 from datetime import datetime
 from fixed_offset import FixedOffset
+from response_lexer import Lexer
 
 
@@ -32 +32 @@
     if not text:
         return
-    src = ResponseTokeniser(text)
+    src = Lexer.create_token_source(text)
+    
     token = None
     try:
@@ -118 +119 @@
 
 
-EOF = object()
-
-# imaplib has poor handling of 'literals' - it both fails to remove the
-# {size} marker, and fails to keep responses grouped into the same logical
-# 'line'.  What we end up with is a list of response 'records', where each
-# record is either a simple string, or tuple of (str_with_lit, literal) -
-# where str_with_lit is a string with the {xxx} marker at its end.  Note
-# that each elt of this list does *not* correspond 1:1 with the untagged
-# responses.
-# (http://bugs.python.org/issue5045 also has comments about this)
-# So: we have a special file-like object for each of these records.  When
-# a string literal is finally processed, we peek into this file-like object
-# to grab the literal.
-class LiteralHandlingIter:
-    def __init__(self, lexer, resp_record):
-        self.pushed = None
-        self.lexer = lexer
-        if isinstance(resp_record, tuple):
-            # A 'record' with a string which includes a literal marker, and
-            # the literal itself.
-            src_text, self.literal = resp_record
-            assert src_text.endswith("}"), src_text
-            self.src_text = src_text
-        else:
-            # just a line with no literals.
-            self.src_text = resp_record
-            self.literal = None
-
-    def __iter__(self):
-        return iter(self.src_text)
-
-
-class ResponseTokeniser(object):
-    def __init__(self, resp_chunks):
-        # initialize the lexer with all the chunks we read.
-        sources = (LiteralHandlingIter(lex, chunk) for chunk in resp_chunks)
-        lex = response_lexer.Lexer(sources)
-        self.tok_src = iter(lex)
-        self.lex = lex
-
-    def __iter__(self):
-        return self.tok_src
-
-
 def atom(src, token):
     if token == "(":
@@ -176 +133 @@
     elif token[0] == '{':
         literal_len = int(token[1:-1])
-        literal_text = src.lex.current_source.literal
+        literal_text = src.current_literal
         if literal_text is None:
            raise ParseError('No literal corresponds to %r' % token)