Coverage for latexcodec/lexer : 58%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
|
# -*- coding: utf-8 -*- LaTeX Lexer ~~~~~~~~~~~
This module contains all classes for lexing LaTeX code, as well as general purpose base classes for incremental LaTeX decoders and encoders, which could be useful in case you are writing your own custom LaTeX codec.
.. autoclass:: Token(name, text)
.. autoclass:: LatexLexer :show-inheritance: :members:
.. autoclass:: LatexIncrementalLexer :show-inheritance: :members:
.. autoclass:: LatexIncrementalDecoder :show-inheritance: :members:
.. autoclass:: LatexIncrementalEncoder :show-inheritance: :members: """
# Copyright (c) 2003, 2008 David Eppstein # Copyright (c) 2011-2014 Matthias C. M. Troffaes # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE.
# implementation note: we derive from IncrementalDecoder because this # class serves excellently as a base class for incremental decoders, # but of course we don't decode yet until later
"chars", b"?" if cls.binary_mode else u"\ufffd")
"""Metaclass for :class:`RegexpLexer`. Compiles tokens into a regular expression. """
b"(?P<" + name.encode("ascii") + b">" + regexp + b")" for name, regexp in cls.tokens))
"""Abstract base class for regexp based lexers."""
"""Tuple containing all token regular expressions."""
"""Whether this lexer processes binary data (bytes) or text data (unicode). """
"""Initialize the codec."""
"""Reset state.""" # buffer for storing last (possibly incomplete) token
"""Get state.""" return (self.raw_buffer.text, 0)
"""Set state. The *state* must correspond to the return value of a previous :meth:`getstate` call. """ self.raw_buffer = Token('unknown', state[0])
"""Yield tokens without any further processing. Tokens are one of:
- ``\\<word>``: a control word (i.e. a command) - ``\\<symbol>``: a control symbol (i.e. \\^ etc.) - ``#<n>``: a parameter - a series of byte characters """ bytes_ = self.raw_buffer.text + bytes_ # yield the buffer token # fill buffer with next token
"""Flush the raw token buffer."""
"""A very simple lexer for tex/latex bytes."""
# implementation note: every token **must** be decodable by inputenc # comment: for ease, and for speed, we handle it as a token (u'comment', br'%.*?\n'), # control tokens # in latex, some control tokens skip following whitespace # ('control-word' and 'control-symbol') # others do not ('control-symbol-x') # XXX TBT says no control symbols skip whitespace (except '\ ') # XXX but tests reveal otherwise? (u'control_word', br'[\\][a-zA-Z]+'), (u'control_symbol', br'[\\][~' br"'" br'"` =^!]'), # TODO should only match ascii (u'control_symbol_x', br'[\\][^a-zA-Z]'), # parameter tokens # also support a lone hash so we can lex things like b'#a' (u'parameter', br'\#[0-9]|\#'), # any remaining characters; for ease we also handle space and # newline as tokens # XXX TBT does not mention \t to be a space character as well # XXX but tests reveal otherwise? (u'space', br' |\t'), (u'newline', br'\n'), (u'mathshift', br'[$][$]|[$]'), # note: some chars joined together to make it easier to detect # symbols that have a special function (i.e. --, ---, etc.) (u'chars', br'---|--|-|[`][`]' br"|['][']" br'|[?][`]|[!][`]' # separate chars because brackets are optional # e.g. fran\\c cais = fran\\c{c}ais in latex # so only way to detect \\c acting on c only is this way br'|[0-9a-zA-Z{}]' # we have to join everything else together to support # multibyte encodings: every token must be decodable!! # this means for instance that \\c öké is NOT equivalent to # \\c{ö}ké br'|[^ %#$\n\\]+'), # trailing garbage which we cannot decode otherwise # (such as a lone '\' at the end of a buffer) # is never emitted, but used internally by the buffer (u'unknown', br'.'), )
"""A very simple incremental lexer for tex/latex code. Roughly follows the state machine described in Tex By Topic, Chapter 2.
The generated tokens satisfy:
* no newline characters: paragraphs are separated by '\\par' * spaces following control tokens are compressed """
# three possible states: # newline (N), skipping spaces (S), and middle of line (M) # inline math mode?
# state 'M' is most common, so let that be zero return ( self.raw_buffer, {'M': 0, 'N': 1, 'S': 2}[self.state] | (4 if self.inline_math else 0) )
self.raw_buffer = state[0] self.state = {0: 'M', 1: 'N', 2: 'S'}[state[1] & 3] self.inline_math = bool(state[1] & 4)
"""Yield tokens while maintaining a state. Also skip whitespace after control words and (some) control symbols. Replaces newlines by spaces and \\par commands depending on the context. """ # current position relative to the start of bytes_ in the sequence # of bytes that have been decoded if self.state == 'N': # if state was 'N', generate new paragraph yield self.partoken elif self.state == 'S': # switch to 'N' state, do not generate a space self.state = 'N' elif self.state == 'M': # switch to 'N' state, generate a space self.state = 'N' yield self.spacetoken else: raise AssertionError( "unknown tex state {0!r}".format(self.state)) # remain in 'N' state, no space token generated pass # remain in 'S' state, no space token generated # in M mode, generate the space, # but switch to space skip mode else: raise AssertionError( "unknown state {0!r}".format(self.state)) self.state = 'M' yield token # go to space skip mode # go to space skip mode # don't skip following space, so go to M mode # go to newline mode, no token is generated # note: comment includes the newline self.state = 'N' elif token.name == 'unknown': if self.errors == 'strict': # hack around a bug in Python: UnicodeDecodeError # expects binary input if not self.binary_mode: bytes_ = bytes_.encode("utf8") # current position within bytes_ # this is the position right after the unknown token raise UnicodeDecodeError( "latex", # codec bytes_, # problematic input pos - len(token.text), # start of problematic token pos, # end of it "unknown token {0!r}".format(token.text)) elif self.errors == 'ignore': # do nothing pass elif self.errors == 'replace': yield self.replacetoken else: raise NotImplementedError( "error mode {0!r} not supported".format(self.errors)) else: raise AssertionError( "unknown token name {0!r}".format(token.name))
"""Simple incremental decoder. Transforms lexed LaTeX tokens into unicode.
To customize decoding, subclass and override :meth:`get_unicode_tokens`. """
"""Input encoding. **Must** extend ascii."""
"""Returns the decoded token text in :attr:`inputenc` encoding.
.. note::
Control words get an extra space added at the back to make sure separation from the next token, so that decoded token sequences can be :meth:`str.join`\ ed together.
For example, the tokens ``b'\\hello'`` and ``b'world'`` will correctly result in ``u'\\hello world'`` (remember that LaTeX eats space following control words). If no space were added, this would wrongfully result in ``u'\\helloworld'``.
""" # in python 3, the token text can be a memoryview # which do not have a decode method; must cast to bytes explicitly if self.binary_mode: text = binary_type(token.text).decode(self.inputenc) else: text = token.text return text if token.name != 'control_word' else text + u' '
"""Decode every token in :attr:`inputenc` encoding. Override to process the tokens in some other way (for example, for token translation). """ for token in self.get_tokens(bytes_, final=final): yield self.decode_token(token)
"""Decode LaTeX *bytes_* into a unicode string.
This implementation calls :meth:`get_unicode_tokens` and joins the resulting unicode strings together. """ try: return u''.join(self.get_unicode_tokens(bytes_, final=final)) except UnicodeDecodeError as e: # API requires that the encode method raises a ValueError # in this case raise ValueError(e)
"""Simple incremental encoder for LaTeX. Transforms unicode into :class:`bytes`.
To customize decoding, subclass and override :meth:`get_latex_bytes`. """
"""Input encoding. **Must** extend ascii."""
"""Whether this encoder processes binary data (bytes) or text data (unicode). """
"""Initialize the codec."""
"""Reset state.""" # buffer for storing last (possibly incomplete) token
"""Get state.""" return self.buffer
"""Set state. The *state* must correspond to the return value of a previous :meth:`getstate` call. """ self.buffer = state
"""Split unicode into tokens so that every token starts with a non-combining character. """ if not isinstance(unicode_, string_types): raise TypeError( "expected unicode for encode input, but got {0} instead" .format(unicode_.__class__.__name__)) for c in unicode_: if not unicodedata.combining(c): for token in self.flush_unicode_tokens(): yield token self.buffer += c if final: for token in self.flush_unicode_tokens(): yield token
"""Flush the buffer.""" if self.buffer: yield self.buffer self.buffer = u""
"""Encode every character in :attr:`inputenc` encoding. Override to process the unicode in some other way (for example, for character translation). """ if self.binary_mode: for token in self.get_unicode_tokens(unicode_, final=final): yield token.encode(self.inputenc, self.errors) else: for token in self.get_unicode_tokens(unicode_, final=final): yield token
"""Encode the *unicode_* string into LaTeX :class:`bytes`.
This implementation calls :meth:`get_latex_bytes` and joins the resulting :class:`bytes` together. """ self.get_latex_bytes(unicode_, final=final)) except UnicodeEncodeError as e: # API requires that the encode method raises a ValueError # in this case raise ValueError(e)
|