Merge pull request #2 from donl/master

Lexer and Style changes
2026-05-15 14:15:53 -06:00 · 2016-02-15 21:23:56 -05:00 · 2016-02-15 21:23:56 -05:00 · a4109d1eff
commit a4109d1eff
parent a3047fdd7b a2e8518f45
3 changed files with 106 additions and 62 deletions
--- a/pygments_xojo/lexer.py
+++ b/pygments_xojo/lexer.py
@ -8,7 +8,7 @@ Lexer for the Xojo language.

 from __future__ import absolute_import
 import re
-from pygments.lexer import RegexLexer, words, include, default, using, this
+from pygments.lexer import RegexLexer, bygroups, words, include, default, using, this
 from pygments.token import Keyword, Name, String, Literal, Number, Punctuation, Comment, \
    Operator, Text, Error

@ -22,8 +22,6 @@ IDENTIFIER_FQ = r'[^\d\W]\w*(\.[^\d\W]\w*)*'
 LITERAL_STRING = r'"(""|[^"])*"'
 LITERAL_UNICODE = r'&u[0-9a-fA-F]+'
 WHITESPACE = r'[\ \t]+'
-LITERAL_COLOR_32 = r'&c[0-9a-fA-F]{8}'
-LITERAL_COLOR_24 = r'&c[0-9a-fA-F]{6}'
 DECLARE = r'(soft\s+)?declare[^)]+\)'
 WORD_SUFFIX = r'\b'

@ -51,7 +49,7 @@ class XojoLexer(_LexerOptionsMixin, RegexLexer):
    BUILTINS = ['AddHandler', 'Call', 'CurrentMethodName',
    'Raise', 'RemoveHandler']
    KEYWORDS = ['Aggregates', 'Assigns', 'Attributes', 'Break', 'ByRef', 'ByVal',
-    'Case', 'Catch', 'Class', 'Continue', 'Delegate', 'Do', 'DownTo', 'Each',
+    'Case', 'Catch', 'Class', 'Continue', 'Do', 'DownTo', 'Each',
    'Enum', 'Else', 'ElseIf', 'End', 'Event', 'Exception', 'Exit', 'Extends', 'Finally',
    'For', 'Function', 'Global', 'Handles', 'If', 'Implements', 'In', 'Inherits', 'Interface',
    'Lib', 'Loop', 'Module', 'Next', 'New', 'Namespace', 'Optional', 'ParamArray', 'Private',
@ -60,7 +58,9 @@ class XojoLexer(_LexerOptionsMixin, RegexLexer):
    '#if', '#else', '#elseif', '#endif', '#pragma']
    OPERATOR_WORDS = ['And', 'Is', 'IsA', 'Mod', 'Not', 'Or', 'Xor', 'AddressOf', 'Array',
    'Ctype', 'GetTypeInfo', 'RaiseEvent', 'Redim', 'WeakAddressOf']
-
+    TYPES = ['Boolean', 'Byte', 'Color', 'Currency', 'Delegate', 'Double', 'Integer',
+    'Int8', 'Int16', 'Int32', 'Int64', 'UInt8', 'UInt16', 'UInt32', 'UInt64', 'Short', 'Single', 'String',
+    'Structure']
    tokens = {
        'whitespace': [
            (r'[\ \t]+', Text),
@ -73,6 +73,7 @@ class XojoLexer(_LexerOptionsMixin, RegexLexer):

        'root': [
            include('whitespace'),
+            (words(['#tag'], suffix=WORD_SUFFIX), Comment.Preproc),
            (words(DECLARATIONS, suffix=WORD_SUFFIX), Keyword.Declaration),
            (words(CONSTANTS, suffix=WORD_SUFFIX), Keyword.Constant),
            (words(PSEUDO_BUILTINS, suffix=WORD_SUFFIX), Name.Builtin.Pseudo),
@ -85,22 +86,38 @@ class XojoLexer(_LexerOptionsMixin, RegexLexer):
            (words(['GOTO'], suffix=WORD_SUFFIX), Keyword.Reserved, 'goto'),
            (words(BUILTINS, suffix=WORD_SUFFIX), Name.Builtin),
            (words(KEYWORDS, suffix=WORD_SUFFIX), Keyword.Reserved),
-
+            (words(TYPES, suffix=WORD_SUFFIX), Keyword.Type),
+                 
            # Literals
            (LITERAL_STRING, String),
-            (LITERAL_UNICODE, Literal),
+            (LITERAL_UNICODE, Literal.Unicode),
+            (r'([0-9]*\.[0-9]+)([eE][-+]?[0-9]+)?', bygroups(Number.Float, Number.Float)),
            (r'[0-9]+', Number.Integer),
-            (r'&b[01]+', Number.Bin),
-            (r'&o[0-7]+', Number.Oct),
-            (r'&h[0-9a-fA-F]+', Number.Hex),
-            (LITERAL_COLOR_32, Literal),
-            (LITERAL_COLOR_24, Literal),
+                 
+            (r'(&b[01]+)(\s+)', bygroups(Number.Bin, Text)),
+            (r'(&b[01]+)(_|,|[)])', bygroups(Number.Bin, Punctuation)),
+            (r'(&b[01]+)(\'|//)', bygroups(Number.Bin, Comment),'comment_url'),
+            (r'(&b[01]+)(<>|<=?|>=?|[-=+*/^\\])', bygroups(Number.Bin, Operator)),
+                 
+            (r'(&o[0-7]+)(\s+)', bygroups(Number.Oct, Text)),
+            (r'(&o[0-7]+)(_|,|[)])', bygroups(Number.Oct, Punctuation)),
+            (r'(&o[0-7]+)(\'|//)', bygroups(Number.Oct, Comment),'comment_url'),
+            (r'(&o[0-7]+)(<>|<=?|>=?|[-=+*/^\\])', bygroups(Number.Oct, Operator)),
+                 
+            (r'(&h[0-9a-fA-F]+)(\s+)', bygroups(Number.Oct, Text)),
+            (r'(&h[0-9a-fA-F]+)(_|,|[)])', bygroups(Number.Oct, Punctuation)),
+            (r'(&h[0-9a-fA-F]+)(\'|//)', bygroups(Number.Hex, Comment),'comment_url'),
+            (r'(&h[0-9a-fA-F]+)(<>|<=?|>=?|[-=+*/^\\])', bygroups(Number.Hex, Operator)),
+           
+            (r'(&c)([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})([0-9a-fA-F]{2})?',
+             bygroups(Name.XojoType.Color, Name.XojoType.Color.Red, Name.XojoType.Color.Green,
+                      Name.XojoType.Color.Blue, Name.XojoType.Color.Alpha)),

            # line continuation
            (r'_(?!\w)', Punctuation),
-            (r"\'.*", Comment),
-            (r'//.*', Comment),
-            (r'REM\b.*', Comment),
+            (r'\'', Comment, 'comment_url'),
+            (r'//', Comment, 'comment_url'),
+            (r'REM\b', Comment, 'comment_url'),

            (r'<>|<=?|>=?|[-=+*/^\\]', Operator),
            (words(OPERATOR_WORDS, suffix=WORD_SUFFIX), Operator.Word),
@ -167,4 +184,16 @@ class XojoLexer(_LexerOptionsMixin, RegexLexer):
            (r'end property', Keyword.Reserved, '#pop'),
            ],

+        'comment_url': [
+            (r'http\://\S*', Comment.URL),
+            (r'https\://\S*', Comment.URL),
+            (r'ftp\://\S*', Comment.URL),
+            (r'ftps\://\S*', Comment.URL),
+            (r'rb-feedback\://\S*', Comment.URL),
+            (r'feedback\://\S*', Comment.URL),
+            (r'mailto\:\S*', Comment.URL),
+            ('\n', Comment, '#pop'),
+            (r'.', Comment),
+            ],
+
        }
--- a/pygments_xojo/styles.py
+++ b/pygments_xojo/styles.py
@ -6,62 +6,70 @@ Style for the Xojo language.

 from pygments.style import Style
 from pygments.token import Keyword, Name, Comment, String, Error, \
-     Number, Operator, Generic, Whitespace
+     Number, Operator, Generic, Whitespace, Literal

 __all__ = ['XojoStyle']

 #pylint: disable=too-few-public-methods
 class XojoStyle(Style):
-    """The default Xojo style."""
+    """The default Xojo IDE style."""

    default_style = ""
    styles = {
-        Whitespace: "#bbbbbb",
-        Comment: "italic #408080",
-        Comment.Preproc: "noitalic #BC7A00",
+        # Whitespace: "#bbbbbb",
+        Comment: "#921100",
+        Comment.URL: "underline #0C33FE",
+        Comment.Preproc: "#0C33FE",
+        Keyword: "#0C33FE",

-        #Keyword: "bold #AA22FF",
-        Keyword: "bold #008000",
-        Keyword.Pseudo: "nobold",
-        Keyword.Type: "nobold #B00040",
+        Operator: "#000000",
+        Operator.Word: "#0C33FE",

-        Operator: "#666666",
-        Operator.Word: "bold #AA22FF",
-
-        Name.Builtin: "#008000",
-        Name.Function: "#0000FF",
-        Name.Class: "bold #0000FF",
-        Name.Namespace: "bold #0000FF",
-        Name.Exception: "bold #D2413A",
-        Name.Variable: "#19177C",
-        Name.Constant: "#880000",
-        Name.Label: "#A0A000",
-        Name.Entity: "bold #999999",
-        Name.Attribute: "#7D9029",
-        Name.Tag: "bold #008000",
+        Name: "#000000",
+        Name.Builtin: "#0C33FE",
+        Name.Function: "#0C33FE",
+        # Name.Class: "#000000",
+        # Name.Namespace: "#000000",
+        # Name.Exception: "bold #D2413A",
+        # Name.Variable: "#000000",
+        # Name.Constant: "#880000",
+        # Name.Label: "#000000",
+        # Name.Entity: "bold #999999",
+        # Name.Attribute: "#7D9029",
+        # Name.Tag: "bold #008000",
        Name.Decorator: "#AA22FF",
-        Name.XojoType: '#010101',
+        Name.XojoType: "#0C33FE",

-        String: "#BA2121",
-        String.Doc: "italic",
-        String.Interpol: "bold #BB6688",
-        String.Escape: "bold #BB6622",
-        String.Regex: "#BB6688",
-        #String.Symbol: "#B8860B",
-        String.Symbol: "#19177C",
-        String.Other: "#008000",
-        Number: "#666666",
+        String: "#7A35FD",
+        # String.Doc: "italic",
+        # String.Interpol: "bold #BB6688",
+        # String.Escape: "bold #BB6622",
+        # String.Regex: "#BB6688",
+        # String.Symbol: "#B8860B",
+        # String.Symbol: "#19177C",
+        # String.Other: "#008000",
+        
+        Number: "#417AA8",
+        Number.Float: "#007642",
+        
+        Literal: "#417AA8",
+        Literal.Unicode: "#7A35FD",
+        Name.XojoType.Color: "#000000",
+        Name.XojoType.Color.Red: "#BA1600",
+        Name.XojoType.Color.Green: "#00AB0D",
+        Name.XojoType.Color.Blue: "#0520AF",
+        Name.XojoType.Color.Alpha: "#000000",

-        Generic.Heading: "bold #000080",
-        Generic.Subheading: "bold #800080",
-        Generic.Deleted: "#A00000",
-        Generic.Inserted: "#00A000",
-        Generic.Error: "#FF0000",
-        Generic.Emph: "italic",
-        Generic.Strong: "bold",
-        Generic.Prompt: "bold #000080",
-        Generic.Output: "#888",
-        Generic.Traceback: "#04D",
+        # Generic.Heading: "bold #000080",
+        # Generic.Subheading: "bold #800080",
+        # Generic.Deleted: "#A00000",
+        # Generic.Inserted: "#00A000",
+        # Generic.Error: "#FF0000",
+        # Generic.Emph: "italic",
+        # Generic.Strong: "bold",
+        # Generic.Prompt: "bold #000080",
+        # Generic.Output: "#888",
+        # Generic.Traceback: "#04D",

-        Error: "border:#FF0000"
+        Error: "border:#FF2500 bg:#E5E5E5"
        }
--- a/pygments_xojo/tests/test_xojo.py
+++ b/pygments_xojo/tests/test_xojo.py
@ -159,11 +159,18 @@ def test_smoke():
        (Text, u' '),
        (Punctuation, u'_'),
        (Text, u' '),
-        (Comment, u'//comment'),
-        (Text, u'\n'),
+        (Comment, u'//'),
+        (Comment, u'c'),
+        (Comment, u'o'),
+        (Comment, u'm'),
+        (Comment, u'm'),
+        (Comment, u'e'),
+        (Comment, u'n'),
+        (Comment, u't'),
+        (Comment, u'\n'),
        (Name.Variable, u'as_'),
        (Text, u'\n'),
-        (Name.Variable, u'String'),
+        (Keyword.Type, u'String'),
        ]),

    ('class Foo\n    property Test as Integer \n        get\n            return 1\n        end get\n        set\n            return\n        end set\n    end property\nend class', [