codekingpro/portable-devtools

1"""2    pygments.lexers.robotframework3    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~4 5    Lexer for Robot Framework.6 7    :copyright: Copyright 2006-present by the Pygments team, see AUTHORS.8    :license: BSD, see LICENSE for details.9"""10 11#  Copyright 2012 Nokia Siemens Networks Oyj12#13#  Licensed under the Apache License, Version 2.0 (the "License");14#  you may not use this file except in compliance with the License.15#  You may obtain a copy of the License at16#17#      http://www.apache.org/licenses/LICENSE-2.018#19#  Unless required by applicable law or agreed to in writing, software20#  distributed under the License is distributed on an "AS IS" BASIS,21#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.22#  See the License for the specific language governing permissions and23#  limitations under the License.24 25import re26 27from pygments.lexer import Lexer28from pygments.token import Token29 30__all__ = ['RobotFrameworkLexer']31 32 33HEADING = Token.Generic.Heading34SETTING = Token.Keyword.Namespace35IMPORT = Token.Name.Namespace36TC_KW_NAME = Token.Generic.Subheading37KEYWORD = Token.Name.Function38ARGUMENT = Token.String39VARIABLE = Token.Name.Variable40COMMENT = Token.Comment41SEPARATOR = Token.Punctuation42SYNTAX = Token.Punctuation43GHERKIN = Token.Generic.Emph44ERROR = Token.Error45 46 47def normalize(string, remove=''):48    string = string.lower()49    for char in remove + ' ':50        if char in string:51            string = string.replace(char, '')52    return string53 54 55class RobotFrameworkLexer(Lexer):56    """57    For Robot Framework test data.58 59    Supports both space and pipe separated plain text formats.60    """61    name = 'RobotFramework'62    url = 'http://robotframework.org'63    aliases = ['robotframework']64    filenames = ['*.robot', '*.resource']65    mimetypes = ['text/x-robotframework']66    version_added = '1.6'67 68    def __init__(self, **options):69        options['tabsize'] = 270        options['encoding'] = 'UTF-8'71        Lexer.__init__(self, **options)72 73    def get_tokens_unprocessed(self, text):74        row_tokenizer = RowTokenizer()75        var_tokenizer = VariableTokenizer()76        index = 077        for row in text.splitlines():78            for value, token in row_tokenizer.tokenize(row):79                for value, token in var_tokenizer.tokenize(value, token):80                    if value:81                        yield index, token, str(value)82                        index += len(value)83 84 85class VariableTokenizer:86 87    def tokenize(self, string, token):88        var = VariableSplitter(string, identifiers='$@%&')89        if var.start < 0 or token in (COMMENT, ERROR):90            yield string, token91            return92        for value, token in self._tokenize(var, string, token):93            if value:94                yield value, token95 96    def _tokenize(self, var, string, orig_token):97        before = string[:var.start]98        yield before, orig_token99        yield var.identifier + '{', SYNTAX100        yield from self.tokenize(var.base, VARIABLE)101        yield '}', SYNTAX102        if var.index is not None:103            yield '[', SYNTAX104            yield from self.tokenize(var.index, VARIABLE)105            yield ']', SYNTAX106        yield from self.tokenize(string[var.end:], orig_token)107 108 109class RowTokenizer:110 111    def __init__(self):112        self._table = UnknownTable()113        self._splitter = RowSplitter()114        testcases = TestCaseTable()115        settings = SettingTable(testcases.set_default_template)116        variables = VariableTable()117        keywords = KeywordTable()118        self._tables = {'settings': settings, 'setting': settings,119                        'metadata': settings,120                        'variables': variables, 'variable': variables,121                        'testcases': testcases, 'testcase': testcases,122                        'tasks': testcases, 'task': testcases,123                        'keywords': keywords, 'keyword': keywords,124                        'userkeywords': keywords, 'userkeyword': keywords}125 126    def tokenize(self, row):127        commented = False128        heading = False129        for index, value in enumerate(self._splitter.split(row)):130            # First value, and every second after that, is a separator.131            index, separator = divmod(index-1, 2)132            if value.startswith('#'):133                commented = True134            elif index == 0 and value.startswith('*'):135                self._table = self._start_table(value)136                heading = True137            yield from self._tokenize(value, index, commented,138                                      separator, heading)139        self._table.end_row()140 141    def _start_table(self, header):142        name = normalize(header, remove='*')143        return self._tables.get(name, UnknownTable())144 145    def _tokenize(self, value, index, commented, separator, heading):146        if commented:147            yield value, COMMENT148        elif separator:149            yield value, SEPARATOR150        elif heading:151            yield value, HEADING152        else:153            yield from self._table.tokenize(value, index)154 155 156class RowSplitter:157    _space_splitter = re.compile('( {2,})')158    _pipe_splitter = re.compile(r'((?:^| +)\|(?: +|$))')159 160    def split(self, row):161        splitter = (row.startswith('| ') and self._split_from_pipes162                    or self._split_from_spaces)163        yield from splitter(row)164        yield '\n'165 166    def _split_from_spaces(self, row):167        yield ''  # Start with (pseudo)separator similarly as with pipes168        yield from self._space_splitter.split(row)169 170    def _split_from_pipes(self, row):171        _, separator, rest = self._pipe_splitter.split(row, 1)172        yield separator173        while self._pipe_splitter.search(rest):174            cell, separator, rest = self._pipe_splitter.split(rest, 1)175            yield cell176            yield separator177        yield rest178 179 180class Tokenizer:181    _tokens = None182 183    def __init__(self):184        self._index = 0185 186    def tokenize(self, value):187        values_and_tokens = self._tokenize(value, self._index)188        self._index += 1189        if isinstance(values_and_tokens, type(Token)):190            values_and_tokens = [(value, values_and_tokens)]191        return values_and_tokens192 193    def _tokenize(self, value, index):194        index = min(index, len(self._tokens) - 1)195        return self._tokens[index]196 197    def _is_assign(self, value):198        if value.endswith('='):199            value = value[:-1].strip()200        var = VariableSplitter(value, identifiers='$@&')201        return var.start == 0 and var.end == len(value)202 203 204class Comment(Tokenizer):205    _tokens = (COMMENT,)206 207 208class Setting(Tokenizer):209    _tokens = (SETTING, ARGUMENT)210    _keyword_settings = ('suitesetup', 'suiteprecondition', 'suiteteardown',211                         'suitepostcondition', 'testsetup', 'tasksetup', 'testprecondition',212                         'testteardown','taskteardown', 'testpostcondition', 'testtemplate', 'tasktemplate')213    _import_settings = ('library', 'resource', 'variables')214    _other_settings = ('documentation', 'metadata', 'forcetags', 'defaulttags',215                       'testtimeout','tasktimeout')216    _custom_tokenizer = None217 218    def __init__(self, template_setter=None):219        Tokenizer.__init__(self)220        self._template_setter = template_setter221 222    def _tokenize(self, value, index):223        if index == 1 and self._template_setter:224            self._template_setter(value)225        if index == 0:226            normalized = normalize(value)227            if normalized in self._keyword_settings:228                self._custom_tokenizer = KeywordCall(support_assign=False)229            elif normalized in self._import_settings:230                self._custom_tokenizer = ImportSetting()231            elif normalized not in self._other_settings:232                return ERROR233        elif self._custom_tokenizer:234            return self._custom_tokenizer.tokenize(value)235        return Tokenizer._tokenize(self, value, index)236 237 238class ImportSetting(Tokenizer):239    _tokens = (IMPORT, ARGUMENT)240 241 242class TestCaseSetting(Setting):243    _keyword_settings = ('setup', 'precondition', 'teardown', 'postcondition',244                         'template')245    _import_settings = ()246    _other_settings = ('documentation', 'tags', 'timeout')247 248    def _tokenize(self, value, index):249        if index == 0:250            type = Setting._tokenize(self, value[1:-1], index)251            return [('[', SYNTAX), (value[1:-1], type), (']', SYNTAX)]252        return Setting._tokenize(self, value, index)253 254 255class KeywordSetting(TestCaseSetting):256    _keyword_settings = ('teardown',)257    _other_settings = ('documentation', 'arguments', 'return', 'timeout', 'tags')258 259 260class Variable(Tokenizer):261    _tokens = (SYNTAX, ARGUMENT)262 263    def _tokenize(self, value, index):264        if index == 0 and not self._is_assign(value):265            return ERROR266        return Tokenizer._tokenize(self, value, index)267 268 269class KeywordCall(Tokenizer):270    _tokens = (KEYWORD, ARGUMENT)271 272    def __init__(self, support_assign=True):273        Tokenizer.__init__(self)274        self._keyword_found = not support_assign275        self._assigns = 0276 277    def _tokenize(self, value, index):278        if not self._keyword_found and self._is_assign(value):279            self._assigns += 1280            return SYNTAX  # VariableTokenizer tokenizes this later.281        if self._keyword_found:282            return Tokenizer._tokenize(self, value, index - self._assigns)283        self._keyword_found = True284        return GherkinTokenizer().tokenize(value, KEYWORD)285 286 287class GherkinTokenizer:288    _gherkin_prefix = re.compile('^(Given|When|Then|And|But) ', re.IGNORECASE)289 290    def tokenize(self, value, token):291        match = self._gherkin_prefix.match(value)292        if not match:293            return [(value, token)]294        end = match.end()295        return [(value[:end], GHERKIN), (value[end:], token)]296 297 298class TemplatedKeywordCall(Tokenizer):299    _tokens = (ARGUMENT,)300 301 302class ForLoop(Tokenizer):303 304    def __init__(self):305        Tokenizer.__init__(self)306        self._in_arguments = False307 308    def _tokenize(self, value, index):309        token = self._in_arguments and ARGUMENT or SYNTAX310        if value.upper() in ('IN', 'IN RANGE'):311            self._in_arguments = True312        return token313 314 315class _Table:316    _tokenizer_class = None317 318    def __init__(self, prev_tokenizer=None):319        self._tokenizer = self._tokenizer_class()320        self._prev_tokenizer = prev_tokenizer321        self._prev_values_on_row = []322 323    def tokenize(self, value, index):324        if self._continues(value, index):325            self._tokenizer = self._prev_tokenizer326            yield value, SYNTAX327        else:328            yield from self._tokenize(value, index)329        self._prev_values_on_row.append(value)330 331    def _continues(self, value, index):332        return value == '...' and all(self._is_empty(t)333                                      for t in self._prev_values_on_row)334 335    def _is_empty(self, value):336        return value in ('', '\\')337 338    def _tokenize(self, value, index):339        return self._tokenizer.tokenize(value)340 341    def end_row(self):342        self.__init__(prev_tokenizer=self._tokenizer)343 344 345class UnknownTable(_Table):346    _tokenizer_class = Comment347 348    def _continues(self, value, index):349        return False350 351 352class VariableTable(_Table):353    _tokenizer_class = Variable354 355 356class SettingTable(_Table):357    _tokenizer_class = Setting358 359    def __init__(self, template_setter, prev_tokenizer=None):360        _Table.__init__(self, prev_tokenizer)361        self._template_setter = template_setter362 363    def _tokenize(self, value, index):364        if index == 0 and normalize(value) == 'testtemplate':365            self._tokenizer = Setting(self._template_setter)366        return _Table._tokenize(self, value, index)367 368    def end_row(self):369        self.__init__(self._template_setter, prev_tokenizer=self._tokenizer)370 371 372class TestCaseTable(_Table):373    _setting_class = TestCaseSetting374    _test_template = None375    _default_template = None376 377    @property378    def _tokenizer_class(self):379        if self._test_template or (self._default_template and380                                   self._test_template is not False):381            return TemplatedKeywordCall382        return KeywordCall383 384    def _continues(self, value, index):385        return index > 0 and _Table._continues(self, value, index)386 387    def _tokenize(self, value, index):388        if index == 0:389            if value:390                self._test_template = None391            return GherkinTokenizer().tokenize(value, TC_KW_NAME)392        if index == 1 and self._is_setting(value):393            if self._is_template(value):394                self._test_template = False395                self._tokenizer = self._setting_class(self.set_test_template)396            else:397                self._tokenizer = self._setting_class()398        if index == 1 and self._is_for_loop(value):399            self._tokenizer = ForLoop()400        if index == 1 and self._is_empty(value):401            return [(value, SYNTAX)]402        return _Table._tokenize(self, value, index)403 404    def _is_setting(self, value):405        return value.startswith('[') and value.endswith(']')406 407    def _is_template(self, value):408        return normalize(value) == '[template]'409 410    def _is_for_loop(self, value):411        return value.startswith(':') and normalize(value, remove=':') == 'for'412 413    def set_test_template(self, template):414        self._test_template = self._is_template_set(template)415 416    def set_default_template(self, template):417        self._default_template = self._is_template_set(template)418 419    def _is_template_set(self, template):420        return normalize(template) not in ('', '\\', 'none', '${empty}')421 422 423class KeywordTable(TestCaseTable):424    _tokenizer_class = KeywordCall425    _setting_class = KeywordSetting426 427    def _is_template(self, value):428        return False429 430 431# Following code copied directly from Robot Framework 2.7.5.432 433class VariableSplitter:434 435    def __init__(self, string, identifiers):436        self.identifier = None437        self.base = None438        self.index = None439        self.start = -1440        self.end = -1441        self._identifiers = identifiers442        self._may_have_internal_variables = False443        try:444            self._split(string)445        except ValueError:446            pass447        else:448            self._finalize()449 450    def get_replaced_base(self, variables):451        if self._may_have_internal_variables:452            return variables.replace_string(self.base)453        return self.base454 455    def _finalize(self):456        self.identifier = self._variable_chars[0]457        self.base = ''.join(self._variable_chars[2:-1])458        self.end = self.start + len(self._variable_chars)459        if self._has_list_or_dict_variable_index():460            self.index = ''.join(self._list_and_dict_variable_index_chars[1:-1])461            self.end += len(self._list_and_dict_variable_index_chars)462 463    def _has_list_or_dict_variable_index(self):464        return self._list_and_dict_variable_index_chars\465        and self._list_and_dict_variable_index_chars[-1] == ']'466 467    def _split(self, string):468        start_index, max_index = self._find_variable(string)469        self.start = start_index470        self._open_curly = 1471        self._state = self._variable_state472        self._variable_chars = [string[start_index], '{']473        self._list_and_dict_variable_index_chars = []474        self._string = string475        start_index += 2476        for index, char in enumerate(string[start_index:]):477            index += start_index  # Giving start to enumerate only in Py 2.6+478            try:479                self._state(char, index)480            except StopIteration:481                return482            if index  == max_index and not self._scanning_list_variable_index():483                return484 485    def _scanning_list_variable_index(self):486        return self._state in [self._waiting_list_variable_index_state,487                               self._list_variable_index_state]488 489    def _find_variable(self, string):490        max_end_index = string.rfind('}')491        if max_end_index == -1:492            raise ValueError('No variable end found')493        if self._is_escaped(string, max_end_index):494            return self._find_variable(string[:max_end_index])495        start_index = self._find_start_index(string, 1, max_end_index)496        if start_index == -1:497            raise ValueError('No variable start found')498        return start_index, max_end_index499 500    def _find_start_index(self, string, start, end):501        index = string.find('{', start, end) - 1502        if index < 0:503            return -1504        if self._start_index_is_ok(string, index):505            return index506        return self._find_start_index(string, index+2, end)507 508    def _start_index_is_ok(self, string, index):509        return string[index] in self._identifiers\510        and not self._is_escaped(string, index)511 512    def _is_escaped(self, string, index):513        escaped = False514        while index > 0 and string[index-1] == '\\':515            index -= 1516            escaped = not escaped517        return escaped518 519    def _variable_state(self, char, index):520        self._variable_chars.append(char)521        if char == '}' and not self._is_escaped(self._string, index):522            self._open_curly -= 1523            if self._open_curly == 0:524                if not self._is_list_or_dict_variable():525                    raise StopIteration526                self._state = self._waiting_list_variable_index_state527        elif char in self._identifiers:528            self._state = self._internal_variable_start_state529 530    def _is_list_or_dict_variable(self):531        return self._variable_chars[0] in ('@','&')532 533    def _internal_variable_start_state(self, char, index):534        self._state = self._variable_state535        if char == '{':536            self._variable_chars.append(char)537            self._open_curly += 1538            self._may_have_internal_variables = True539        else:540            self._variable_state(char, index)541 542    def _waiting_list_variable_index_state(self, char, index):543        if char != '[':544            raise StopIteration545        self._list_and_dict_variable_index_chars.append(char)546        self._state = self._list_variable_index_state547 548    def _list_variable_index_state(self, char, index):549        self._list_and_dict_variable_index_chars.append(char)550        if char == ']':551            raise StopIteration552