diff --git a/python/helpers/pydev/ply/__init__.py b/python/helpers/pydev/ply/__init__.py deleted file mode 100644 index 2461a4491215..000000000000 --- a/python/helpers/pydev/ply/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# PLY package -# Author: David Beazley (dave@dabeaz.com) - -__version__ = '3.7' -__all__ = ['lex','yacc'] diff --git a/python/helpers/pydev/ply/cpp.py b/python/helpers/pydev/ply/cpp.py deleted file mode 100644 index 2f6a0300de79..000000000000 --- a/python/helpers/pydev/ply/cpp.py +++ /dev/null @@ -1,908 +0,0 @@ -# ----------------------------------------------------------------------------- -# cpp.py -# -# Author: David Beazley (http://www.dabeaz.com) -# Copyright (C) 2007 -# All rights reserved -# -# This module implements an ANSI-C style lexical preprocessor for PLY. -# ----------------------------------------------------------------------------- -from __future__ import generators - -# ----------------------------------------------------------------------------- -# Default preprocessor lexer definitions. These tokens are enough to get -# a basic preprocessor working. Other modules may import these if they want -# ----------------------------------------------------------------------------- - -tokens = ( - 'CPP_ID','CPP_INTEGER', 'CPP_FLOAT', 'CPP_STRING', 'CPP_CHAR', 'CPP_WS', 'CPP_COMMENT1', 'CPP_COMMENT2', 'CPP_POUND','CPP_DPOUND' -) - -literals = "+-*/%|&~^<>=!?()[]{}.,;:\\\'\"" - -# Whitespace -def t_CPP_WS(t): - r'\s+' - t.lexer.lineno += t.value.count("\n") - return t - -t_CPP_POUND = r'\#' -t_CPP_DPOUND = r'\#\#' - -# Identifier -t_CPP_ID = r'[A-Za-z_][\w_]*' - -# Integer literal -def CPP_INTEGER(t): - r'(((((0x)|(0X))[0-9a-fA-F]+)|(\d+))([uU][lL]|[lL][uU]|[uU]|[lL])?)' - return t - -t_CPP_INTEGER = CPP_INTEGER - -# Floating literal -t_CPP_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' - -# String literal -def t_CPP_STRING(t): - r'\"([^\\\n]|(\\(.|\n)))*?\"' - t.lexer.lineno += t.value.count("\n") - return t - -# Character constant 'c' or L'c' -def t_CPP_CHAR(t): - r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' - t.lexer.lineno += t.value.count("\n") - return t - -# Comment -def t_CPP_COMMENT1(t): - r'(/\*(.|\n)*?\*/)' - ncr = t.value.count("\n") - t.lexer.lineno += ncr - # replace with one space or a number of '\n' - t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' ' - return t - -# Line comment -def t_CPP_COMMENT2(t): - r'(//.*?(\n|$))' - # replace with '/n' - t.type = 'CPP_WS'; t.value = '\n' - -def t_error(t): - t.type = t.value[0] - t.value = t.value[0] - t.lexer.skip(1) - return t - -import re -import copy -import time -import os.path - -# ----------------------------------------------------------------------------- -# trigraph() -# -# Given an input string, this function replaces all trigraph sequences. -# The following mapping is used: -# -# ??= # -# ??/ \ -# ??' ^ -# ??( [ -# ??) ] -# ??! | -# ??< { -# ??> } -# ??- ~ -# ----------------------------------------------------------------------------- - -_trigraph_pat = re.compile(r'''\?\?[=/\'\(\)\!<>\-]''') -_trigraph_rep = { - '=':'#', - '/':'\\', - "'":'^', - '(':'[', - ')':']', - '!':'|', - '<':'{', - '>':'}', - '-':'~' -} - -def trigraph(input): - return _trigraph_pat.sub(lambda g: _trigraph_rep[g.group()[-1]],input) - -# ------------------------------------------------------------------ -# Macro object -# -# This object holds information about preprocessor macros -# -# .name - Macro name (string) -# .value - Macro value (a list of tokens) -# .arglist - List of argument names -# .variadic - Boolean indicating whether or not variadic macro -# .vararg - Name of the variadic parameter -# -# When a macro is created, the macro replacement token sequence is -# pre-scanned and used to create patch lists that are later used -# during macro expansion -# ------------------------------------------------------------------ - -class Macro(object): - def __init__(self,name,value,arglist=None,variadic=False): - self.name = name - self.value = value - self.arglist = arglist - self.variadic = variadic - if variadic: - self.vararg = arglist[-1] - self.source = None - -# ------------------------------------------------------------------ -# Preprocessor object -# -# Object representing a preprocessor. Contains macro definitions, -# include directories, and other information -# ------------------------------------------------------------------ - -class Preprocessor(object): - def __init__(self,lexer=None): - if lexer is None: - lexer = lex.lexer - self.lexer = lexer - self.macros = { } - self.path = [] - self.temp_path = [] - - # Probe the lexer for selected tokens - self.lexprobe() - - tm = time.localtime() - self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) - self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) - self.parser = None - - # ----------------------------------------------------------------------------- - # tokenize() - # - # Utility function. Given a string of text, tokenize into a list of tokens - # ----------------------------------------------------------------------------- - - def tokenize(self,text): - tokens = [] - self.lexer.input(text) - while True: - tok = self.lexer.token() - if not tok: break - tokens.append(tok) - return tokens - - # --------------------------------------------------------------------- - # error() - # - # Report a preprocessor error/warning of some kind - # ---------------------------------------------------------------------- - - def error(self,file,line,msg): - print("%s:%d %s" % (file,line,msg)) - - # ---------------------------------------------------------------------- - # lexprobe() - # - # This method probes the preprocessor lexer object to discover - # the token types of symbols that are important to the preprocessor. - # If this works right, the preprocessor will simply "work" - # with any suitable lexer regardless of how tokens have been named. - # ---------------------------------------------------------------------- - - def lexprobe(self): - - # Determine the token type for identifiers - self.lexer.input("identifier") - tok = self.lexer.token() - if not tok or tok.value != "identifier": - print("Couldn't determine identifier type") - else: - self.t_ID = tok.type - - # Determine the token type for integers - self.lexer.input("12345") - tok = self.lexer.token() - if not tok or int(tok.value) != 12345: - print("Couldn't determine integer type") - else: - self.t_INTEGER = tok.type - self.t_INTEGER_TYPE = type(tok.value) - - # Determine the token type for strings enclosed in double quotes - self.lexer.input("\"filename\"") - tok = self.lexer.token() - if not tok or tok.value != "\"filename\"": - print("Couldn't determine string type") - else: - self.t_STRING = tok.type - - # Determine the token type for whitespace--if any - self.lexer.input(" ") - tok = self.lexer.token() - if not tok or tok.value != " ": - self.t_SPACE = None - else: - self.t_SPACE = tok.type - - # Determine the token type for newlines - self.lexer.input("\n") - tok = self.lexer.token() - if not tok or tok.value != "\n": - self.t_NEWLINE = None - print("Couldn't determine token for newlines") - else: - self.t_NEWLINE = tok.type - - self.t_WS = (self.t_SPACE, self.t_NEWLINE) - - # Check for other characters used by the preprocessor - chars = [ '<','>','#','##','\\','(',')',',','.'] - for c in chars: - self.lexer.input(c) - tok = self.lexer.token() - if not tok or tok.value != c: - print("Unable to lex '%s' required for preprocessor" % c) - - # ---------------------------------------------------------------------- - # add_path() - # - # Adds a search path to the preprocessor. - # ---------------------------------------------------------------------- - - def add_path(self,path): - self.path.append(path) - - # ---------------------------------------------------------------------- - # group_lines() - # - # Given an input string, this function splits it into lines. Trailing whitespace - # is removed. Any line ending with \ is grouped with the next line. This - # function forms the lowest level of the preprocessor---grouping into text into - # a line-by-line format. - # ---------------------------------------------------------------------- - - def group_lines(self,input): - lex = self.lexer.clone() - lines = [x.rstrip() for x in input.splitlines()] - for i in xrange(len(lines)): - j = i+1 - while lines[i].endswith('\\') and (j < len(lines)): - lines[i] = lines[i][:-1]+lines[j] - lines[j] = "" - j += 1 - - input = "\n".join(lines) - lex.input(input) - lex.lineno = 1 - - current_line = [] - while True: - tok = lex.token() - if not tok: - break - current_line.append(tok) - if tok.type in self.t_WS and '\n' in tok.value: - yield current_line - current_line = [] - - if current_line: - yield current_line - - # ---------------------------------------------------------------------- - # tokenstrip() - # - # Remove leading/trailing whitespace tokens from a token list - # ---------------------------------------------------------------------- - - def tokenstrip(self,tokens): - i = 0 - while i < len(tokens) and tokens[i].type in self.t_WS: - i += 1 - del tokens[:i] - i = len(tokens)-1 - while i >= 0 and tokens[i].type in self.t_WS: - i -= 1 - del tokens[i+1:] - return tokens - - - # ---------------------------------------------------------------------- - # collect_args() - # - # Collects comma separated arguments from a list of tokens. The arguments - # must be enclosed in parenthesis. Returns a tuple (tokencount,args,positions) - # where tokencount is the number of tokens consumed, args is a list of arguments, - # and positions is a list of integers containing the starting index of each - # argument. Each argument is represented by a list of tokens. - # - # When collecting arguments, leading and trailing whitespace is removed - # from each argument. - # - # This function properly handles nested parenthesis and commas---these do not - # define new arguments. - # ---------------------------------------------------------------------- - - def collect_args(self,tokenlist): - args = [] - positions = [] - current_arg = [] - nesting = 1 - tokenlen = len(tokenlist) - - # Search for the opening '('. - i = 0 - while (i < tokenlen) and (tokenlist[i].type in self.t_WS): - i += 1 - - if (i < tokenlen) and (tokenlist[i].value == '('): - positions.append(i+1) - else: - self.error(self.source,tokenlist[0].lineno,"Missing '(' in macro arguments") - return 0, [], [] - - i += 1 - - while i < tokenlen: - t = tokenlist[i] - if t.value == '(': - current_arg.append(t) - nesting += 1 - elif t.value == ')': - nesting -= 1 - if nesting == 0: - if current_arg: - args.append(self.tokenstrip(current_arg)) - positions.append(i) - return i+1,args,positions - current_arg.append(t) - elif t.value == ',' and nesting == 1: - args.append(self.tokenstrip(current_arg)) - positions.append(i+1) - current_arg = [] - else: - current_arg.append(t) - i += 1 - - # Missing end argument - self.error(self.source,tokenlist[-1].lineno,"Missing ')' in macro arguments") - return 0, [],[] - - # ---------------------------------------------------------------------- - # macro_prescan() - # - # Examine the macro value (token sequence) and identify patch points - # This is used to speed up macro expansion later on---we'll know - # right away where to apply patches to the value to form the expansion - # ---------------------------------------------------------------------- - - def macro_prescan(self,macro): - macro.patch = [] # Standard macro arguments - macro.str_patch = [] # String conversion expansion - macro.var_comma_patch = [] # Variadic macro comma patch - i = 0 - while i < len(macro.value): - if macro.value[i].type == self.t_ID and macro.value[i].value in macro.arglist: - argnum = macro.arglist.index(macro.value[i].value) - # Conversion of argument to a string - if i > 0 and macro.value[i-1].value == '#': - macro.value[i] = copy.copy(macro.value[i]) - macro.value[i].type = self.t_STRING - del macro.value[i-1] - macro.str_patch.append((argnum,i-1)) - continue - # Concatenation - elif (i > 0 and macro.value[i-1].value == '##'): - macro.patch.append(('c',argnum,i-1)) - del macro.value[i-1] - continue - elif ((i+1) < len(macro.value) and macro.value[i+1].value == '##'): - macro.patch.append(('c',argnum,i)) - i += 1 - continue - # Standard expansion - else: - macro.patch.append(('e',argnum,i)) - elif macro.value[i].value == '##': - if macro.variadic and (i > 0) and (macro.value[i-1].value == ',') and \ - ((i+1) < len(macro.value)) and (macro.value[i+1].type == self.t_ID) and \ - (macro.value[i+1].value == macro.vararg): - macro.var_comma_patch.append(i-1) - i += 1 - macro.patch.sort(key=lambda x: x[2],reverse=True) - - # ---------------------------------------------------------------------- - # macro_expand_args() - # - # Given a Macro and list of arguments (each a token list), this method - # returns an expanded version of a macro. The return value is a token sequence - # representing the replacement macro tokens - # ---------------------------------------------------------------------- - - def macro_expand_args(self,macro,args): - # Make a copy of the macro token sequence - rep = [copy.copy(_x) for _x in macro.value] - - # Make string expansion patches. These do not alter the length of the replacement sequence - - str_expansion = {} - for argnum, i in macro.str_patch: - if argnum not in str_expansion: - str_expansion[argnum] = ('"%s"' % "".join([x.value for x in args[argnum]])).replace("\\","\\\\") - rep[i] = copy.copy(rep[i]) - rep[i].value = str_expansion[argnum] - - # Make the variadic macro comma patch. If the variadic macro argument is empty, we get rid - comma_patch = False - if macro.variadic and not args[-1]: - for i in macro.var_comma_patch: - rep[i] = None - comma_patch = True - - # Make all other patches. The order of these matters. It is assumed that the patch list - # has been sorted in reverse order of patch location since replacements will cause the - # size of the replacement sequence to expand from the patch point. - - expanded = { } - for ptype, argnum, i in macro.patch: - # Concatenation. Argument is left unexpanded - if ptype == 'c': - rep[i:i+1] = args[argnum] - # Normal expansion. Argument is macro expanded first - elif ptype == 'e': - if argnum not in expanded: - expanded[argnum] = self.expand_macros(args[argnum]) - rep[i:i+1] = expanded[argnum] - - # Get rid of removed comma if necessary - if comma_patch: - rep = [_i for _i in rep if _i] - - return rep - - - # ---------------------------------------------------------------------- - # expand_macros() - # - # Given a list of tokens, this function performs macro expansion. - # The expanded argument is a dictionary that contains macros already - # expanded. This is used to prevent infinite recursion. - # ---------------------------------------------------------------------- - - def expand_macros(self,tokens,expanded=None): - if expanded is None: - expanded = {} - i = 0 - while i < len(tokens): - t = tokens[i] - if t.type == self.t_ID: - if t.value in self.macros and t.value not in expanded: - # Yes, we found a macro match - expanded[t.value] = True - - m = self.macros[t.value] - if not m.arglist: - # A simple macro - ex = self.expand_macros([copy.copy(_x) for _x in m.value],expanded) - for e in ex: - e.lineno = t.lineno - tokens[i:i+1] = ex - i += len(ex) - else: - # A macro with arguments - j = i + 1 - while j < len(tokens) and tokens[j].type in self.t_WS: - j += 1 - if tokens[j].value == '(': - tokcount,args,positions = self.collect_args(tokens[j:]) - if not m.variadic and len(args) != len(m.arglist): - self.error(self.source,t.lineno,"Macro %s requires %d arguments" % (t.value,len(m.arglist))) - i = j + tokcount - elif m.variadic and len(args) < len(m.arglist)-1: - if len(m.arglist) > 2: - self.error(self.source,t.lineno,"Macro %s must have at least %d arguments" % (t.value, len(m.arglist)-1)) - else: - self.error(self.source,t.lineno,"Macro %s must have at least %d argument" % (t.value, len(m.arglist)-1)) - i = j + tokcount - else: - if m.variadic: - if len(args) == len(m.arglist)-1: - args.append([]) - else: - args[len(m.arglist)-1] = tokens[j+positions[len(m.arglist)-1]:j+tokcount-1] - del args[len(m.arglist):] - - # Get macro replacement text - rep = self.macro_expand_args(m,args) - rep = self.expand_macros(rep,expanded) - for r in rep: - r.lineno = t.lineno - tokens[i:j+tokcount] = rep - i += len(rep) - del expanded[t.value] - continue - elif t.value == '__LINE__': - t.type = self.t_INTEGER - t.value = self.t_INTEGER_TYPE(t.lineno) - - i += 1 - return tokens - - # ---------------------------------------------------------------------- - # evalexpr() - # - # Evaluate an expression token sequence for the purposes of evaluating - # integral expressions. - # ---------------------------------------------------------------------- - - def evalexpr(self,tokens): - # tokens = tokenize(line) - # Search for defined macros - i = 0 - while i < len(tokens): - if tokens[i].type == self.t_ID and tokens[i].value == 'defined': - j = i + 1 - needparen = False - result = "0L" - while j < len(tokens): - if tokens[j].type in self.t_WS: - j += 1 - continue - elif tokens[j].type == self.t_ID: - if tokens[j].value in self.macros: - result = "1L" - else: - result = "0L" - if not needparen: break - elif tokens[j].value == '(': - needparen = True - elif tokens[j].value == ')': - break - else: - self.error(self.source,tokens[i].lineno,"Malformed defined()") - j += 1 - tokens[i].type = self.t_INTEGER - tokens[i].value = self.t_INTEGER_TYPE(result) - del tokens[i+1:j+1] - i += 1 - tokens = self.expand_macros(tokens) - for i,t in enumerate(tokens): - if t.type == self.t_ID: - tokens[i] = copy.copy(t) - tokens[i].type = self.t_INTEGER - tokens[i].value = self.t_INTEGER_TYPE("0L") - elif t.type == self.t_INTEGER: - tokens[i] = copy.copy(t) - # Strip off any trailing suffixes - tokens[i].value = str(tokens[i].value) - while tokens[i].value[-1] not in "0123456789abcdefABCDEF": - tokens[i].value = tokens[i].value[:-1] - - expr = "".join([str(x.value) for x in tokens]) - expr = expr.replace("&&"," and ") - expr = expr.replace("||"," or ") - expr = expr.replace("!"," not ") - try: - result = eval(expr) - except StandardError: - self.error(self.source,tokens[0].lineno,"Couldn't evaluate expression") - result = 0 - return result - - # ---------------------------------------------------------------------- - # parsegen() - # - # Parse an input string/ - # ---------------------------------------------------------------------- - def parsegen(self,input,source=None): - - # Replace trigraph sequences - t = trigraph(input) - lines = self.group_lines(t) - - if not source: - source = "" - - self.define("__FILE__ \"%s\"" % source) - - self.source = source - chunk = [] - enable = True - iftrigger = False - ifstack = [] - - for x in lines: - for i,tok in enumerate(x): - if tok.type not in self.t_WS: break - if tok.value == '#': - # Preprocessor directive - - # insert necessary whitespace instead of eaten tokens - for tok in x: - if tok.type in self.t_WS and '\n' in tok.value: - chunk.append(tok) - - dirtokens = self.tokenstrip(x[i+1:]) - if dirtokens: - name = dirtokens[0].value - args = self.tokenstrip(dirtokens[1:]) - else: - name = "" - args = [] - - if name == 'define': - if enable: - for tok in self.expand_macros(chunk): - yield tok - chunk = [] - self.define(args) - elif name == 'include': - if enable: - for tok in self.expand_macros(chunk): - yield tok - chunk = [] - oldfile = self.macros['__FILE__'] - for tok in self.include(args): - yield tok - self.macros['__FILE__'] = oldfile - self.source = source - elif name == 'undef': - if enable: - for tok in self.expand_macros(chunk): - yield tok - chunk = [] - self.undef(args) - elif name == 'ifdef': - ifstack.append((enable,iftrigger)) - if enable: - if not args[0].value in self.macros: - enable = False - iftrigger = False - else: - iftrigger = True - elif name == 'ifndef': - ifstack.append((enable,iftrigger)) - if enable: - if args[0].value in self.macros: - enable = False - iftrigger = False - else: - iftrigger = True - elif name == 'if': - ifstack.append((enable,iftrigger)) - if enable: - result = self.evalexpr(args) - if not result: - enable = False - iftrigger = False - else: - iftrigger = True - elif name == 'elif': - if ifstack: - if ifstack[-1][0]: # We only pay attention if outer "if" allows this - if enable: # If already true, we flip enable False - enable = False - elif not iftrigger: # If False, but not triggered yet, we'll check expression - result = self.evalexpr(args) - if result: - enable = True - iftrigger = True - else: - self.error(self.source,dirtokens[0].lineno,"Misplaced #elif") - - elif name == 'else': - if ifstack: - if ifstack[-1][0]: - if enable: - enable = False - elif not iftrigger: - enable = True - iftrigger = True - else: - self.error(self.source,dirtokens[0].lineno,"Misplaced #else") - - elif name == 'endif': - if ifstack: - enable,iftrigger = ifstack.pop() - else: - self.error(self.source,dirtokens[0].lineno,"Misplaced #endif") - else: - # Unknown preprocessor directive - pass - - else: - # Normal text - if enable: - chunk.extend(x) - - for tok in self.expand_macros(chunk): - yield tok - chunk = [] - - # ---------------------------------------------------------------------- - # include() - # - # Implementation of file-inclusion - # ---------------------------------------------------------------------- - - def include(self,tokens): - # Try to extract the filename and then process an include file - if not tokens: - return - if tokens: - if tokens[0].value != '<' and tokens[0].type != self.t_STRING: - tokens = self.expand_macros(tokens) - - if tokens[0].value == '<': - # Include <...> - i = 1 - while i < len(tokens): - if tokens[i].value == '>': - break - i += 1 - else: - print("Malformed #include <...>") - return - filename = "".join([x.value for x in tokens[1:i]]) - path = self.path + [""] + self.temp_path - elif tokens[0].type == self.t_STRING: - filename = tokens[0].value[1:-1] - path = self.temp_path + [""] + self.path - else: - print("Malformed #include statement") - return - for p in path: - iname = os.path.join(p,filename) - try: - data = open(iname,"r").read() - dname = os.path.dirname(iname) - if dname: - self.temp_path.insert(0,dname) - for tok in self.parsegen(data,filename): - yield tok - if dname: - del self.temp_path[0] - break - except IOError: - pass - else: - print("Couldn't find '%s'" % filename) - - # ---------------------------------------------------------------------- - # define() - # - # Define a new macro - # ---------------------------------------------------------------------- - - def define(self,tokens): - if isinstance(tokens,(str,unicode)): - tokens = self.tokenize(tokens) - - linetok = tokens - try: - name = linetok[0] - if len(linetok) > 1: - mtype = linetok[1] - else: - mtype = None - if not mtype: - m = Macro(name.value,[]) - self.macros[name.value] = m - elif mtype.type in self.t_WS: - # A normal macro - m = Macro(name.value,self.tokenstrip(linetok[2:])) - self.macros[name.value] = m - elif mtype.value == '(': - # A macro with arguments - tokcount, args, positions = self.collect_args(linetok[1:]) - variadic = False - for a in args: - if variadic: - print("No more arguments may follow a variadic argument") - break - astr = "".join([str(_i.value) for _i in a]) - if astr == "...": - variadic = True - a[0].type = self.t_ID - a[0].value = '__VA_ARGS__' - variadic = True - del a[1:] - continue - elif astr[-3:] == "..." and a[0].type == self.t_ID: - variadic = True - del a[1:] - # If, for some reason, "." is part of the identifier, strip off the name for the purposes - # of macro expansion - if a[0].value[-3:] == '...': - a[0].value = a[0].value[:-3] - continue - if len(a) > 1 or a[0].type != self.t_ID: - print("Invalid macro argument") - break - else: - mvalue = self.tokenstrip(linetok[1+tokcount:]) - i = 0 - while i < len(mvalue): - if i+1 < len(mvalue): - if mvalue[i].type in self.t_WS and mvalue[i+1].value == '##': - del mvalue[i] - continue - elif mvalue[i].value == '##' and mvalue[i+1].type in self.t_WS: - del mvalue[i+1] - i += 1 - m = Macro(name.value,mvalue,[x[0].value for x in args],variadic) - self.macro_prescan(m) - self.macros[name.value] = m - else: - print("Bad macro definition") - except LookupError: - print("Bad macro definition") - - # ---------------------------------------------------------------------- - # undef() - # - # Undefine a macro - # ---------------------------------------------------------------------- - - def undef(self,tokens): - id = tokens[0].value - try: - del self.macros[id] - except LookupError: - pass - - # ---------------------------------------------------------------------- - # parse() - # - # Parse input text. - # ---------------------------------------------------------------------- - def parse(self,input,source=None,ignore={}): - self.ignore = ignore - self.parser = self.parsegen(input,source) - - # ---------------------------------------------------------------------- - # token() - # - # Method to return individual tokens - # ---------------------------------------------------------------------- - def token(self): - try: - while True: - tok = next(self.parser) - if tok.type not in self.ignore: return tok - except StopIteration: - self.parser = None - return None - -if __name__ == '__main__': - import ply.lex as lex - lexer = lex.lex() - - # Run a preprocessor - import sys - f = open(sys.argv[1]) - input = f.read() - - p = Preprocessor(lexer) - p.parse(input,sys.argv[1]) - while True: - tok = p.token() - if not tok: break - print(p.source, tok) - - - - - - - - - - - diff --git a/python/helpers/pydev/ply/ctokens.py b/python/helpers/pydev/ply/ctokens.py deleted file mode 100644 index f6f6952d605e..000000000000 --- a/python/helpers/pydev/ply/ctokens.py +++ /dev/null @@ -1,133 +0,0 @@ -# ---------------------------------------------------------------------- -# ctokens.py -# -# Token specifications for symbols in ANSI C and C++. This file is -# meant to be used as a library in other tokenizers. -# ---------------------------------------------------------------------- - -# Reserved words - -tokens = [ - # Literals (identifier, integer constant, float constant, string constant, char const) - 'ID', 'TYPEID', 'INTEGER', 'FLOAT', 'STRING', 'CHARACTER', - - # Operators (+,-,*,/,%,|,&,~,^,<<,>>, ||, &&, !, <, <=, >, >=, ==, !=) - 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MODULO', - 'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT', - 'LOR', 'LAND', 'LNOT', - 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', - - # Assignment (=, *=, /=, %=, +=, -=, <<=, >>=, &=, ^=, |=) - 'EQUALS', 'TIMESEQUAL', 'DIVEQUAL', 'MODEQUAL', 'PLUSEQUAL', 'MINUSEQUAL', - 'LSHIFTEQUAL','RSHIFTEQUAL', 'ANDEQUAL', 'XOREQUAL', 'OREQUAL', - - # Increment/decrement (++,--) - 'INCREMENT', 'DECREMENT', - - # Structure dereference (->) - 'ARROW', - - # Ternary operator (?) - 'TERNARY', - - # Delimeters ( ) [ ] { } , . ; : - 'LPAREN', 'RPAREN', - 'LBRACKET', 'RBRACKET', - 'LBRACE', 'RBRACE', - 'COMMA', 'PERIOD', 'SEMI', 'COLON', - - # Ellipsis (...) - 'ELLIPSIS', -] - -# Operators -t_PLUS = r'\+' -t_MINUS = r'-' -t_TIMES = r'\*' -t_DIVIDE = r'/' -t_MODULO = r'%' -t_OR = r'\|' -t_AND = r'&' -t_NOT = r'~' -t_XOR = r'\^' -t_LSHIFT = r'<<' -t_RSHIFT = r'>>' -t_LOR = r'\|\|' -t_LAND = r'&&' -t_LNOT = r'!' -t_LT = r'<' -t_GT = r'>' -t_LE = r'<=' -t_GE = r'>=' -t_EQ = r'==' -t_NE = r'!=' - -# Assignment operators - -t_EQUALS = r'=' -t_TIMESEQUAL = r'\*=' -t_DIVEQUAL = r'/=' -t_MODEQUAL = r'%=' -t_PLUSEQUAL = r'\+=' -t_MINUSEQUAL = r'-=' -t_LSHIFTEQUAL = r'<<=' -t_RSHIFTEQUAL = r'>>=' -t_ANDEQUAL = r'&=' -t_OREQUAL = r'\|=' -t_XOREQUAL = r'\^=' - -# Increment/decrement -t_INCREMENT = r'\+\+' -t_DECREMENT = r'--' - -# -> -t_ARROW = r'->' - -# ? -t_TERNARY = r'\?' - -# Delimeters -t_LPAREN = r'\(' -t_RPAREN = r'\)' -t_LBRACKET = r'\[' -t_RBRACKET = r'\]' -t_LBRACE = r'\{' -t_RBRACE = r'\}' -t_COMMA = r',' -t_PERIOD = r'\.' -t_SEMI = r';' -t_COLON = r':' -t_ELLIPSIS = r'\.\.\.' - -# Identifiers -t_ID = r'[A-Za-z_][A-Za-z0-9_]*' - -# Integer literal -t_INTEGER = r'\d+([uU]|[lL]|[uU][lL]|[lL][uU])?' - -# Floating literal -t_FLOAT = r'((\d+)(\.\d+)(e(\+|-)?(\d+))? | (\d+)e(\+|-)?(\d+))([lL]|[fF])?' - -# String literal -t_STRING = r'\"([^\\\n]|(\\.))*?\"' - -# Character constant 'c' or L'c' -t_CHARACTER = r'(L)?\'([^\\\n]|(\\.))*?\'' - -# Comment (C-Style) -def t_COMMENT(t): - r'/\*(.|\n)*?\*/' - t.lexer.lineno += t.value.count('\n') - return t - -# Comment (C++-Style) -def t_CPPCOMMENT(t): - r'//.*\n' - t.lexer.lineno += 1 - return t - - - - - - diff --git a/python/helpers/pydev/ply/lex.py b/python/helpers/pydev/ply/lex.py deleted file mode 100644 index 0883173ec934..000000000000 --- a/python/helpers/pydev/ply/lex.py +++ /dev/null @@ -1,1097 +0,0 @@ -# ----------------------------------------------------------------------------- -# ply: lex.py -# -# Copyright (C) 2001-2015, -# David M. Beazley (Dabeaz LLC) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of the David Beazley or Dabeaz LLC may be used to -# endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ----------------------------------------------------------------------------- - -__version__ = '3.8' -__tabversion__ = '3.8' - -import copy -import inspect -import os -import re -import sys -import types - -# This tuple contains known string types -try: - # Python 2.6 - StringTypes = (types.StringType, types.UnicodeType) -except AttributeError: - # Python 3.0 - StringTypes = (str, bytes) - -# This regular expression is used to match valid token names -_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$') - -# Exception thrown when invalid token encountered and no default error -# handler is defined. -class LexError(Exception): - def __init__(self, message, s): - self.args = (message,) - self.text = s - - -# Token class. This class is used to represent the tokens produced. -class LexToken(object): - def __str__(self): - return 'LexToken(%s,%r,%d,%d)' % (self.type, self.value, self.lineno, self.lexpos) - - def __repr__(self): - return str(self) - - -# This object is a stand-in for a logging object created by the -# logging module. - -class PlyLogger(object): - def __init__(self, f): - self.f = f - - def critical(self, msg, *args, **kwargs): - self.f.write((msg % args) + '\n') - - def warning(self, msg, *args, **kwargs): - self.f.write('WARNING: ' + (msg % args) + '\n') - - def error(self, msg, *args, **kwargs): - self.f.write('ERROR: ' + (msg % args) + '\n') - - info = critical - debug = critical - - -# Null logger is used when no output is generated. Does nothing. -class NullLogger(object): - def __getattribute__(self, name): - return self - - def __call__(self, *args, **kwargs): - return self - - -# ----------------------------------------------------------------------------- -# === Lexing Engine === -# -# The following Lexer class implements the lexer runtime. There are only -# a few public methods and attributes: -# -# input() - Store a new string in the lexer -# token() - Get the next token -# clone() - Clone the lexer -# -# lineno - Current line number -# lexpos - Current position in the input string -# ----------------------------------------------------------------------------- - -class Lexer: - def __init__(self): - self.lexre = None # Master regular expression. This is a list of - # tuples (re, findex) where re is a compiled - # regular expression and findex is a list - # mapping regex group numbers to rules - self.lexretext = None # Current regular expression strings - self.lexstatere = {} # Dictionary mapping lexer states to master regexs - self.lexstateretext = {} # Dictionary mapping lexer states to regex strings - self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names - self.lexstate = 'INITIAL' # Current lexer state - self.lexstatestack = [] # Stack of lexer states - self.lexstateinfo = None # State information - self.lexstateignore = {} # Dictionary of ignored characters for each state - self.lexstateerrorf = {} # Dictionary of error functions for each state - self.lexstateeoff = {} # Dictionary of eof functions for each state - self.lexreflags = 0 # Optional re compile flags - self.lexdata = None # Actual input data (as a string) - self.lexpos = 0 # Current position in input text - self.lexlen = 0 # Length of the input text - self.lexerrorf = None # Error rule (if any) - self.lexeoff = None # EOF rule (if any) - self.lextokens = None # List of valid tokens - self.lexignore = '' # Ignored characters - self.lexliterals = '' # Literal characters that can be passed through - self.lexmodule = None # Module - self.lineno = 1 # Current line number - self.lexoptimize = False # Optimized mode - - def clone(self, object=None): - c = copy.copy(self) - - # If the object parameter has been supplied, it means we are attaching the - # lexer to a new object. In this case, we have to rebind all methods in - # the lexstatere and lexstateerrorf tables. - - if object: - newtab = {} - for key, ritem in self.lexstatere.items(): - newre = [] - for cre, findex in ritem: - newfindex = [] - for f in findex: - if not f or not f[0]: - newfindex.append(f) - continue - newfindex.append((getattr(object, f[0].__name__), f[1])) - newre.append((cre, newfindex)) - newtab[key] = newre - c.lexstatere = newtab - c.lexstateerrorf = {} - for key, ef in self.lexstateerrorf.items(): - c.lexstateerrorf[key] = getattr(object, ef.__name__) - c.lexmodule = object - return c - - # ------------------------------------------------------------ - # writetab() - Write lexer information to a table file - # ------------------------------------------------------------ - def writetab(self, lextab, outputdir=''): - if isinstance(lextab, types.ModuleType): - raise IOError("Won't overwrite existing lextab module") - basetabmodule = lextab.split('.')[-1] - filename = os.path.join(outputdir, basetabmodule) + '.py' - with open(filename, 'w') as tf: - tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__)) - tf.write('_tabversion = %s\n' % repr(__tabversion__)) - tf.write('_lextokens = %s\n' % repr(self.lextokens)) - tf.write('_lexreflags = %s\n' % repr(self.lexreflags)) - tf.write('_lexliterals = %s\n' % repr(self.lexliterals)) - tf.write('_lexstateinfo = %s\n' % repr(self.lexstateinfo)) - - # Rewrite the lexstatere table, replacing function objects with function names - tabre = {} - for statename, lre in self.lexstatere.items(): - titem = [] - for (pat, func), retext, renames in zip(lre, self.lexstateretext[statename], self.lexstaterenames[statename]): - titem.append((retext, _funcs_to_names(func, renames))) - tabre[statename] = titem - - tf.write('_lexstatere = %s\n' % repr(tabre)) - tf.write('_lexstateignore = %s\n' % repr(self.lexstateignore)) - - taberr = {} - for statename, ef in self.lexstateerrorf.items(): - taberr[statename] = ef.__name__ if ef else None - tf.write('_lexstateerrorf = %s\n' % repr(taberr)) - - tabeof = {} - for statename, ef in self.lexstateeoff.items(): - tabeof[statename] = ef.__name__ if ef else None - tf.write('_lexstateeoff = %s\n' % repr(tabeof)) - - # ------------------------------------------------------------ - # readtab() - Read lexer information from a tab file - # ------------------------------------------------------------ - def readtab(self, tabfile, fdict): - if isinstance(tabfile, types.ModuleType): - lextab = tabfile - else: - exec('import %s' % tabfile) - lextab = sys.modules[tabfile] - - if getattr(lextab, '_tabversion', '0.0') != __tabversion__: - raise ImportError('Inconsistent PLY version') - - self.lextokens = lextab._lextokens - self.lexreflags = lextab._lexreflags - self.lexliterals = lextab._lexliterals - self.lextokens_all = self.lextokens | set(self.lexliterals) - self.lexstateinfo = lextab._lexstateinfo - self.lexstateignore = lextab._lexstateignore - self.lexstatere = {} - self.lexstateretext = {} - for statename, lre in lextab._lexstatere.items(): - titem = [] - txtitem = [] - for pat, func_name in lre: - titem.append((re.compile(pat, lextab._lexreflags | re.VERBOSE), _names_to_funcs(func_name, fdict))) - - self.lexstatere[statename] = titem - self.lexstateretext[statename] = txtitem - - self.lexstateerrorf = {} - for statename, ef in lextab._lexstateerrorf.items(): - self.lexstateerrorf[statename] = fdict[ef] - - self.lexstateeoff = {} - for statename, ef in lextab._lexstateeoff.items(): - self.lexstateeoff[statename] = fdict[ef] - - self.begin('INITIAL') - - # ------------------------------------------------------------ - # input() - Push a new string into the lexer - # ------------------------------------------------------------ - def input(self, s): - # Pull off the first character to see if s looks like a string - c = s[:1] - if not isinstance(c, StringTypes): - raise ValueError('Expected a string') - self.lexdata = s - self.lexpos = 0 - self.lexlen = len(s) - - # ------------------------------------------------------------ - # begin() - Changes the lexing state - # ------------------------------------------------------------ - def begin(self, state): - if state not in self.lexstatere: - raise ValueError('Undefined state') - self.lexre = self.lexstatere[state] - self.lexretext = self.lexstateretext[state] - self.lexignore = self.lexstateignore.get(state, '') - self.lexerrorf = self.lexstateerrorf.get(state, None) - self.lexeoff = self.lexstateeoff.get(state, None) - self.lexstate = state - - # ------------------------------------------------------------ - # push_state() - Changes the lexing state and saves old on stack - # ------------------------------------------------------------ - def push_state(self, state): - self.lexstatestack.append(self.lexstate) - self.begin(state) - - # ------------------------------------------------------------ - # pop_state() - Restores the previous state - # ------------------------------------------------------------ - def pop_state(self): - self.begin(self.lexstatestack.pop()) - - # ------------------------------------------------------------ - # current_state() - Returns the current lexing state - # ------------------------------------------------------------ - def current_state(self): - return self.lexstate - - # ------------------------------------------------------------ - # skip() - Skip ahead n characters - # ------------------------------------------------------------ - def skip(self, n): - self.lexpos += n - - # ------------------------------------------------------------ - # opttoken() - Return the next token from the Lexer - # - # Note: This function has been carefully implemented to be as fast - # as possible. Don't make changes unless you really know what - # you are doing - # ------------------------------------------------------------ - def token(self): - # Make local copies of frequently referenced attributes - lexpos = self.lexpos - lexlen = self.lexlen - lexignore = self.lexignore - lexdata = self.lexdata - - while lexpos < lexlen: - # This code provides some short-circuit code for whitespace, tabs, and other ignored characters - if lexdata[lexpos] in lexignore: - lexpos += 1 - continue - - # Look for a regular expression match - for lexre, lexindexfunc in self.lexre: - m = lexre.match(lexdata, lexpos) - if not m: - continue - - # Create a token for return - tok = LexToken() - tok.value = m.group() - tok.lineno = self.lineno - tok.lexpos = lexpos - - i = m.lastindex - func, tok.type = lexindexfunc[i] - - if not func: - # If no token type was set, it's an ignored token - if tok.type: - self.lexpos = m.end() - return tok - else: - lexpos = m.end() - break - - lexpos = m.end() - - # If token is processed by a function, call it - - tok.lexer = self # Set additional attributes useful in token rules - self.lexmatch = m - self.lexpos = lexpos - - newtok = func(tok) - - # Every function must return a token, if nothing, we just move to next token - if not newtok: - lexpos = self.lexpos # This is here in case user has updated lexpos. - lexignore = self.lexignore # This is here in case there was a state change - break - - # Verify type of the token. If not in the token map, raise an error - if not self.lexoptimize: - if newtok.type not in self.lextokens_all: - raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % ( - func.__code__.co_filename, func.__code__.co_firstlineno, - func.__name__, newtok.type), lexdata[lexpos:]) - - return newtok - else: - # No match, see if in literals - if lexdata[lexpos] in self.lexliterals: - tok = LexToken() - tok.value = lexdata[lexpos] - tok.lineno = self.lineno - tok.type = tok.value - tok.lexpos = lexpos - self.lexpos = lexpos + 1 - return tok - - # No match. Call t_error() if defined. - if self.lexerrorf: - tok = LexToken() - tok.value = self.lexdata[lexpos:] - tok.lineno = self.lineno - tok.type = 'error' - tok.lexer = self - tok.lexpos = lexpos - self.lexpos = lexpos - newtok = self.lexerrorf(tok) - if lexpos == self.lexpos: - # Error method didn't change text position at all. This is an error. - raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:]) - lexpos = self.lexpos - if not newtok: - continue - return newtok - - self.lexpos = lexpos - raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) - - if self.lexeoff: - tok = LexToken() - tok.type = 'eof' - tok.value = '' - tok.lineno = self.lineno - tok.lexpos = lexpos - tok.lexer = self - self.lexpos = lexpos - newtok = self.lexeoff(tok) - return newtok - - self.lexpos = lexpos + 1 - if self.lexdata is None: - raise RuntimeError('No input string given with input()') - return None - - # Iterator interface - def __iter__(self): - return self - - def next(self): - t = self.token() - if t is None: - raise StopIteration - return t - - __next__ = next - -# ----------------------------------------------------------------------------- -# ==== Lex Builder === -# -# The functions and classes below are used to collect lexing information -# and build a Lexer object from it. -# ----------------------------------------------------------------------------- - -# ----------------------------------------------------------------------------- -# _get_regex(func) -# -# Returns the regular expression assigned to a function either as a doc string -# or as a .regex attribute attached by the @TOKEN decorator. -# ----------------------------------------------------------------------------- -def _get_regex(func): - return getattr(func, 'regex', func.__doc__) - -# ----------------------------------------------------------------------------- -# get_caller_module_dict() -# -# This function returns a dictionary containing all of the symbols defined within -# a caller further down the call stack. This is used to get the environment -# associated with the yacc() call if none was provided. -# ----------------------------------------------------------------------------- -def get_caller_module_dict(levels): - f = sys._getframe(levels) - ldict = f.f_globals.copy() - if f.f_globals != f.f_locals: - ldict.update(f.f_locals) - return ldict - -# ----------------------------------------------------------------------------- -# _funcs_to_names() -# -# Given a list of regular expression functions, this converts it to a list -# suitable for output to a table file -# ----------------------------------------------------------------------------- -def _funcs_to_names(funclist, namelist): - result = [] - for f, name in zip(funclist, namelist): - if f and f[0]: - result.append((name, f[1])) - else: - result.append(f) - return result - -# ----------------------------------------------------------------------------- -# _names_to_funcs() -# -# Given a list of regular expression function names, this converts it back to -# functions. -# ----------------------------------------------------------------------------- -def _names_to_funcs(namelist, fdict): - result = [] - for n in namelist: - if n and n[0]: - result.append((fdict[n[0]], n[1])) - else: - result.append(n) - return result - -# ----------------------------------------------------------------------------- -# _form_master_re() -# -# This function takes a list of all of the regex components and attempts to -# form the master regular expression. Given limitations in the Python re -# module, it may be necessary to break the master regex into separate expressions. -# ----------------------------------------------------------------------------- -def _form_master_re(relist, reflags, ldict, toknames): - if not relist: - return [] - regex = '|'.join(relist) - try: - lexre = re.compile(regex, re.VERBOSE | reflags) - - # Build the index to function map for the matching engine - lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1) - lexindexnames = lexindexfunc[:] - - for f, i in lexre.groupindex.items(): - handle = ldict.get(f, None) - if type(handle) in (types.FunctionType, types.MethodType): - lexindexfunc[i] = (handle, toknames[f]) - lexindexnames[i] = f - elif handle is not None: - lexindexnames[i] = f - if f.find('ignore_') > 0: - lexindexfunc[i] = (None, None) - else: - lexindexfunc[i] = (None, toknames[f]) - - return [(lexre, lexindexfunc)], [regex], [lexindexnames] - except Exception: - m = int(len(relist)/2) - if m == 0: - m = 1 - llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames) - rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames) - return (llist+rlist), (lre+rre), (lnames+rnames) - -# ----------------------------------------------------------------------------- -# def _statetoken(s,names) -# -# Given a declaration name s of the form "t_" and a dictionary whose keys are -# state names, this function returns a tuple (states,tokenname) where states -# is a tuple of state names and tokenname is the name of the token. For example, -# calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM') -# ----------------------------------------------------------------------------- -def _statetoken(s, names): - nonstate = 1 - parts = s.split('_') - for i, part in enumerate(parts[1:], 1): - if part not in names and part != 'ANY': - break - - if i > 1: - states = tuple(parts[1:i]) - else: - states = ('INITIAL',) - - if 'ANY' in states: - states = tuple(names) - - tokenname = '_'.join(parts[i:]) - return (states, tokenname) - - -# ----------------------------------------------------------------------------- -# LexerReflect() -# -# This class represents information needed to build a lexer as extracted from a -# user's input file. -# ----------------------------------------------------------------------------- -class LexerReflect(object): - def __init__(self, ldict, log=None, reflags=0): - self.ldict = ldict - self.error_func = None - self.tokens = [] - self.reflags = reflags - self.stateinfo = {'INITIAL': 'inclusive'} - self.modules = set() - self.error = False - self.log = PlyLogger(sys.stderr) if log is None else log - - # Get all of the basic information - def get_all(self): - self.get_tokens() - self.get_literals() - self.get_states() - self.get_rules() - - # Validate all of the information - def validate_all(self): - self.validate_tokens() - self.validate_literals() - self.validate_rules() - return self.error - - # Get the tokens map - def get_tokens(self): - tokens = self.ldict.get('tokens', None) - if not tokens: - self.log.error('No token list is defined') - self.error = True - return - - if not isinstance(tokens, (list, tuple)): - self.log.error('tokens must be a list or tuple') - self.error = True - return - - if not tokens: - self.log.error('tokens is empty') - self.error = True - return - - self.tokens = tokens - - # Validate the tokens - def validate_tokens(self): - terminals = {} - for n in self.tokens: - if not _is_identifier.match(n): - self.log.error("Bad token name '%s'", n) - self.error = True - if n in terminals: - self.log.warning("Token '%s' multiply defined", n) - terminals[n] = 1 - - # Get the literals specifier - def get_literals(self): - self.literals = self.ldict.get('literals', '') - if not self.literals: - self.literals = '' - - # Validate literals - def validate_literals(self): - try: - for c in self.literals: - if not isinstance(c, StringTypes) or len(c) > 1: - self.log.error('Invalid literal %s. Must be a single character', repr(c)) - self.error = True - - except TypeError: - self.log.error('Invalid literals specification. literals must be a sequence of characters') - self.error = True - - def get_states(self): - self.states = self.ldict.get('states', None) - # Build statemap - if self.states: - if not isinstance(self.states, (tuple, list)): - self.log.error('states must be defined as a tuple or list') - self.error = True - else: - for s in self.states: - if not isinstance(s, tuple) or len(s) != 2: - self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')", repr(s)) - self.error = True - continue - name, statetype = s - if not isinstance(name, StringTypes): - self.log.error('State name %s must be a string', repr(name)) - self.error = True - continue - if not (statetype == 'inclusive' or statetype == 'exclusive'): - self.log.error("State type for state %s must be 'inclusive' or 'exclusive'", name) - self.error = True - continue - if name in self.stateinfo: - self.log.error("State '%s' already defined", name) - self.error = True - continue - self.stateinfo[name] = statetype - - # Get all of the symbols with a t_ prefix and sort them into various - # categories (functions, strings, error functions, and ignore characters) - - def get_rules(self): - tsymbols = [f for f in self.ldict if f[:2] == 't_'] - - # Now build up a list of functions and a list of strings - self.toknames = {} # Mapping of symbols to token names - self.funcsym = {} # Symbols defined as functions - self.strsym = {} # Symbols defined as strings - self.ignore = {} # Ignore strings by state - self.errorf = {} # Error functions by state - self.eoff = {} # EOF functions by state - - for s in self.stateinfo: - self.funcsym[s] = [] - self.strsym[s] = [] - - if len(tsymbols) == 0: - self.log.error('No rules of the form t_rulename are defined') - self.error = True - return - - for f in tsymbols: - t = self.ldict[f] - states, tokname = _statetoken(f, self.stateinfo) - self.toknames[f] = tokname - - if hasattr(t, '__call__'): - if tokname == 'error': - for s in states: - self.errorf[s] = t - elif tokname == 'eof': - for s in states: - self.eoff[s] = t - elif tokname == 'ignore': - line = t.__code__.co_firstlineno - file = t.__code__.co_filename - self.log.error("%s:%d: Rule '%s' must be defined as a string", file, line, t.__name__) - self.error = True - else: - for s in states: - self.funcsym[s].append((f, t)) - elif isinstance(t, StringTypes): - if tokname == 'ignore': - for s in states: - self.ignore[s] = t - if '\\' in t: - self.log.warning("%s contains a literal backslash '\\'", f) - - elif tokname == 'error': - self.log.error("Rule '%s' must be defined as a function", f) - self.error = True - else: - for s in states: - self.strsym[s].append((f, t)) - else: - self.log.error('%s not defined as a function or string', f) - self.error = True - - # Sort the functions by line number - for f in self.funcsym.values(): - f.sort(key=lambda x: x[1].__code__.co_firstlineno) - - # Sort the strings by regular expression length - for s in self.strsym.values(): - s.sort(key=lambda x: len(x[1]), reverse=True) - - # Validate all of the t_rules collected - def validate_rules(self): - for state in self.stateinfo: - # Validate all rules defined by functions - - for fname, f in self.funcsym[state]: - line = f.__code__.co_firstlineno - file = f.__code__.co_filename - module = inspect.getmodule(f) - self.modules.add(module) - - tokname = self.toknames[fname] - if isinstance(f, types.MethodType): - reqargs = 2 - else: - reqargs = 1 - nargs = f.__code__.co_argcount - if nargs > reqargs: - self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) - self.error = True - continue - - if nargs < reqargs: - self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__) - self.error = True - continue - - if not _get_regex(f): - self.log.error("%s:%d: No regular expression defined for rule '%s'", file, line, f.__name__) - self.error = True - continue - - try: - c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), re.VERBOSE | self.reflags) - if c.match(''): - self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file, line, f.__name__) - self.error = True - except re.error as e: - self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file, line, f.__name__, e) - if '#' in _get_regex(f): - self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'", file, line, f.__name__) - self.error = True - - # Validate all rules defined by strings - for name, r in self.strsym[state]: - tokname = self.toknames[name] - if tokname == 'error': - self.log.error("Rule '%s' must be defined as a function", name) - self.error = True - continue - - if tokname not in self.tokens and tokname.find('ignore_') < 0: - self.log.error("Rule '%s' defined for an unspecified token %s", name, tokname) - self.error = True - continue - - try: - c = re.compile('(?P<%s>%s)' % (name, r), re.VERBOSE | self.reflags) - if (c.match('')): - self.log.error("Regular expression for rule '%s' matches empty string", name) - self.error = True - except re.error as e: - self.log.error("Invalid regular expression for rule '%s'. %s", name, e) - if '#' in r: - self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'", name) - self.error = True - - if not self.funcsym[state] and not self.strsym[state]: - self.log.error("No rules defined for state '%s'", state) - self.error = True - - # Validate the error function - efunc = self.errorf.get(state, None) - if efunc: - f = efunc - line = f.__code__.co_firstlineno - file = f.__code__.co_filename - module = inspect.getmodule(f) - self.modules.add(module) - - if isinstance(f, types.MethodType): - reqargs = 2 - else: - reqargs = 1 - nargs = f.__code__.co_argcount - if nargs > reqargs: - self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) - self.error = True - - if nargs < reqargs: - self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__) - self.error = True - - for module in self.modules: - self.validate_module(module) - - # ----------------------------------------------------------------------------- - # validate_module() - # - # This checks to see if there are duplicated t_rulename() functions or strings - # in the parser input file. This is done using a simple regular expression - # match on each line in the source code of the given module. - # ----------------------------------------------------------------------------- - - def validate_module(self, module): - lines, linen = inspect.getsourcelines(module) - - fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(') - sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=') - - counthash = {} - linen += 1 - for line in lines: - m = fre.match(line) - if not m: - m = sre.match(line) - if m: - name = m.group(1) - prev = counthash.get(name) - if not prev: - counthash[name] = linen - else: - filename = inspect.getsourcefile(module) - self.log.error('%s:%d: Rule %s redefined. Previously defined on line %d', filename, linen, name, prev) - self.error = True - linen += 1 - -# ----------------------------------------------------------------------------- -# lex(module) -# -# Build all of the regular expression rules from definitions in the supplied module -# ----------------------------------------------------------------------------- -def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab', - reflags=0, nowarn=False, outputdir=None, debuglog=None, errorlog=None): - - if lextab is None: - lextab = 'lextab' - - global lexer - - ldict = None - stateinfo = {'INITIAL': 'inclusive'} - lexobj = Lexer() - lexobj.lexoptimize = optimize - global token, input - - if errorlog is None: - errorlog = PlyLogger(sys.stderr) - - if debug: - if debuglog is None: - debuglog = PlyLogger(sys.stderr) - - # Get the module dictionary used for the lexer - if object: - module = object - - # Get the module dictionary used for the parser - if module: - _items = [(k, getattr(module, k)) for k in dir(module)] - ldict = dict(_items) - # If no __file__ attribute is available, try to obtain it from the __module__ instead - if '__file__' not in ldict: - ldict['__file__'] = sys.modules[ldict['__module__']].__file__ - else: - ldict = get_caller_module_dict(2) - - # Determine if the module is package of a package or not. - # If so, fix the tabmodule setting so that tables load correctly - pkg = ldict.get('__package__') - if pkg and isinstance(lextab, str): - if '.' not in lextab: - lextab = pkg + '.' + lextab - - # Collect parser information from the dictionary - linfo = LexerReflect(ldict, log=errorlog, reflags=reflags) - linfo.get_all() - if not optimize: - if linfo.validate_all(): - raise SyntaxError("Can't build lexer") - - if optimize and lextab: - try: - lexobj.readtab(lextab, ldict) - token = lexobj.token - input = lexobj.input - lexer = lexobj - return lexobj - - except ImportError: - pass - - # Dump some basic debugging information - if debug: - debuglog.info('lex: tokens = %r', linfo.tokens) - debuglog.info('lex: literals = %r', linfo.literals) - debuglog.info('lex: states = %r', linfo.stateinfo) - - # Build a dictionary of valid token names - lexobj.lextokens = set() - for n in linfo.tokens: - lexobj.lextokens.add(n) - - # Get literals specification - if isinstance(linfo.literals, (list, tuple)): - lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals) - else: - lexobj.lexliterals = linfo.literals - - lexobj.lextokens_all = lexobj.lextokens | set(lexobj.lexliterals) - - # Get the stateinfo dictionary - stateinfo = linfo.stateinfo - - regexs = {} - # Build the master regular expressions - for state in stateinfo: - regex_list = [] - - # Add rules defined by functions first - for fname, f in linfo.funcsym[state]: - line = f.__code__.co_firstlineno - file = f.__code__.co_filename - regex_list.append('(?P<%s>%s)' % (fname, _get_regex(f))) - if debug: - debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", fname, _get_regex(f), state) - - # Now add all of the simple rules - for name, r in linfo.strsym[state]: - regex_list.append('(?P<%s>%s)' % (name, r)) - if debug: - debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", name, r, state) - - regexs[state] = regex_list - - # Build the master regular expressions - - if debug: - debuglog.info('lex: ==== MASTER REGEXS FOLLOW ====') - - for state in regexs: - lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames) - lexobj.lexstatere[state] = lexre - lexobj.lexstateretext[state] = re_text - lexobj.lexstaterenames[state] = re_names - if debug: - for i, text in enumerate(re_text): - debuglog.info("lex: state '%s' : regex[%d] = '%s'", state, i, text) - - # For inclusive states, we need to add the regular expressions from the INITIAL state - for state, stype in stateinfo.items(): - if state != 'INITIAL' and stype == 'inclusive': - lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL']) - lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL']) - lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL']) - - lexobj.lexstateinfo = stateinfo - lexobj.lexre = lexobj.lexstatere['INITIAL'] - lexobj.lexretext = lexobj.lexstateretext['INITIAL'] - lexobj.lexreflags = reflags - - # Set up ignore variables - lexobj.lexstateignore = linfo.ignore - lexobj.lexignore = lexobj.lexstateignore.get('INITIAL', '') - - # Set up error functions - lexobj.lexstateerrorf = linfo.errorf - lexobj.lexerrorf = linfo.errorf.get('INITIAL', None) - if not lexobj.lexerrorf: - errorlog.warning('No t_error rule is defined') - - # Set up eof functions - lexobj.lexstateeoff = linfo.eoff - lexobj.lexeoff = linfo.eoff.get('INITIAL', None) - - # Check state information for ignore and error rules - for s, stype in stateinfo.items(): - if stype == 'exclusive': - if s not in linfo.errorf: - errorlog.warning("No error rule is defined for exclusive state '%s'", s) - if s not in linfo.ignore and lexobj.lexignore: - errorlog.warning("No ignore rule is defined for exclusive state '%s'", s) - elif stype == 'inclusive': - if s not in linfo.errorf: - linfo.errorf[s] = linfo.errorf.get('INITIAL', None) - if s not in linfo.ignore: - linfo.ignore[s] = linfo.ignore.get('INITIAL', '') - - # Create global versions of the token() and input() functions - token = lexobj.token - input = lexobj.input - lexer = lexobj - - # If in optimize mode, we write the lextab - if lextab and optimize: - if outputdir is None: - # If no output directory is set, the location of the output files - # is determined according to the following rules: - # - If lextab specifies a package, files go into that package directory - # - Otherwise, files go in the same directory as the specifying module - if isinstance(lextab, types.ModuleType): - srcfile = lextab.__file__ - else: - if '.' not in lextab: - srcfile = ldict['__file__'] - else: - parts = lextab.split('.') - pkgname = '.'.join(parts[:-1]) - exec('import %s' % pkgname) - srcfile = getattr(sys.modules[pkgname], '__file__', '') - outputdir = os.path.dirname(srcfile) - try: - lexobj.writetab(lextab, outputdir) - except IOError as e: - errorlog.warning("Couldn't write lextab module %r. %s" % (lextab, e)) - - return lexobj - -# ----------------------------------------------------------------------------- -# runmain() -# -# This runs the lexer as a main program -# ----------------------------------------------------------------------------- - -def runmain(lexer=None, data=None): - if not data: - try: - filename = sys.argv[1] - f = open(filename) - data = f.read() - f.close() - except IndexError: - sys.stdout.write('Reading from standard input (type EOF to end):\n') - data = sys.stdin.read() - - if lexer: - _input = lexer.input - else: - _input = input - _input(data) - if lexer: - _token = lexer.token - else: - _token = token - - while True: - tok = _token() - if not tok: - break - sys.stdout.write('(%s,%r,%d,%d)\n' % (tok.type, tok.value, tok.lineno, tok.lexpos)) - -# ----------------------------------------------------------------------------- -# @TOKEN(regex) -# -# This decorator function can be used to set the regex expression on a function -# when its docstring might need to be set in an alternative way -# ----------------------------------------------------------------------------- - -def TOKEN(r): - def set_regex(f): - if hasattr(r, '__call__'): - f.regex = _get_regex(r) - else: - f.regex = r - return f - return set_regex - -# Alternative spelling of the TOKEN decorator -Token = TOKEN - diff --git a/python/helpers/pydev/ply/yacc.py b/python/helpers/pydev/ply/yacc.py deleted file mode 100644 index 63e74e7f4e26..000000000000 --- a/python/helpers/pydev/ply/yacc.py +++ /dev/null @@ -1,3471 +0,0 @@ -# ----------------------------------------------------------------------------- -# ply: yacc.py -# -# Copyright (C) 2001-2015, -# David M. Beazley (Dabeaz LLC) -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# * Neither the name of the David Beazley or Dabeaz LLC may be used to -# endorse or promote products derived from this software without -# specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# ----------------------------------------------------------------------------- -# -# This implements an LR parser that is constructed from grammar rules defined -# as Python functions. The grammer is specified by supplying the BNF inside -# Python documentation strings. The inspiration for this technique was borrowed -# from John Aycock's Spark parsing system. PLY might be viewed as cross between -# Spark and the GNU bison utility. -# -# The current implementation is only somewhat object-oriented. The -# LR parser itself is defined in terms of an object (which allows multiple -# parsers to co-exist). However, most of the variables used during table -# construction are defined in terms of global variables. Users shouldn't -# notice unless they are trying to define multiple parsers at the same -# time using threads (in which case they should have their head examined). -# -# This implementation supports both SLR and LALR(1) parsing. LALR(1) -# support was originally implemented by Elias Ioup (ezioup@alumni.uchicago.edu), -# using the algorithm found in Aho, Sethi, and Ullman "Compilers: Principles, -# Techniques, and Tools" (The Dragon Book). LALR(1) has since been replaced -# by the more efficient DeRemer and Pennello algorithm. -# -# :::::::: WARNING ::::::: -# -# Construction of LR parsing tables is fairly complicated and expensive. -# To make this module run fast, a *LOT* of work has been put into -# optimization---often at the expensive of readability and what might -# consider to be good Python "coding style." Modify the code at your -# own risk! -# ---------------------------------------------------------------------------- - -import base64 -import inspect -import os.path -import re -import sys -import types -import warnings - -__version__ = '3.8' -__tabversion__ = '3.8' - -#----------------------------------------------------------------------------- -# === User configurable parameters === -# -# Change these to modify the default behavior of yacc (if you wish) -#----------------------------------------------------------------------------- - -yaccdebug = True # Debugging mode. If set, yacc generates a - # a 'parser.out' file in the current directory - -debug_file = 'parser.out' # Default name of the debugging file -tab_module = 'parsetab' # Default name of the table module -default_lr = 'LALR' # Default LR table generation method - -error_count = 3 # Number of symbols that must be shifted to leave recovery mode - -yaccdevel = False # Set to True if developing yacc. This turns off optimized - # implementations of certain functions. - -resultlimit = 40 # Size limit of results when running in debug mode. - -pickle_protocol = 0 # Protocol to use when writing pickle files - -# String type-checking compatibility -if sys.version_info[0] < 3: - string_types = basestring -else: - string_types = str - -MAXINT = sys.maxsize - -# This object is a stand-in for a logging object created by the -# logging module. PLY will use this by default to create things -# such as the parser.out file. If a user wants more detailed -# information, they can create their own logging object and pass -# it into PLY. - -class PlyLogger(object): - def __init__(self, f): - self.f = f - - def debug(self, msg, *args, **kwargs): - self.f.write((msg % args) + '\n') - - info = debug - - def warning(self, msg, *args, **kwargs): - self.f.write('WARNING: ' + (msg % args) + '\n') - - def error(self, msg, *args, **kwargs): - self.f.write('ERROR: ' + (msg % args) + '\n') - - critical = debug - -# Null logger is used when no output is generated. Does nothing. -class NullLogger(object): - def __getattribute__(self, name): - return self - - def __call__(self, *args, **kwargs): - return self - -# Exception raised for yacc-related errors -class YaccError(Exception): - pass - -# Format the result message that the parser produces when running in debug mode. -def format_result(r): - repr_str = repr(r) - if '\n' in repr_str: - repr_str = repr(repr_str) - if len(repr_str) > resultlimit: - repr_str = repr_str[:resultlimit] + ' ...' - result = '<%s @ 0x%x> (%s)' % (type(r).__name__, id(r), repr_str) - return result - -# Format stack entries when the parser is running in debug mode -def format_stack_entry(r): - repr_str = repr(r) - if '\n' in repr_str: - repr_str = repr(repr_str) - if len(repr_str) < 16: - return repr_str - else: - return '<%s @ 0x%x>' % (type(r).__name__, id(r)) - -# Panic mode error recovery support. This feature is being reworked--much of the -# code here is to offer a deprecation/backwards compatible transition - -_errok = None -_token = None -_restart = None -_warnmsg = '''PLY: Don't use global functions errok(), token(), and restart() in p_error(). -Instead, invoke the methods on the associated parser instance: - - def p_error(p): - ... - # Use parser.errok(), parser.token(), parser.restart() - ... - - parser = yacc.yacc() -''' - -def errok(): - warnings.warn(_warnmsg) - return _errok() - -def restart(): - warnings.warn(_warnmsg) - return _restart() - -def token(): - warnings.warn(_warnmsg) - return _token() - -# Utility function to call the p_error() function with some deprecation hacks -def call_errorfunc(errorfunc, token, parser): - global _errok, _token, _restart - _errok = parser.errok - _token = parser.token - _restart = parser.restart - r = errorfunc(token) - try: - del _errok, _token, _restart - except NameError: - pass - return r - -#----------------------------------------------------------------------------- -# === LR Parsing Engine === -# -# The following classes are used for the LR parser itself. These are not -# used during table construction and are independent of the actual LR -# table generation algorithm -#----------------------------------------------------------------------------- - -# This class is used to hold non-terminal grammar symbols during parsing. -# It normally has the following attributes set: -# .type = Grammar symbol type -# .value = Symbol value -# .lineno = Starting line number -# .endlineno = Ending line number (optional, set automatically) -# .lexpos = Starting lex position -# .endlexpos = Ending lex position (optional, set automatically) - -class YaccSymbol: - def __str__(self): - return self.type - - def __repr__(self): - return str(self) - -# This class is a wrapper around the objects actually passed to each -# grammar rule. Index lookup and assignment actually assign the -# .value attribute of the underlying YaccSymbol object. -# The lineno() method returns the line number of a given -# item (or 0 if not defined). The linespan() method returns -# a tuple of (startline,endline) representing the range of lines -# for a symbol. The lexspan() method returns a tuple (lexpos,endlexpos) -# representing the range of positional information for a symbol. - -class YaccProduction: - def __init__(self, s, stack=None): - self.slice = s - self.stack = stack - self.lexer = None - self.parser = None - - def __getitem__(self, n): - if isinstance(n, slice): - return [s.value for s in self.slice[n]] - elif n >= 0: - return self.slice[n].value - else: - return self.stack[n].value - - def __setitem__(self, n, v): - self.slice[n].value = v - - def __getslice__(self, i, j): - return [s.value for s in self.slice[i:j]] - - def __len__(self): - return len(self.slice) - - def lineno(self, n): - return getattr(self.slice[n], 'lineno', 0) - - def set_lineno(self, n, lineno): - self.slice[n].lineno = lineno - - def linespan(self, n): - startline = getattr(self.slice[n], 'lineno', 0) - endline = getattr(self.slice[n], 'endlineno', startline) - return startline, endline - - def lexpos(self, n): - return getattr(self.slice[n], 'lexpos', 0) - - def lexspan(self, n): - startpos = getattr(self.slice[n], 'lexpos', 0) - endpos = getattr(self.slice[n], 'endlexpos', startpos) - return startpos, endpos - - def error(self): - raise SyntaxError - -# ----------------------------------------------------------------------------- -# == LRParser == -# -# The LR Parsing engine. -# ----------------------------------------------------------------------------- - -class LRParser: - def __init__(self, lrtab, errorf): - self.productions = lrtab.lr_productions - self.action = lrtab.lr_action - self.goto = lrtab.lr_goto - self.errorfunc = errorf - self.set_defaulted_states() - self.errorok = True - - def errok(self): - self.errorok = True - - def restart(self): - del self.statestack[:] - del self.symstack[:] - sym = YaccSymbol() - sym.type = '$end' - self.symstack.append(sym) - self.statestack.append(0) - - # Defaulted state support. - # This method identifies parser states where there is only one possible reduction action. - # For such states, the parser can make a choose to make a rule reduction without consuming - # the next look-ahead token. This delayed invocation of the tokenizer can be useful in - # certain kinds of advanced parsing situations where the lexer and parser interact with - # each other or change states (i.e., manipulation of scope, lexer states, etc.). - # - # See: http://www.gnu.org/software/bison/manual/html_node/Default-Reductions.html#Default-Reductions - def set_defaulted_states(self): - self.defaulted_states = {} - for state, actions in self.action.items(): - rules = list(actions.values()) - if len(rules) == 1 and rules[0] < 0: - self.defaulted_states[state] = rules[0] - - def disable_defaulted_states(self): - self.defaulted_states = {} - - def parse(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): - if debug or yaccdevel: - if isinstance(debug, int): - debug = PlyLogger(sys.stderr) - return self.parsedebug(input, lexer, debug, tracking, tokenfunc) - elif tracking: - return self.parseopt(input, lexer, debug, tracking, tokenfunc) - else: - return self.parseopt_notrack(input, lexer, debug, tracking, tokenfunc) - - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # parsedebug(). - # - # This is the debugging enabled version of parse(). All changes made to the - # parsing engine should be made here. Optimized versions of this function - # are automatically created by the ply/ygen.py script. This script cuts out - # sections enclosed in markers such as this: - # - # #--! DEBUG - # statements - # #--! DEBUG - # - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - def parsedebug(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): - #--! parsedebug-start - lookahead = None # Current lookahead symbol - lookaheadstack = [] # Stack of lookahead symbols - actions = self.action # Local reference to action table (to avoid lookup on self.) - goto = self.goto # Local reference to goto table (to avoid lookup on self.) - prod = self.productions # Local reference to production list (to avoid lookup on self.) - defaulted_states = self.defaulted_states # Local reference to defaulted states - pslice = YaccProduction(None) # Production object passed to grammar rules - errorcount = 0 # Used during error recovery - - #--! DEBUG - debug.info('PLY: PARSE DEBUG START') - #--! DEBUG - - # If no lexer was given, we will try to use the lex module - if not lexer: - from . import lex - lexer = lex.lexer - - # Set up the lexer and parser objects on pslice - pslice.lexer = lexer - pslice.parser = self - - # If input was supplied, pass to lexer - if input is not None: - lexer.input(input) - - if tokenfunc is None: - # Tokenize function - get_token = lexer.token - else: - get_token = tokenfunc - - # Set the parser() token method (sometimes used in error recovery) - self.token = get_token - - # Set up the state and symbol stacks - - statestack = [] # Stack of parsing states - self.statestack = statestack - symstack = [] # Stack of grammar symbols - self.symstack = symstack - - pslice.stack = symstack # Put in the production - errtoken = None # Err token - - # The start state is assumed to be (0,$end) - - statestack.append(0) - sym = YaccSymbol() - sym.type = '$end' - symstack.append(sym) - state = 0 - while True: - # Get the next symbol on the input. If a lookahead symbol - # is already set, we just use that. Otherwise, we'll pull - # the next token off of the lookaheadstack or from the lexer - - #--! DEBUG - debug.debug('') - debug.debug('State : %s', state) - #--! DEBUG - - if state not in defaulted_states: - if not lookahead: - if not lookaheadstack: - lookahead = get_token() # Get the next token - else: - lookahead = lookaheadstack.pop() - if not lookahead: - lookahead = YaccSymbol() - lookahead.type = '$end' - - # Check the action table - ltype = lookahead.type - t = actions[state].get(ltype) - else: - t = defaulted_states[state] - #--! DEBUG - debug.debug('Defaulted state %s: Reduce using %d', state, -t) - #--! DEBUG - - #--! DEBUG - debug.debug('Stack : %s', - ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip()) - #--! DEBUG - - if t is not None: - if t > 0: - # shift a symbol on the stack - statestack.append(t) - state = t - - #--! DEBUG - debug.debug('Action : Shift and goto state %s', t) - #--! DEBUG - - symstack.append(lookahead) - lookahead = None - - # Decrease error count on successful shift - if errorcount: - errorcount -= 1 - continue - - if t < 0: - # reduce a symbol on the stack, emit a production - p = prod[-t] - pname = p.name - plen = p.len - - # Get production function - sym = YaccSymbol() - sym.type = pname # Production name - sym.value = None - - #--! DEBUG - if plen: - debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, - '['+','.join([format_stack_entry(_v.value) for _v in symstack[-plen:]])+']', - goto[statestack[-1-plen]][pname]) - else: - debug.info('Action : Reduce rule [%s] with %s and goto state %d', p.str, [], - goto[statestack[-1]][pname]) - - #--! DEBUG - - if plen: - targ = symstack[-plen-1:] - targ[0] = sym - - #--! TRACKING - if tracking: - t1 = targ[1] - sym.lineno = t1.lineno - sym.lexpos = t1.lexpos - t1 = targ[-1] - sym.endlineno = getattr(t1, 'endlineno', t1.lineno) - sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos) - #--! TRACKING - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # below as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - del symstack[-plen:] - del statestack[-plen:] - p.callable(pslice) - #--! DEBUG - debug.info('Result : %s', format_result(pslice[0])) - #--! DEBUG - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - else: - - #--! TRACKING - if tracking: - sym.lineno = lexer.lineno - sym.lexpos = lexer.lexpos - #--! TRACKING - - targ = [sym] - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # above as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - p.callable(pslice) - #--! DEBUG - debug.info('Result : %s', format_result(pslice[0])) - #--! DEBUG - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - if t == 0: - n = symstack[-1] - result = getattr(n, 'value', None) - #--! DEBUG - debug.info('Done : Returning %s', format_result(result)) - debug.info('PLY: PARSE DEBUG END') - #--! DEBUG - return result - - if t is None: - - #--! DEBUG - debug.error('Error : %s', - ('%s . %s' % (' '.join([xx.type for xx in symstack][1:]), str(lookahead))).lstrip()) - #--! DEBUG - - # We have some kind of parsing error here. To handle - # this, we are going to push the current token onto - # the tokenstack and replace it with an 'error' token. - # If there are any synchronization rules, they may - # catch it. - # - # In addition to pushing the error token, we call call - # the user defined p_error() function if this is the - # first syntax error. This function is only called if - # errorcount == 0. - if errorcount == 0 or self.errorok: - errorcount = error_count - self.errorok = False - errtoken = lookahead - if errtoken.type == '$end': - errtoken = None # End of file! - if self.errorfunc: - if errtoken and not hasattr(errtoken, 'lexer'): - errtoken.lexer = lexer - tok = call_errorfunc(self.errorfunc, errtoken, self) - if self.errorok: - # User must have done some kind of panic - # mode recovery on their own. The - # returned token is the next lookahead - lookahead = tok - errtoken = None - continue - else: - if errtoken: - if hasattr(errtoken, 'lineno'): - lineno = lookahead.lineno - else: - lineno = 0 - if lineno: - sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) - else: - sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) - else: - sys.stderr.write('yacc: Parse error in input. EOF\n') - return - - else: - errorcount = error_count - - # case 1: the statestack only has 1 entry on it. If we're in this state, the - # entire parse has been rolled back and we're completely hosed. The token is - # discarded and we just keep going. - - if len(statestack) <= 1 and lookahead.type != '$end': - lookahead = None - errtoken = None - state = 0 - # Nuke the pushback stack - del lookaheadstack[:] - continue - - # case 2: the statestack has a couple of entries on it, but we're - # at the end of the file. nuke the top entry and generate an error token - - # Start nuking entries on the stack - if lookahead.type == '$end': - # Whoa. We're really hosed here. Bail out - return - - if lookahead.type != 'error': - sym = symstack[-1] - if sym.type == 'error': - # Hmmm. Error is on top of stack, we'll just nuke input - # symbol and continue - #--! TRACKING - if tracking: - sym.endlineno = getattr(lookahead, 'lineno', sym.lineno) - sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos) - #--! TRACKING - lookahead = None - continue - - # Create the error symbol for the first time and make it the new lookahead symbol - t = YaccSymbol() - t.type = 'error' - - if hasattr(lookahead, 'lineno'): - t.lineno = t.endlineno = lookahead.lineno - if hasattr(lookahead, 'lexpos'): - t.lexpos = t.endlexpos = lookahead.lexpos - t.value = lookahead - lookaheadstack.append(lookahead) - lookahead = t - else: - sym = symstack.pop() - #--! TRACKING - if tracking: - lookahead.lineno = sym.lineno - lookahead.lexpos = sym.lexpos - #--! TRACKING - statestack.pop() - state = statestack[-1] - - continue - - # Call an error function here - raise RuntimeError('yacc: internal parser error!!!\n') - - #--! parsedebug-end - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # parseopt(). - # - # Optimized version of parse() method. DO NOT EDIT THIS CODE DIRECTLY! - # This code is automatically generated by the ply/ygen.py script. Make - # changes to the parsedebug() method instead. - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - def parseopt(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): - #--! parseopt-start - lookahead = None # Current lookahead symbol - lookaheadstack = [] # Stack of lookahead symbols - actions = self.action # Local reference to action table (to avoid lookup on self.) - goto = self.goto # Local reference to goto table (to avoid lookup on self.) - prod = self.productions # Local reference to production list (to avoid lookup on self.) - defaulted_states = self.defaulted_states # Local reference to defaulted states - pslice = YaccProduction(None) # Production object passed to grammar rules - errorcount = 0 # Used during error recovery - - - # If no lexer was given, we will try to use the lex module - if not lexer: - from . import lex - lexer = lex.lexer - - # Set up the lexer and parser objects on pslice - pslice.lexer = lexer - pslice.parser = self - - # If input was supplied, pass to lexer - if input is not None: - lexer.input(input) - - if tokenfunc is None: - # Tokenize function - get_token = lexer.token - else: - get_token = tokenfunc - - # Set the parser() token method (sometimes used in error recovery) - self.token = get_token - - # Set up the state and symbol stacks - - statestack = [] # Stack of parsing states - self.statestack = statestack - symstack = [] # Stack of grammar symbols - self.symstack = symstack - - pslice.stack = symstack # Put in the production - errtoken = None # Err token - - # The start state is assumed to be (0,$end) - - statestack.append(0) - sym = YaccSymbol() - sym.type = '$end' - symstack.append(sym) - state = 0 - while True: - # Get the next symbol on the input. If a lookahead symbol - # is already set, we just use that. Otherwise, we'll pull - # the next token off of the lookaheadstack or from the lexer - - - if state not in defaulted_states: - if not lookahead: - if not lookaheadstack: - lookahead = get_token() # Get the next token - else: - lookahead = lookaheadstack.pop() - if not lookahead: - lookahead = YaccSymbol() - lookahead.type = '$end' - - # Check the action table - ltype = lookahead.type - t = actions[state].get(ltype) - else: - t = defaulted_states[state] - - - if t is not None: - if t > 0: - # shift a symbol on the stack - statestack.append(t) - state = t - - - symstack.append(lookahead) - lookahead = None - - # Decrease error count on successful shift - if errorcount: - errorcount -= 1 - continue - - if t < 0: - # reduce a symbol on the stack, emit a production - p = prod[-t] - pname = p.name - plen = p.len - - # Get production function - sym = YaccSymbol() - sym.type = pname # Production name - sym.value = None - - - if plen: - targ = symstack[-plen-1:] - targ[0] = sym - - #--! TRACKING - if tracking: - t1 = targ[1] - sym.lineno = t1.lineno - sym.lexpos = t1.lexpos - t1 = targ[-1] - sym.endlineno = getattr(t1, 'endlineno', t1.lineno) - sym.endlexpos = getattr(t1, 'endlexpos', t1.lexpos) - #--! TRACKING - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # below as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - del symstack[-plen:] - del statestack[-plen:] - p.callable(pslice) - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - else: - - #--! TRACKING - if tracking: - sym.lineno = lexer.lineno - sym.lexpos = lexer.lexpos - #--! TRACKING - - targ = [sym] - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # above as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - p.callable(pslice) - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - if t == 0: - n = symstack[-1] - result = getattr(n, 'value', None) - return result - - if t is None: - - - # We have some kind of parsing error here. To handle - # this, we are going to push the current token onto - # the tokenstack and replace it with an 'error' token. - # If there are any synchronization rules, they may - # catch it. - # - # In addition to pushing the error token, we call call - # the user defined p_error() function if this is the - # first syntax error. This function is only called if - # errorcount == 0. - if errorcount == 0 or self.errorok: - errorcount = error_count - self.errorok = False - errtoken = lookahead - if errtoken.type == '$end': - errtoken = None # End of file! - if self.errorfunc: - if errtoken and not hasattr(errtoken, 'lexer'): - errtoken.lexer = lexer - tok = call_errorfunc(self.errorfunc, errtoken, self) - if self.errorok: - # User must have done some kind of panic - # mode recovery on their own. The - # returned token is the next lookahead - lookahead = tok - errtoken = None - continue - else: - if errtoken: - if hasattr(errtoken, 'lineno'): - lineno = lookahead.lineno - else: - lineno = 0 - if lineno: - sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) - else: - sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) - else: - sys.stderr.write('yacc: Parse error in input. EOF\n') - return - - else: - errorcount = error_count - - # case 1: the statestack only has 1 entry on it. If we're in this state, the - # entire parse has been rolled back and we're completely hosed. The token is - # discarded and we just keep going. - - if len(statestack) <= 1 and lookahead.type != '$end': - lookahead = None - errtoken = None - state = 0 - # Nuke the pushback stack - del lookaheadstack[:] - continue - - # case 2: the statestack has a couple of entries on it, but we're - # at the end of the file. nuke the top entry and generate an error token - - # Start nuking entries on the stack - if lookahead.type == '$end': - # Whoa. We're really hosed here. Bail out - return - - if lookahead.type != 'error': - sym = symstack[-1] - if sym.type == 'error': - # Hmmm. Error is on top of stack, we'll just nuke input - # symbol and continue - #--! TRACKING - if tracking: - sym.endlineno = getattr(lookahead, 'lineno', sym.lineno) - sym.endlexpos = getattr(lookahead, 'lexpos', sym.lexpos) - #--! TRACKING - lookahead = None - continue - - # Create the error symbol for the first time and make it the new lookahead symbol - t = YaccSymbol() - t.type = 'error' - - if hasattr(lookahead, 'lineno'): - t.lineno = t.endlineno = lookahead.lineno - if hasattr(lookahead, 'lexpos'): - t.lexpos = t.endlexpos = lookahead.lexpos - t.value = lookahead - lookaheadstack.append(lookahead) - lookahead = t - else: - sym = symstack.pop() - #--! TRACKING - if tracking: - lookahead.lineno = sym.lineno - lookahead.lexpos = sym.lexpos - #--! TRACKING - statestack.pop() - state = statestack[-1] - - continue - - # Call an error function here - raise RuntimeError('yacc: internal parser error!!!\n') - - #--! parseopt-end - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # parseopt_notrack(). - # - # Optimized version of parseopt() with line number tracking removed. - # DO NOT EDIT THIS CODE DIRECTLY. This code is automatically generated - # by the ply/ygen.py script. Make changes to the parsedebug() method instead. - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - def parseopt_notrack(self, input=None, lexer=None, debug=False, tracking=False, tokenfunc=None): - #--! parseopt-notrack-start - lookahead = None # Current lookahead symbol - lookaheadstack = [] # Stack of lookahead symbols - actions = self.action # Local reference to action table (to avoid lookup on self.) - goto = self.goto # Local reference to goto table (to avoid lookup on self.) - prod = self.productions # Local reference to production list (to avoid lookup on self.) - defaulted_states = self.defaulted_states # Local reference to defaulted states - pslice = YaccProduction(None) # Production object passed to grammar rules - errorcount = 0 # Used during error recovery - - - # If no lexer was given, we will try to use the lex module - if not lexer: - from . import lex - lexer = lex.lexer - - # Set up the lexer and parser objects on pslice - pslice.lexer = lexer - pslice.parser = self - - # If input was supplied, pass to lexer - if input is not None: - lexer.input(input) - - if tokenfunc is None: - # Tokenize function - get_token = lexer.token - else: - get_token = tokenfunc - - # Set the parser() token method (sometimes used in error recovery) - self.token = get_token - - # Set up the state and symbol stacks - - statestack = [] # Stack of parsing states - self.statestack = statestack - symstack = [] # Stack of grammar symbols - self.symstack = symstack - - pslice.stack = symstack # Put in the production - errtoken = None # Err token - - # The start state is assumed to be (0,$end) - - statestack.append(0) - sym = YaccSymbol() - sym.type = '$end' - symstack.append(sym) - state = 0 - while True: - # Get the next symbol on the input. If a lookahead symbol - # is already set, we just use that. Otherwise, we'll pull - # the next token off of the lookaheadstack or from the lexer - - - if state not in defaulted_states: - if not lookahead: - if not lookaheadstack: - lookahead = get_token() # Get the next token - else: - lookahead = lookaheadstack.pop() - if not lookahead: - lookahead = YaccSymbol() - lookahead.type = '$end' - - # Check the action table - ltype = lookahead.type - t = actions[state].get(ltype) - else: - t = defaulted_states[state] - - - if t is not None: - if t > 0: - # shift a symbol on the stack - statestack.append(t) - state = t - - - symstack.append(lookahead) - lookahead = None - - # Decrease error count on successful shift - if errorcount: - errorcount -= 1 - continue - - if t < 0: - # reduce a symbol on the stack, emit a production - p = prod[-t] - pname = p.name - plen = p.len - - # Get production function - sym = YaccSymbol() - sym.type = pname # Production name - sym.value = None - - - if plen: - targ = symstack[-plen-1:] - targ[0] = sym - - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # below as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - del symstack[-plen:] - del statestack[-plen:] - p.callable(pslice) - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - else: - - - targ = [sym] - - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - # The code enclosed in this section is duplicated - # above as a performance optimization. Make sure - # changes get made in both locations. - - pslice.slice = targ - - try: - # Call the grammar rule with our special slice object - p.callable(pslice) - symstack.append(sym) - state = goto[statestack[-1]][pname] - statestack.append(state) - except SyntaxError: - # If an error was set. Enter error recovery state - lookaheadstack.append(lookahead) - symstack.pop() - statestack.pop() - state = statestack[-1] - sym.type = 'error' - lookahead = sym - errorcount = error_count - self.errorok = False - continue - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - if t == 0: - n = symstack[-1] - result = getattr(n, 'value', None) - return result - - if t is None: - - - # We have some kind of parsing error here. To handle - # this, we are going to push the current token onto - # the tokenstack and replace it with an 'error' token. - # If there are any synchronization rules, they may - # catch it. - # - # In addition to pushing the error token, we call call - # the user defined p_error() function if this is the - # first syntax error. This function is only called if - # errorcount == 0. - if errorcount == 0 or self.errorok: - errorcount = error_count - self.errorok = False - errtoken = lookahead - if errtoken.type == '$end': - errtoken = None # End of file! - if self.errorfunc: - if errtoken and not hasattr(errtoken, 'lexer'): - errtoken.lexer = lexer - tok = call_errorfunc(self.errorfunc, errtoken, self) - if self.errorok: - # User must have done some kind of panic - # mode recovery on their own. The - # returned token is the next lookahead - lookahead = tok - errtoken = None - continue - else: - if errtoken: - if hasattr(errtoken, 'lineno'): - lineno = lookahead.lineno - else: - lineno = 0 - if lineno: - sys.stderr.write('yacc: Syntax error at line %d, token=%s\n' % (lineno, errtoken.type)) - else: - sys.stderr.write('yacc: Syntax error, token=%s' % errtoken.type) - else: - sys.stderr.write('yacc: Parse error in input. EOF\n') - return - - else: - errorcount = error_count - - # case 1: the statestack only has 1 entry on it. If we're in this state, the - # entire parse has been rolled back and we're completely hosed. The token is - # discarded and we just keep going. - - if len(statestack) <= 1 and lookahead.type != '$end': - lookahead = None - errtoken = None - state = 0 - # Nuke the pushback stack - del lookaheadstack[:] - continue - - # case 2: the statestack has a couple of entries on it, but we're - # at the end of the file. nuke the top entry and generate an error token - - # Start nuking entries on the stack - if lookahead.type == '$end': - # Whoa. We're really hosed here. Bail out - return - - if lookahead.type != 'error': - sym = symstack[-1] - if sym.type == 'error': - # Hmmm. Error is on top of stack, we'll just nuke input - # symbol and continue - lookahead = None - continue - - # Create the error symbol for the first time and make it the new lookahead symbol - t = YaccSymbol() - t.type = 'error' - - if hasattr(lookahead, 'lineno'): - t.lineno = t.endlineno = lookahead.lineno - if hasattr(lookahead, 'lexpos'): - t.lexpos = t.endlexpos = lookahead.lexpos - t.value = lookahead - lookaheadstack.append(lookahead) - lookahead = t - else: - sym = symstack.pop() - statestack.pop() - state = statestack[-1] - - continue - - # Call an error function here - raise RuntimeError('yacc: internal parser error!!!\n') - - #--! parseopt-notrack-end - -# ----------------------------------------------------------------------------- -# === Grammar Representation === -# -# The following functions, classes, and variables are used to represent and -# manipulate the rules that make up a grammar. -# ----------------------------------------------------------------------------- - -# regex matching identifiers -_is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$') - -# ----------------------------------------------------------------------------- -# class Production: -# -# This class stores the raw information about a single production or grammar rule. -# A grammar rule refers to a specification such as this: -# -# expr : expr PLUS term -# -# Here are the basic attributes defined on all productions -# -# name - Name of the production. For example 'expr' -# prod - A list of symbols on the right side ['expr','PLUS','term'] -# prec - Production precedence level -# number - Production number. -# func - Function that executes on reduce -# file - File where production function is defined -# lineno - Line number where production function is defined -# -# The following attributes are defined or optional. -# -# len - Length of the production (number of symbols on right hand side) -# usyms - Set of unique symbols found in the production -# ----------------------------------------------------------------------------- - -class Production(object): - reduced = 0 - def __init__(self, number, name, prod, precedence=('right', 0), func=None, file='', line=0): - self.name = name - self.prod = tuple(prod) - self.number = number - self.func = func - self.callable = None - self.file = file - self.line = line - self.prec = precedence - - # Internal settings used during table construction - - self.len = len(self.prod) # Length of the production - - # Create a list of unique production symbols used in the production - self.usyms = [] - for s in self.prod: - if s not in self.usyms: - self.usyms.append(s) - - # List of all LR items for the production - self.lr_items = [] - self.lr_next = None - - # Create a string representation - if self.prod: - self.str = '%s -> %s' % (self.name, ' '.join(self.prod)) - else: - self.str = '%s -> ' % self.name - - def __str__(self): - return self.str - - def __repr__(self): - return 'Production(' + str(self) + ')' - - def __len__(self): - return len(self.prod) - - def __nonzero__(self): - return 1 - - def __getitem__(self, index): - return self.prod[index] - - # Return the nth lr_item from the production (or None if at the end) - def lr_item(self, n): - if n > len(self.prod): - return None - p = LRItem(self, n) - # Precompute the list of productions immediately following. - try: - p.lr_after = Prodnames[p.prod[n+1]] - except (IndexError, KeyError): - p.lr_after = [] - try: - p.lr_before = p.prod[n-1] - except IndexError: - p.lr_before = None - return p - - # Bind the production function name to a callable - def bind(self, pdict): - if self.func: - self.callable = pdict[self.func] - -# This class serves as a minimal standin for Production objects when -# reading table data from files. It only contains information -# actually used by the LR parsing engine, plus some additional -# debugging information. -class MiniProduction(object): - def __init__(self, str, name, len, func, file, line): - self.name = name - self.len = len - self.func = func - self.callable = None - self.file = file - self.line = line - self.str = str - - def __str__(self): - return self.str - - def __repr__(self): - return 'MiniProduction(%s)' % self.str - - # Bind the production function name to a callable - def bind(self, pdict): - if self.func: - self.callable = pdict[self.func] - - -# ----------------------------------------------------------------------------- -# class LRItem -# -# This class represents a specific stage of parsing a production rule. For -# example: -# -# expr : expr . PLUS term -# -# In the above, the "." represents the current location of the parse. Here -# basic attributes: -# -# name - Name of the production. For example 'expr' -# prod - A list of symbols on the right side ['expr','.', 'PLUS','term'] -# number - Production number. -# -# lr_next Next LR item. Example, if we are ' expr -> expr . PLUS term' -# then lr_next refers to 'expr -> expr PLUS . term' -# lr_index - LR item index (location of the ".") in the prod list. -# lookaheads - LALR lookahead symbols for this item -# len - Length of the production (number of symbols on right hand side) -# lr_after - List of all productions that immediately follow -# lr_before - Grammar symbol immediately before -# ----------------------------------------------------------------------------- - -class LRItem(object): - def __init__(self, p, n): - self.name = p.name - self.prod = list(p.prod) - self.number = p.number - self.lr_index = n - self.lookaheads = {} - self.prod.insert(n, '.') - self.prod = tuple(self.prod) - self.len = len(self.prod) - self.usyms = p.usyms - - def __str__(self): - if self.prod: - s = '%s -> %s' % (self.name, ' '.join(self.prod)) - else: - s = '%s -> ' % self.name - return s - - def __repr__(self): - return 'LRItem(' + str(self) + ')' - -# ----------------------------------------------------------------------------- -# rightmost_terminal() -# -# Return the rightmost terminal from a list of symbols. Used in add_production() -# ----------------------------------------------------------------------------- -def rightmost_terminal(symbols, terminals): - i = len(symbols) - 1 - while i >= 0: - if symbols[i] in terminals: - return symbols[i] - i -= 1 - return None - -# ----------------------------------------------------------------------------- -# === GRAMMAR CLASS === -# -# The following class represents the contents of the specified grammar along -# with various computed properties such as first sets, follow sets, LR items, etc. -# This data is used for critical parts of the table generation process later. -# ----------------------------------------------------------------------------- - -class GrammarError(YaccError): - pass - -class Grammar(object): - def __init__(self, terminals): - self.Productions = [None] # A list of all of the productions. The first - # entry is always reserved for the purpose of - # building an augmented grammar - - self.Prodnames = {} # A dictionary mapping the names of nonterminals to a list of all - # productions of that nonterminal. - - self.Prodmap = {} # A dictionary that is only used to detect duplicate - # productions. - - self.Terminals = {} # A dictionary mapping the names of terminal symbols to a - # list of the rules where they are used. - - for term in terminals: - self.Terminals[term] = [] - - self.Terminals['error'] = [] - - self.Nonterminals = {} # A dictionary mapping names of nonterminals to a list - # of rule numbers where they are used. - - self.First = {} # A dictionary of precomputed FIRST(x) symbols - - self.Follow = {} # A dictionary of precomputed FOLLOW(x) symbols - - self.Precedence = {} # Precedence rules for each terminal. Contains tuples of the - # form ('right',level) or ('nonassoc', level) or ('left',level) - - self.UsedPrecedence = set() # Precedence rules that were actually used by the grammer. - # This is only used to provide error checking and to generate - # a warning about unused precedence rules. - - self.Start = None # Starting symbol for the grammar - - - def __len__(self): - return len(self.Productions) - - def __getitem__(self, index): - return self.Productions[index] - - # ----------------------------------------------------------------------------- - # set_precedence() - # - # Sets the precedence for a given terminal. assoc is the associativity such as - # 'left','right', or 'nonassoc'. level is a numeric level. - # - # ----------------------------------------------------------------------------- - - def set_precedence(self, term, assoc, level): - assert self.Productions == [None], 'Must call set_precedence() before add_production()' - if term in self.Precedence: - raise GrammarError('Precedence already specified for terminal %r' % term) - if assoc not in ['left', 'right', 'nonassoc']: - raise GrammarError("Associativity must be one of 'left','right', or 'nonassoc'") - self.Precedence[term] = (assoc, level) - - # ----------------------------------------------------------------------------- - # add_production() - # - # Given an action function, this function assembles a production rule and - # computes its precedence level. - # - # The production rule is supplied as a list of symbols. For example, - # a rule such as 'expr : expr PLUS term' has a production name of 'expr' and - # symbols ['expr','PLUS','term']. - # - # Precedence is determined by the precedence of the right-most non-terminal - # or the precedence of a terminal specified by %prec. - # - # A variety of error checks are performed to make sure production symbols - # are valid and that %prec is used correctly. - # ----------------------------------------------------------------------------- - - def add_production(self, prodname, syms, func=None, file='', line=0): - - if prodname in self.Terminals: - raise GrammarError('%s:%d: Illegal rule name %r. Already defined as a token' % (file, line, prodname)) - if prodname == 'error': - raise GrammarError('%s:%d: Illegal rule name %r. error is a reserved word' % (file, line, prodname)) - if not _is_identifier.match(prodname): - raise GrammarError('%s:%d: Illegal rule name %r' % (file, line, prodname)) - - # Look for literal tokens - for n, s in enumerate(syms): - if s[0] in "'\"": - try: - c = eval(s) - if (len(c) > 1): - raise GrammarError('%s:%d: Literal token %s in rule %r may only be a single character' % - (file, line, s, prodname)) - if c not in self.Terminals: - self.Terminals[c] = [] - syms[n] = c - continue - except SyntaxError: - pass - if not _is_identifier.match(s) and s != '%prec': - raise GrammarError('%s:%d: Illegal name %r in rule %r' % (file, line, s, prodname)) - - # Determine the precedence level - if '%prec' in syms: - if syms[-1] == '%prec': - raise GrammarError('%s:%d: Syntax error. Nothing follows %%prec' % (file, line)) - if syms[-2] != '%prec': - raise GrammarError('%s:%d: Syntax error. %%prec can only appear at the end of a grammar rule' % - (file, line)) - precname = syms[-1] - prodprec = self.Precedence.get(precname) - if not prodprec: - raise GrammarError('%s:%d: Nothing known about the precedence of %r' % (file, line, precname)) - else: - self.UsedPrecedence.add(precname) - del syms[-2:] # Drop %prec from the rule - else: - # If no %prec, precedence is determined by the rightmost terminal symbol - precname = rightmost_terminal(syms, self.Terminals) - prodprec = self.Precedence.get(precname, ('right', 0)) - - # See if the rule is already in the rulemap - map = '%s -> %s' % (prodname, syms) - if map in self.Prodmap: - m = self.Prodmap[map] - raise GrammarError('%s:%d: Duplicate rule %s. ' % (file, line, m) + - 'Previous definition at %s:%d' % (m.file, m.line)) - - # From this point on, everything is valid. Create a new Production instance - pnumber = len(self.Productions) - if prodname not in self.Nonterminals: - self.Nonterminals[prodname] = [] - - # Add the production number to Terminals and Nonterminals - for t in syms: - if t in self.Terminals: - self.Terminals[t].append(pnumber) - else: - if t not in self.Nonterminals: - self.Nonterminals[t] = [] - self.Nonterminals[t].append(pnumber) - - # Create a production and add it to the list of productions - p = Production(pnumber, prodname, syms, prodprec, func, file, line) - self.Productions.append(p) - self.Prodmap[map] = p - - # Add to the global productions list - try: - self.Prodnames[prodname].append(p) - except KeyError: - self.Prodnames[prodname] = [p] - - # ----------------------------------------------------------------------------- - # set_start() - # - # Sets the starting symbol and creates the augmented grammar. Production - # rule 0 is S' -> start where start is the start symbol. - # ----------------------------------------------------------------------------- - - def set_start(self, start=None): - if not start: - start = self.Productions[1].name - if start not in self.Nonterminals: - raise GrammarError('start symbol %s undefined' % start) - self.Productions[0] = Production(0, "S'", [start]) - self.Nonterminals[start].append(0) - self.Start = start - - # ----------------------------------------------------------------------------- - # find_unreachable() - # - # Find all of the nonterminal symbols that can't be reached from the starting - # symbol. Returns a list of nonterminals that can't be reached. - # ----------------------------------------------------------------------------- - - def find_unreachable(self): - - # Mark all symbols that are reachable from a symbol s - def mark_reachable_from(s): - if s in reachable: - return - reachable.add(s) - for p in self.Prodnames.get(s, []): - for r in p.prod: - mark_reachable_from(r) - - reachable = set() - mark_reachable_from(self.Productions[0].prod[0]) - return [s for s in self.Nonterminals if s not in reachable] - - # ----------------------------------------------------------------------------- - # infinite_cycles() - # - # This function looks at the various parsing rules and tries to detect - # infinite recursion cycles (grammar rules where there is no possible way - # to derive a string of only terminals). - # ----------------------------------------------------------------------------- - - def infinite_cycles(self): - terminates = {} - - # Terminals: - for t in self.Terminals: - terminates[t] = True - - terminates['$end'] = True - - # Nonterminals: - - # Initialize to false: - for n in self.Nonterminals: - terminates[n] = False - - # Then propagate termination until no change: - while True: - some_change = False - for (n, pl) in self.Prodnames.items(): - # Nonterminal n terminates iff any of its productions terminates. - for p in pl: - # Production p terminates iff all of its rhs symbols terminate. - for s in p.prod: - if not terminates[s]: - # The symbol s does not terminate, - # so production p does not terminate. - p_terminates = False - break - else: - # didn't break from the loop, - # so every symbol s terminates - # so production p terminates. - p_terminates = True - - if p_terminates: - # symbol n terminates! - if not terminates[n]: - terminates[n] = True - some_change = True - # Don't need to consider any more productions for this n. - break - - if not some_change: - break - - infinite = [] - for (s, term) in terminates.items(): - if not term: - if s not in self.Prodnames and s not in self.Terminals and s != 'error': - # s is used-but-not-defined, and we've already warned of that, - # so it would be overkill to say that it's also non-terminating. - pass - else: - infinite.append(s) - - return infinite - - # ----------------------------------------------------------------------------- - # undefined_symbols() - # - # Find all symbols that were used the grammar, but not defined as tokens or - # grammar rules. Returns a list of tuples (sym, prod) where sym in the symbol - # and prod is the production where the symbol was used. - # ----------------------------------------------------------------------------- - def undefined_symbols(self): - result = [] - for p in self.Productions: - if not p: - continue - - for s in p.prod: - if s not in self.Prodnames and s not in self.Terminals and s != 'error': - result.append((s, p)) - return result - - # ----------------------------------------------------------------------------- - # unused_terminals() - # - # Find all terminals that were defined, but not used by the grammar. Returns - # a list of all symbols. - # ----------------------------------------------------------------------------- - def unused_terminals(self): - unused_tok = [] - for s, v in self.Terminals.items(): - if s != 'error' and not v: - unused_tok.append(s) - - return unused_tok - - # ------------------------------------------------------------------------------ - # unused_rules() - # - # Find all grammar rules that were defined, but not used (maybe not reachable) - # Returns a list of productions. - # ------------------------------------------------------------------------------ - - def unused_rules(self): - unused_prod = [] - for s, v in self.Nonterminals.items(): - if not v: - p = self.Prodnames[s][0] - unused_prod.append(p) - return unused_prod - - # ----------------------------------------------------------------------------- - # unused_precedence() - # - # Returns a list of tuples (term,precedence) corresponding to precedence - # rules that were never used by the grammar. term is the name of the terminal - # on which precedence was applied and precedence is a string such as 'left' or - # 'right' corresponding to the type of precedence. - # ----------------------------------------------------------------------------- - - def unused_precedence(self): - unused = [] - for termname in self.Precedence: - if not (termname in self.Terminals or termname in self.UsedPrecedence): - unused.append((termname, self.Precedence[termname][0])) - - return unused - - # ------------------------------------------------------------------------- - # _first() - # - # Compute the value of FIRST1(beta) where beta is a tuple of symbols. - # - # During execution of compute_first1, the result may be incomplete. - # Afterward (e.g., when called from compute_follow()), it will be complete. - # ------------------------------------------------------------------------- - def _first(self, beta): - - # We are computing First(x1,x2,x3,...,xn) - result = [] - for x in beta: - x_produces_empty = False - - # Add all the non- symbols of First[x] to the result. - for f in self.First[x]: - if f == '': - x_produces_empty = True - else: - if f not in result: - result.append(f) - - if x_produces_empty: - # We have to consider the next x in beta, - # i.e. stay in the loop. - pass - else: - # We don't have to consider any further symbols in beta. - break - else: - # There was no 'break' from the loop, - # so x_produces_empty was true for all x in beta, - # so beta produces empty as well. - result.append('') - - return result - - # ------------------------------------------------------------------------- - # compute_first() - # - # Compute the value of FIRST1(X) for all symbols - # ------------------------------------------------------------------------- - def compute_first(self): - if self.First: - return self.First - - # Terminals: - for t in self.Terminals: - self.First[t] = [t] - - self.First['$end'] = ['$end'] - - # Nonterminals: - - # Initialize to the empty set: - for n in self.Nonterminals: - self.First[n] = [] - - # Then propagate symbols until no change: - while True: - some_change = False - for n in self.Nonterminals: - for p in self.Prodnames[n]: - for f in self._first(p.prod): - if f not in self.First[n]: - self.First[n].append(f) - some_change = True - if not some_change: - break - - return self.First - - # --------------------------------------------------------------------- - # compute_follow() - # - # Computes all of the follow sets for every non-terminal symbol. The - # follow set is the set of all symbols that might follow a given - # non-terminal. See the Dragon book, 2nd Ed. p. 189. - # --------------------------------------------------------------------- - def compute_follow(self, start=None): - # If already computed, return the result - if self.Follow: - return self.Follow - - # If first sets not computed yet, do that first. - if not self.First: - self.compute_first() - - # Add '$end' to the follow list of the start symbol - for k in self.Nonterminals: - self.Follow[k] = [] - - if not start: - start = self.Productions[1].name - - self.Follow[start] = ['$end'] - - while True: - didadd = False - for p in self.Productions[1:]: - # Here is the production set - for i, B in enumerate(p.prod): - if B in self.Nonterminals: - # Okay. We got a non-terminal in a production - fst = self._first(p.prod[i+1:]) - hasempty = False - for f in fst: - if f != '' and f not in self.Follow[B]: - self.Follow[B].append(f) - didadd = True - if f == '': - hasempty = True - if hasempty or i == (len(p.prod)-1): - # Add elements of follow(a) to follow(b) - for f in self.Follow[p.name]: - if f not in self.Follow[B]: - self.Follow[B].append(f) - didadd = True - if not didadd: - break - return self.Follow - - - # ----------------------------------------------------------------------------- - # build_lritems() - # - # This function walks the list of productions and builds a complete set of the - # LR items. The LR items are stored in two ways: First, they are uniquely - # numbered and placed in the list _lritems. Second, a linked list of LR items - # is built for each production. For example: - # - # E -> E PLUS E - # - # Creates the list - # - # [E -> . E PLUS E, E -> E . PLUS E, E -> E PLUS . E, E -> E PLUS E . ] - # ----------------------------------------------------------------------------- - - def build_lritems(self): - for p in self.Productions: - lastlri = p - i = 0 - lr_items = [] - while True: - if i > len(p): - lri = None - else: - lri = LRItem(p, i) - # Precompute the list of productions immediately following - try: - lri.lr_after = self.Prodnames[lri.prod[i+1]] - except (IndexError, KeyError): - lri.lr_after = [] - try: - lri.lr_before = lri.prod[i-1] - except IndexError: - lri.lr_before = None - - lastlri.lr_next = lri - if not lri: - break - lr_items.append(lri) - lastlri = lri - i += 1 - p.lr_items = lr_items - -# ----------------------------------------------------------------------------- -# == Class LRTable == -# -# This basic class represents a basic table of LR parsing information. -# Methods for generating the tables are not defined here. They are defined -# in the derived class LRGeneratedTable. -# ----------------------------------------------------------------------------- - -class VersionError(YaccError): - pass - -class LRTable(object): - def __init__(self): - self.lr_action = None - self.lr_goto = None - self.lr_productions = None - self.lr_method = None - - def read_table(self, module): - if isinstance(module, types.ModuleType): - parsetab = module - else: - exec('import %s' % module) - parsetab = sys.modules[module] - - if parsetab._tabversion != __tabversion__: - raise VersionError('yacc table file version is out of date') - - self.lr_action = parsetab._lr_action - self.lr_goto = parsetab._lr_goto - - self.lr_productions = [] - for p in parsetab._lr_productions: - self.lr_productions.append(MiniProduction(*p)) - - self.lr_method = parsetab._lr_method - return parsetab._lr_signature - - def read_pickle(self, filename): - try: - import cPickle as pickle - except ImportError: - import pickle - - if not os.path.exists(filename): - raise ImportError - - in_f = open(filename, 'rb') - - tabversion = pickle.load(in_f) - if tabversion != __tabversion__: - raise VersionError('yacc table file version is out of date') - self.lr_method = pickle.load(in_f) - signature = pickle.load(in_f) - self.lr_action = pickle.load(in_f) - self.lr_goto = pickle.load(in_f) - productions = pickle.load(in_f) - - self.lr_productions = [] - for p in productions: - self.lr_productions.append(MiniProduction(*p)) - - in_f.close() - return signature - - # Bind all production function names to callable objects in pdict - def bind_callables(self, pdict): - for p in self.lr_productions: - p.bind(pdict) - - -# ----------------------------------------------------------------------------- -# === LR Generator === -# -# The following classes and functions are used to generate LR parsing tables on -# a grammar. -# ----------------------------------------------------------------------------- - -# ----------------------------------------------------------------------------- -# digraph() -# traverse() -# -# The following two functions are used to compute set valued functions -# of the form: -# -# F(x) = F'(x) U U{F(y) | x R y} -# -# This is used to compute the values of Read() sets as well as FOLLOW sets -# in LALR(1) generation. -# -# Inputs: X - An input set -# R - A relation -# FP - Set-valued function -# ------------------------------------------------------------------------------ - -def digraph(X, R, FP): - N = {} - for x in X: - N[x] = 0 - stack = [] - F = {} - for x in X: - if N[x] == 0: - traverse(x, N, stack, F, X, R, FP) - return F - -def traverse(x, N, stack, F, X, R, FP): - stack.append(x) - d = len(stack) - N[x] = d - F[x] = FP(x) # F(X) <- F'(x) - - rel = R(x) # Get y's related to x - for y in rel: - if N[y] == 0: - traverse(y, N, stack, F, X, R, FP) - N[x] = min(N[x], N[y]) - for a in F.get(y, []): - if a not in F[x]: - F[x].append(a) - if N[x] == d: - N[stack[-1]] = MAXINT - F[stack[-1]] = F[x] - element = stack.pop() - while element != x: - N[stack[-1]] = MAXINT - F[stack[-1]] = F[x] - element = stack.pop() - -class LALRError(YaccError): - pass - -# ----------------------------------------------------------------------------- -# == LRGeneratedTable == -# -# This class implements the LR table generation algorithm. There are no -# public methods except for write() -# ----------------------------------------------------------------------------- - -class LRGeneratedTable(LRTable): - def __init__(self, grammar, method='LALR', log=None): - if method not in ['SLR', 'LALR']: - raise LALRError('Unsupported method %s' % method) - - self.grammar = grammar - self.lr_method = method - - # Set up the logger - if not log: - log = NullLogger() - self.log = log - - # Internal attributes - self.lr_action = {} # Action table - self.lr_goto = {} # Goto table - self.lr_productions = grammar.Productions # Copy of grammar Production array - self.lr_goto_cache = {} # Cache of computed gotos - self.lr0_cidhash = {} # Cache of closures - - self._add_count = 0 # Internal counter used to detect cycles - - # Diagonistic information filled in by the table generator - self.sr_conflict = 0 - self.rr_conflict = 0 - self.conflicts = [] # List of conflicts - - self.sr_conflicts = [] - self.rr_conflicts = [] - - # Build the tables - self.grammar.build_lritems() - self.grammar.compute_first() - self.grammar.compute_follow() - self.lr_parse_table() - - # Compute the LR(0) closure operation on I, where I is a set of LR(0) items. - - def lr0_closure(self, I): - self._add_count += 1 - - # Add everything in I to J - J = I[:] - didadd = True - while didadd: - didadd = False - for j in J: - for x in j.lr_after: - if getattr(x, 'lr0_added', 0) == self._add_count: - continue - # Add B --> .G to J - J.append(x.lr_next) - x.lr0_added = self._add_count - didadd = True - - return J - - # Compute the LR(0) goto function goto(I,X) where I is a set - # of LR(0) items and X is a grammar symbol. This function is written - # in a way that guarantees uniqueness of the generated goto sets - # (i.e. the same goto set will never be returned as two different Python - # objects). With uniqueness, we can later do fast set comparisons using - # id(obj) instead of element-wise comparison. - - def lr0_goto(self, I, x): - # First we look for a previously cached entry - g = self.lr_goto_cache.get((id(I), x)) - if g: - return g - - # Now we generate the goto set in a way that guarantees uniqueness - # of the result - - s = self.lr_goto_cache.get(x) - if not s: - s = {} - self.lr_goto_cache[x] = s - - gs = [] - for p in I: - n = p.lr_next - if n and n.lr_before == x: - s1 = s.get(id(n)) - if not s1: - s1 = {} - s[id(n)] = s1 - gs.append(n) - s = s1 - g = s.get('$end') - if not g: - if gs: - g = self.lr0_closure(gs) - s['$end'] = g - else: - s['$end'] = gs - self.lr_goto_cache[(id(I), x)] = g - return g - - # Compute the LR(0) sets of item function - def lr0_items(self): - C = [self.lr0_closure([self.grammar.Productions[0].lr_next])] - i = 0 - for I in C: - self.lr0_cidhash[id(I)] = i - i += 1 - - # Loop over the items in C and each grammar symbols - i = 0 - while i < len(C): - I = C[i] - i += 1 - - # Collect all of the symbols that could possibly be in the goto(I,X) sets - asyms = {} - for ii in I: - for s in ii.usyms: - asyms[s] = None - - for x in asyms: - g = self.lr0_goto(I, x) - if not g or id(g) in self.lr0_cidhash: - continue - self.lr0_cidhash[id(g)] = len(C) - C.append(g) - - return C - - # ----------------------------------------------------------------------------- - # ==== LALR(1) Parsing ==== - # - # LALR(1) parsing is almost exactly the same as SLR except that instead of - # relying upon Follow() sets when performing reductions, a more selective - # lookahead set that incorporates the state of the LR(0) machine is utilized. - # Thus, we mainly just have to focus on calculating the lookahead sets. - # - # The method used here is due to DeRemer and Pennelo (1982). - # - # DeRemer, F. L., and T. J. Pennelo: "Efficient Computation of LALR(1) - # Lookahead Sets", ACM Transactions on Programming Languages and Systems, - # Vol. 4, No. 4, Oct. 1982, pp. 615-649 - # - # Further details can also be found in: - # - # J. Tremblay and P. Sorenson, "The Theory and Practice of Compiler Writing", - # McGraw-Hill Book Company, (1985). - # - # ----------------------------------------------------------------------------- - - # ----------------------------------------------------------------------------- - # compute_nullable_nonterminals() - # - # Creates a dictionary containing all of the non-terminals that might produce - # an empty production. - # ----------------------------------------------------------------------------- - - def compute_nullable_nonterminals(self): - nullable = set() - num_nullable = 0 - while True: - for p in self.grammar.Productions[1:]: - if p.len == 0: - nullable.add(p.name) - continue - for t in p.prod: - if t not in nullable: - break - else: - nullable.add(p.name) - if len(nullable) == num_nullable: - break - num_nullable = len(nullable) - return nullable - - # ----------------------------------------------------------------------------- - # find_nonterminal_trans(C) - # - # Given a set of LR(0) items, this functions finds all of the non-terminal - # transitions. These are transitions in which a dot appears immediately before - # a non-terminal. Returns a list of tuples of the form (state,N) where state - # is the state number and N is the nonterminal symbol. - # - # The input C is the set of LR(0) items. - # ----------------------------------------------------------------------------- - - def find_nonterminal_transitions(self, C): - trans = [] - for stateno, state in enumerate(C): - for p in state: - if p.lr_index < p.len - 1: - t = (stateno, p.prod[p.lr_index+1]) - if t[1] in self.grammar.Nonterminals: - if t not in trans: - trans.append(t) - return trans - - # ----------------------------------------------------------------------------- - # dr_relation() - # - # Computes the DR(p,A) relationships for non-terminal transitions. The input - # is a tuple (state,N) where state is a number and N is a nonterminal symbol. - # - # Returns a list of terminals. - # ----------------------------------------------------------------------------- - - def dr_relation(self, C, trans, nullable): - dr_set = {} - state, N = trans - terms = [] - - g = self.lr0_goto(C[state], N) - for p in g: - if p.lr_index < p.len - 1: - a = p.prod[p.lr_index+1] - if a in self.grammar.Terminals: - if a not in terms: - terms.append(a) - - # This extra bit is to handle the start state - if state == 0 and N == self.grammar.Productions[0].prod[0]: - terms.append('$end') - - return terms - - # ----------------------------------------------------------------------------- - # reads_relation() - # - # Computes the READS() relation (p,A) READS (t,C). - # ----------------------------------------------------------------------------- - - def reads_relation(self, C, trans, empty): - # Look for empty transitions - rel = [] - state, N = trans - - g = self.lr0_goto(C[state], N) - j = self.lr0_cidhash.get(id(g), -1) - for p in g: - if p.lr_index < p.len - 1: - a = p.prod[p.lr_index + 1] - if a in empty: - rel.append((j, a)) - - return rel - - # ----------------------------------------------------------------------------- - # compute_lookback_includes() - # - # Determines the lookback and includes relations - # - # LOOKBACK: - # - # This relation is determined by running the LR(0) state machine forward. - # For example, starting with a production "N : . A B C", we run it forward - # to obtain "N : A B C ." We then build a relationship between this final - # state and the starting state. These relationships are stored in a dictionary - # lookdict. - # - # INCLUDES: - # - # Computes the INCLUDE() relation (p,A) INCLUDES (p',B). - # - # This relation is used to determine non-terminal transitions that occur - # inside of other non-terminal transition states. (p,A) INCLUDES (p', B) - # if the following holds: - # - # B -> LAT, where T -> epsilon and p' -L-> p - # - # L is essentially a prefix (which may be empty), T is a suffix that must be - # able to derive an empty string. State p' must lead to state p with the string L. - # - # ----------------------------------------------------------------------------- - - def compute_lookback_includes(self, C, trans, nullable): - lookdict = {} # Dictionary of lookback relations - includedict = {} # Dictionary of include relations - - # Make a dictionary of non-terminal transitions - dtrans = {} - for t in trans: - dtrans[t] = 1 - - # Loop over all transitions and compute lookbacks and includes - for state, N in trans: - lookb = [] - includes = [] - for p in C[state]: - if p.name != N: - continue - - # Okay, we have a name match. We now follow the production all the way - # through the state machine until we get the . on the right hand side - - lr_index = p.lr_index - j = state - while lr_index < p.len - 1: - lr_index = lr_index + 1 - t = p.prod[lr_index] - - # Check to see if this symbol and state are a non-terminal transition - if (j, t) in dtrans: - # Yes. Okay, there is some chance that this is an includes relation - # the only way to know for certain is whether the rest of the - # production derives empty - - li = lr_index + 1 - while li < p.len: - if p.prod[li] in self.grammar.Terminals: - break # No forget it - if p.prod[li] not in nullable: - break - li = li + 1 - else: - # Appears to be a relation between (j,t) and (state,N) - includes.append((j, t)) - - g = self.lr0_goto(C[j], t) # Go to next set - j = self.lr0_cidhash.get(id(g), -1) # Go to next state - - # When we get here, j is the final state, now we have to locate the production - for r in C[j]: - if r.name != p.name: - continue - if r.len != p.len: - continue - i = 0 - # This look is comparing a production ". A B C" with "A B C ." - while i < r.lr_index: - if r.prod[i] != p.prod[i+1]: - break - i = i + 1 - else: - lookb.append((j, r)) - for i in includes: - if i not in includedict: - includedict[i] = [] - includedict[i].append((state, N)) - lookdict[(state, N)] = lookb - - return lookdict, includedict - - # ----------------------------------------------------------------------------- - # compute_read_sets() - # - # Given a set of LR(0) items, this function computes the read sets. - # - # Inputs: C = Set of LR(0) items - # ntrans = Set of nonterminal transitions - # nullable = Set of empty transitions - # - # Returns a set containing the read sets - # ----------------------------------------------------------------------------- - - def compute_read_sets(self, C, ntrans, nullable): - FP = lambda x: self.dr_relation(C, x, nullable) - R = lambda x: self.reads_relation(C, x, nullable) - F = digraph(ntrans, R, FP) - return F - - # ----------------------------------------------------------------------------- - # compute_follow_sets() - # - # Given a set of LR(0) items, a set of non-terminal transitions, a readset, - # and an include set, this function computes the follow sets - # - # Follow(p,A) = Read(p,A) U U {Follow(p',B) | (p,A) INCLUDES (p',B)} - # - # Inputs: - # ntrans = Set of nonterminal transitions - # readsets = Readset (previously computed) - # inclsets = Include sets (previously computed) - # - # Returns a set containing the follow sets - # ----------------------------------------------------------------------------- - - def compute_follow_sets(self, ntrans, readsets, inclsets): - FP = lambda x: readsets[x] - R = lambda x: inclsets.get(x, []) - F = digraph(ntrans, R, FP) - return F - - # ----------------------------------------------------------------------------- - # add_lookaheads() - # - # Attaches the lookahead symbols to grammar rules. - # - # Inputs: lookbacks - Set of lookback relations - # followset - Computed follow set - # - # This function directly attaches the lookaheads to productions contained - # in the lookbacks set - # ----------------------------------------------------------------------------- - - def add_lookaheads(self, lookbacks, followset): - for trans, lb in lookbacks.items(): - # Loop over productions in lookback - for state, p in lb: - if state not in p.lookaheads: - p.lookaheads[state] = [] - f = followset.get(trans, []) - for a in f: - if a not in p.lookaheads[state]: - p.lookaheads[state].append(a) - - # ----------------------------------------------------------------------------- - # add_lalr_lookaheads() - # - # This function does all of the work of adding lookahead information for use - # with LALR parsing - # ----------------------------------------------------------------------------- - - def add_lalr_lookaheads(self, C): - # Determine all of the nullable nonterminals - nullable = self.compute_nullable_nonterminals() - - # Find all non-terminal transitions - trans = self.find_nonterminal_transitions(C) - - # Compute read sets - readsets = self.compute_read_sets(C, trans, nullable) - - # Compute lookback/includes relations - lookd, included = self.compute_lookback_includes(C, trans, nullable) - - # Compute LALR FOLLOW sets - followsets = self.compute_follow_sets(trans, readsets, included) - - # Add all of the lookaheads - self.add_lookaheads(lookd, followsets) - - # ----------------------------------------------------------------------------- - # lr_parse_table() - # - # This function constructs the parse tables for SLR or LALR - # ----------------------------------------------------------------------------- - def lr_parse_table(self): - Productions = self.grammar.Productions - Precedence = self.grammar.Precedence - goto = self.lr_goto # Goto array - action = self.lr_action # Action array - log = self.log # Logger for output - - actionp = {} # Action production array (temporary) - - log.info('Parsing method: %s', self.lr_method) - - # Step 1: Construct C = { I0, I1, ... IN}, collection of LR(0) items - # This determines the number of states - - C = self.lr0_items() - - if self.lr_method == 'LALR': - self.add_lalr_lookaheads(C) - - # Build the parser table, state by state - st = 0 - for I in C: - # Loop over each production in I - actlist = [] # List of actions - st_action = {} - st_actionp = {} - st_goto = {} - log.info('') - log.info('state %d', st) - log.info('') - for p in I: - log.info(' (%d) %s', p.number, p) - log.info('') - - for p in I: - if p.len == p.lr_index + 1: - if p.name == "S'": - # Start symbol. Accept! - st_action['$end'] = 0 - st_actionp['$end'] = p - else: - # We are at the end of a production. Reduce! - if self.lr_method == 'LALR': - laheads = p.lookaheads[st] - else: - laheads = self.grammar.Follow[p.name] - for a in laheads: - actlist.append((a, p, 'reduce using rule %d (%s)' % (p.number, p))) - r = st_action.get(a) - if r is not None: - # Whoa. Have a shift/reduce or reduce/reduce conflict - if r > 0: - # Need to decide on shift or reduce here - # By default we favor shifting. Need to add - # some precedence rules here. - sprec, slevel = Productions[st_actionp[a].number].prec - rprec, rlevel = Precedence.get(a, ('right', 0)) - if (slevel < rlevel) or ((slevel == rlevel) and (rprec == 'left')): - # We really need to reduce here. - st_action[a] = -p.number - st_actionp[a] = p - if not slevel and not rlevel: - log.info(' ! shift/reduce conflict for %s resolved as reduce', a) - self.sr_conflicts.append((st, a, 'reduce')) - Productions[p.number].reduced += 1 - elif (slevel == rlevel) and (rprec == 'nonassoc'): - st_action[a] = None - else: - # Hmmm. Guess we'll keep the shift - if not rlevel: - log.info(' ! shift/reduce conflict for %s resolved as shift', a) - self.sr_conflicts.append((st, a, 'shift')) - elif r < 0: - # Reduce/reduce conflict. In this case, we favor the rule - # that was defined first in the grammar file - oldp = Productions[-r] - pp = Productions[p.number] - if oldp.line > pp.line: - st_action[a] = -p.number - st_actionp[a] = p - chosenp, rejectp = pp, oldp - Productions[p.number].reduced += 1 - Productions[oldp.number].reduced -= 1 - else: - chosenp, rejectp = oldp, pp - self.rr_conflicts.append((st, chosenp, rejectp)) - log.info(' ! reduce/reduce conflict for %s resolved using rule %d (%s)', - a, st_actionp[a].number, st_actionp[a]) - else: - raise LALRError('Unknown conflict in state %d' % st) - else: - st_action[a] = -p.number - st_actionp[a] = p - Productions[p.number].reduced += 1 - else: - i = p.lr_index - a = p.prod[i+1] # Get symbol right after the "." - if a in self.grammar.Terminals: - g = self.lr0_goto(I, a) - j = self.lr0_cidhash.get(id(g), -1) - if j >= 0: - # We are in a shift state - actlist.append((a, p, 'shift and go to state %d' % j)) - r = st_action.get(a) - if r is not None: - # Whoa have a shift/reduce or shift/shift conflict - if r > 0: - if r != j: - raise LALRError('Shift/shift conflict in state %d' % st) - elif r < 0: - # Do a precedence check. - # - if precedence of reduce rule is higher, we reduce. - # - if precedence of reduce is same and left assoc, we reduce. - # - otherwise we shift - rprec, rlevel = Productions[st_actionp[a].number].prec - sprec, slevel = Precedence.get(a, ('right', 0)) - if (slevel > rlevel) or ((slevel == rlevel) and (rprec == 'right')): - # We decide to shift here... highest precedence to shift - Productions[st_actionp[a].number].reduced -= 1 - st_action[a] = j - st_actionp[a] = p - if not rlevel: - log.info(' ! shift/reduce conflict for %s resolved as shift', a) - self.sr_conflicts.append((st, a, 'shift')) - elif (slevel == rlevel) and (rprec == 'nonassoc'): - st_action[a] = None - else: - # Hmmm. Guess we'll keep the reduce - if not slevel and not rlevel: - log.info(' ! shift/reduce conflict for %s resolved as reduce', a) - self.sr_conflicts.append((st, a, 'reduce')) - - else: - raise LALRError('Unknown conflict in state %d' % st) - else: - st_action[a] = j - st_actionp[a] = p - - # Print the actions associated with each terminal - _actprint = {} - for a, p, m in actlist: - if a in st_action: - if p is st_actionp[a]: - log.info(' %-15s %s', a, m) - _actprint[(a, m)] = 1 - log.info('') - # Print the actions that were not used. (debugging) - not_used = 0 - for a, p, m in actlist: - if a in st_action: - if p is not st_actionp[a]: - if not (a, m) in _actprint: - log.debug(' ! %-15s [ %s ]', a, m) - not_used = 1 - _actprint[(a, m)] = 1 - if not_used: - log.debug('') - - # Construct the goto table for this state - - nkeys = {} - for ii in I: - for s in ii.usyms: - if s in self.grammar.Nonterminals: - nkeys[s] = None - for n in nkeys: - g = self.lr0_goto(I, n) - j = self.lr0_cidhash.get(id(g), -1) - if j >= 0: - st_goto[n] = j - log.info(' %-30s shift and go to state %d', n, j) - - action[st] = st_action - actionp[st] = st_actionp - goto[st] = st_goto - st += 1 - - # ----------------------------------------------------------------------------- - # write() - # - # This function writes the LR parsing tables to a file - # ----------------------------------------------------------------------------- - - def write_table(self, tabmodule, outputdir='', signature=''): - if isinstance(tabmodule, types.ModuleType): - raise IOError("Won't overwrite existing tabmodule") - - basemodulename = tabmodule.split('.')[-1] - filename = os.path.join(outputdir, basemodulename) + '.py' - try: - f = open(filename, 'w') - - f.write(''' -# %s -# This file is automatically generated. Do not edit. -_tabversion = %r - -_lr_method = %r - -_lr_signature = %r - ''' % (os.path.basename(filename), __tabversion__, self.lr_method, signature)) - - # Change smaller to 0 to go back to original tables - smaller = 1 - - # Factor out names to try and make smaller - if smaller: - items = {} - - for s, nd in self.lr_action.items(): - for name, v in nd.items(): - i = items.get(name) - if not i: - i = ([], []) - items[name] = i - i[0].append(s) - i[1].append(v) - - f.write('\n_lr_action_items = {') - for k, v in items.items(): - f.write('%r:([' % k) - for i in v[0]: - f.write('%r,' % i) - f.write('],[') - for i in v[1]: - f.write('%r,' % i) - - f.write(']),') - f.write('}\n') - - f.write(''' -_lr_action = {} -for _k, _v in _lr_action_items.items(): - for _x,_y in zip(_v[0],_v[1]): - if not _x in _lr_action: _lr_action[_x] = {} - _lr_action[_x][_k] = _y -del _lr_action_items -''') - - else: - f.write('\n_lr_action = { ') - for k, v in self.lr_action.items(): - f.write('(%r,%r):%r,' % (k[0], k[1], v)) - f.write('}\n') - - if smaller: - # Factor out names to try and make smaller - items = {} - - for s, nd in self.lr_goto.items(): - for name, v in nd.items(): - i = items.get(name) - if not i: - i = ([], []) - items[name] = i - i[0].append(s) - i[1].append(v) - - f.write('\n_lr_goto_items = {') - for k, v in items.items(): - f.write('%r:([' % k) - for i in v[0]: - f.write('%r,' % i) - f.write('],[') - for i in v[1]: - f.write('%r,' % i) - - f.write(']),') - f.write('}\n') - - f.write(''' -_lr_goto = {} -for _k, _v in _lr_goto_items.items(): - for _x, _y in zip(_v[0], _v[1]): - if not _x in _lr_goto: _lr_goto[_x] = {} - _lr_goto[_x][_k] = _y -del _lr_goto_items -''') - else: - f.write('\n_lr_goto = { ') - for k, v in self.lr_goto.items(): - f.write('(%r,%r):%r,' % (k[0], k[1], v)) - f.write('}\n') - - # Write production table - f.write('_lr_productions = [\n') - for p in self.lr_productions: - if p.func: - f.write(' (%r,%r,%d,%r,%r,%d),\n' % (p.str, p.name, p.len, - p.func, os.path.basename(p.file), p.line)) - else: - f.write(' (%r,%r,%d,None,None,None),\n' % (str(p), p.name, p.len)) - f.write(']\n') - f.close() - - except IOError as e: - raise - - - # ----------------------------------------------------------------------------- - # pickle_table() - # - # This function pickles the LR parsing tables to a supplied file object - # ----------------------------------------------------------------------------- - - def pickle_table(self, filename, signature=''): - try: - import cPickle as pickle - except ImportError: - import pickle - with open(filename, 'wb') as outf: - pickle.dump(__tabversion__, outf, pickle_protocol) - pickle.dump(self.lr_method, outf, pickle_protocol) - pickle.dump(signature, outf, pickle_protocol) - pickle.dump(self.lr_action, outf, pickle_protocol) - pickle.dump(self.lr_goto, outf, pickle_protocol) - - outp = [] - for p in self.lr_productions: - if p.func: - outp.append((p.str, p.name, p.len, p.func, os.path.basename(p.file), p.line)) - else: - outp.append((str(p), p.name, p.len, None, None, None)) - pickle.dump(outp, outf, pickle_protocol) - -# ----------------------------------------------------------------------------- -# === INTROSPECTION === -# -# The following functions and classes are used to implement the PLY -# introspection features followed by the yacc() function itself. -# ----------------------------------------------------------------------------- - -# ----------------------------------------------------------------------------- -# get_caller_module_dict() -# -# This function returns a dictionary containing all of the symbols defined within -# a caller further down the call stack. This is used to get the environment -# associated with the yacc() call if none was provided. -# ----------------------------------------------------------------------------- - -def get_caller_module_dict(levels): - f = sys._getframe(levels) - ldict = f.f_globals.copy() - if f.f_globals != f.f_locals: - ldict.update(f.f_locals) - return ldict - -# ----------------------------------------------------------------------------- -# parse_grammar() -# -# This takes a raw grammar rule string and parses it into production data -# ----------------------------------------------------------------------------- -def parse_grammar(doc, file, line): - grammar = [] - # Split the doc string into lines - pstrings = doc.splitlines() - lastp = None - dline = line - for ps in pstrings: - dline += 1 - p = ps.split() - if not p: - continue - try: - if p[0] == '|': - # This is a continuation of a previous rule - if not lastp: - raise SyntaxError("%s:%d: Misplaced '|'" % (file, dline)) - prodname = lastp - syms = p[1:] - else: - prodname = p[0] - lastp = prodname - syms = p[2:] - assign = p[1] - if assign != ':' and assign != '::=': - raise SyntaxError("%s:%d: Syntax error. Expected ':'" % (file, dline)) - - grammar.append((file, dline, prodname, syms)) - except SyntaxError: - raise - except Exception: - raise SyntaxError('%s:%d: Syntax error in rule %r' % (file, dline, ps.strip())) - - return grammar - -# ----------------------------------------------------------------------------- -# ParserReflect() -# -# This class represents information extracted for building a parser including -# start symbol, error function, tokens, precedence list, action functions, -# etc. -# ----------------------------------------------------------------------------- -class ParserReflect(object): - def __init__(self, pdict, log=None): - self.pdict = pdict - self.start = None - self.error_func = None - self.tokens = None - self.modules = set() - self.grammar = [] - self.error = False - - if log is None: - self.log = PlyLogger(sys.stderr) - else: - self.log = log - - # Get all of the basic information - def get_all(self): - self.get_start() - self.get_error_func() - self.get_tokens() - self.get_precedence() - self.get_pfunctions() - - # Validate all of the information - def validate_all(self): - self.validate_start() - self.validate_error_func() - self.validate_tokens() - self.validate_precedence() - self.validate_pfunctions() - self.validate_modules() - return self.error - - # Compute a signature over the grammar - def signature(self): - try: - from hashlib import md5 - except ImportError: - from md5 import md5 - try: - sig = md5() - if self.start: - sig.update(self.start.encode('latin-1')) - if self.prec: - sig.update(''.join([''.join(p) for p in self.prec]).encode('latin-1')) - if self.tokens: - sig.update(' '.join(self.tokens).encode('latin-1')) - for f in self.pfuncs: - if f[3]: - sig.update(f[3].encode('latin-1')) - except (TypeError, ValueError): - pass - - digest = base64.b16encode(sig.digest()) - if sys.version_info[0] >= 3: - digest = digest.decode('latin-1') - return digest - - # ----------------------------------------------------------------------------- - # validate_modules() - # - # This method checks to see if there are duplicated p_rulename() functions - # in the parser module file. Without this function, it is really easy for - # users to make mistakes by cutting and pasting code fragments (and it's a real - # bugger to try and figure out why the resulting parser doesn't work). Therefore, - # we just do a little regular expression pattern matching of def statements - # to try and detect duplicates. - # ----------------------------------------------------------------------------- - - def validate_modules(self): - # Match def p_funcname( - fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(') - - for module in self.modules: - lines, linen = inspect.getsourcelines(module) - - counthash = {} - for linen, line in enumerate(lines): - linen += 1 - m = fre.match(line) - if m: - name = m.group(1) - prev = counthash.get(name) - if not prev: - counthash[name] = linen - else: - filename = inspect.getsourcefile(module) - self.log.warning('%s:%d: Function %s redefined. Previously defined on line %d', - filename, linen, name, prev) - - # Get the start symbol - def get_start(self): - self.start = self.pdict.get('start') - - # Validate the start symbol - def validate_start(self): - if self.start is not None: - if not isinstance(self.start, string_types): - self.log.error("'start' must be a string") - - # Look for error handler - def get_error_func(self): - self.error_func = self.pdict.get('p_error') - - # Validate the error function - def validate_error_func(self): - if self.error_func: - if isinstance(self.error_func, types.FunctionType): - ismethod = 0 - elif isinstance(self.error_func, types.MethodType): - ismethod = 1 - else: - self.log.error("'p_error' defined, but is not a function or method") - self.error = True - return - - eline = self.error_func.__code__.co_firstlineno - efile = self.error_func.__code__.co_filename - module = inspect.getmodule(self.error_func) - self.modules.add(module) - - argcount = self.error_func.__code__.co_argcount - ismethod - if argcount != 1: - self.log.error('%s:%d: p_error() requires 1 argument', efile, eline) - self.error = True - - # Get the tokens map - def get_tokens(self): - tokens = self.pdict.get('tokens') - if not tokens: - self.log.error('No token list is defined') - self.error = True - return - - if not isinstance(tokens, (list, tuple)): - self.log.error('tokens must be a list or tuple') - self.error = True - return - - if not tokens: - self.log.error('tokens is empty') - self.error = True - return - - self.tokens = tokens - - # Validate the tokens - def validate_tokens(self): - # Validate the tokens. - if 'error' in self.tokens: - self.log.error("Illegal token name 'error'. Is a reserved word") - self.error = True - return - - terminals = set() - for n in self.tokens: - if n in terminals: - self.log.warning('Token %r multiply defined', n) - terminals.add(n) - - # Get the precedence map (if any) - def get_precedence(self): - self.prec = self.pdict.get('precedence') - - # Validate and parse the precedence map - def validate_precedence(self): - preclist = [] - if self.prec: - if not isinstance(self.prec, (list, tuple)): - self.log.error('precedence must be a list or tuple') - self.error = True - return - for level, p in enumerate(self.prec): - if not isinstance(p, (list, tuple)): - self.log.error('Bad precedence table') - self.error = True - return - - if len(p) < 2: - self.log.error('Malformed precedence entry %s. Must be (assoc, term, ..., term)', p) - self.error = True - return - assoc = p[0] - if not isinstance(assoc, string_types): - self.log.error('precedence associativity must be a string') - self.error = True - return - for term in p[1:]: - if not isinstance(term, string_types): - self.log.error('precedence items must be strings') - self.error = True - return - preclist.append((term, assoc, level+1)) - self.preclist = preclist - - # Get all p_functions from the grammar - def get_pfunctions(self): - p_functions = [] - for name, item in self.pdict.items(): - if not name.startswith('p_') or name == 'p_error': - continue - if isinstance(item, (types.FunctionType, types.MethodType)): - line = item.__code__.co_firstlineno - module = inspect.getmodule(item) - p_functions.append((line, module, name, item.__doc__)) - - # Sort all of the actions by line number; make sure to stringify - # modules to make them sortable, since `line` may not uniquely sort all - # p functions - p_functions.sort(key=lambda p_function: ( - p_function[0], - str(p_function[1]), - p_function[2], - p_function[3])) - self.pfuncs = p_functions - - # Validate all of the p_functions - def validate_pfunctions(self): - grammar = [] - # Check for non-empty symbols - if len(self.pfuncs) == 0: - self.log.error('no rules of the form p_rulename are defined') - self.error = True - return - - for line, module, name, doc in self.pfuncs: - file = inspect.getsourcefile(module) - func = self.pdict[name] - if isinstance(func, types.MethodType): - reqargs = 2 - else: - reqargs = 1 - if func.__code__.co_argcount > reqargs: - self.log.error('%s:%d: Rule %r has too many arguments', file, line, func.__name__) - self.error = True - elif func.__code__.co_argcount < reqargs: - self.log.error('%s:%d: Rule %r requires an argument', file, line, func.__name__) - self.error = True - elif not func.__doc__: - self.log.warning('%s:%d: No documentation string specified in function %r (ignored)', - file, line, func.__name__) - else: - try: - parsed_g = parse_grammar(doc, file, line) - for g in parsed_g: - grammar.append((name, g)) - except SyntaxError as e: - self.log.error(str(e)) - self.error = True - - # Looks like a valid grammar rule - # Mark the file in which defined. - self.modules.add(module) - - # Secondary validation step that looks for p_ definitions that are not functions - # or functions that look like they might be grammar rules. - - for n, v in self.pdict.items(): - if n.startswith('p_') and isinstance(v, (types.FunctionType, types.MethodType)): - continue - if n.startswith('t_'): - continue - if n.startswith('p_') and n != 'p_error': - self.log.warning('%r not defined as a function', n) - if ((isinstance(v, types.FunctionType) and v.__code__.co_argcount == 1) or - (isinstance(v, types.MethodType) and v.__func__.__code__.co_argcount == 2)): - if v.__doc__: - try: - doc = v.__doc__.split(' ') - if doc[1] == ':': - self.log.warning('%s:%d: Possible grammar rule %r defined without p_ prefix', - v.__code__.co_filename, v.__code__.co_firstlineno, n) - except IndexError: - pass - - self.grammar = grammar - -# ----------------------------------------------------------------------------- -# yacc(module) -# -# Build a parser -# ----------------------------------------------------------------------------- - -def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, start=None, - check_recursion=True, optimize=False, write_tables=True, debugfile=debug_file, - outputdir=None, debuglog=None, errorlog=None, picklefile=None): - - if tabmodule is None: - tabmodule = tab_module - - # Reference to the parsing method of the last built parser - global parse - - # If pickling is enabled, table files are not created - if picklefile: - write_tables = 0 - - if errorlog is None: - errorlog = PlyLogger(sys.stderr) - - # Get the module dictionary used for the parser - if module: - _items = [(k, getattr(module, k)) for k in dir(module)] - pdict = dict(_items) - # If no __file__ attribute is available, try to obtain it from the __module__ instead - if '__file__' not in pdict: - pdict['__file__'] = sys.modules[pdict['__module__']].__file__ - else: - pdict = get_caller_module_dict(2) - - if outputdir is None: - # If no output directory is set, the location of the output files - # is determined according to the following rules: - # - If tabmodule specifies a package, files go into that package directory - # - Otherwise, files go in the same directory as the specifying module - if isinstance(tabmodule, types.ModuleType): - srcfile = tabmodule.__file__ - else: - if '.' not in tabmodule: - srcfile = pdict['__file__'] - else: - parts = tabmodule.split('.') - pkgname = '.'.join(parts[:-1]) - exec('import %s' % pkgname) - srcfile = getattr(sys.modules[pkgname], '__file__', '') - outputdir = os.path.dirname(srcfile) - - # Determine if the module is package of a package or not. - # If so, fix the tabmodule setting so that tables load correctly - pkg = pdict.get('__package__') - if pkg and isinstance(tabmodule, str): - if '.' not in tabmodule: - tabmodule = pkg + '.' + tabmodule - - - - # Set start symbol if it's specified directly using an argument - if start is not None: - pdict['start'] = start - - # Collect parser information from the dictionary - pinfo = ParserReflect(pdict, log=errorlog) - pinfo.get_all() - - if pinfo.error: - raise YaccError('Unable to build parser') - - # Check signature against table files (if any) - signature = pinfo.signature() - - # Read the tables - try: - lr = LRTable() - if picklefile: - read_signature = lr.read_pickle(picklefile) - else: - read_signature = lr.read_table(tabmodule) - if optimize or (read_signature == signature): - try: - lr.bind_callables(pinfo.pdict) - parser = LRParser(lr, pinfo.error_func) - parse = parser.parse - return parser - except Exception as e: - errorlog.warning('There was a problem loading the table file: %r', e) - except VersionError as e: - errorlog.warning(str(e)) - except ImportError: - pass - - if debuglog is None: - if debug: - try: - debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w')) - except IOError as e: - errorlog.warning("Couldn't open %r. %s" % (debugfile, e)) - debuglog = NullLogger() - else: - debuglog = NullLogger() - - debuglog.info('Created by PLY version %s (http://www.dabeaz.com/ply)', __version__) - - errors = False - - # Validate the parser information - if pinfo.validate_all(): - raise YaccError('Unable to build parser') - - if not pinfo.error_func: - errorlog.warning('no p_error() function is defined') - - # Create a grammar object - grammar = Grammar(pinfo.tokens) - - # Set precedence level for terminals - for term, assoc, level in pinfo.preclist: - try: - grammar.set_precedence(term, assoc, level) - except GrammarError as e: - errorlog.warning('%s', e) - - # Add productions to the grammar - for funcname, gram in pinfo.grammar: - file, line, prodname, syms = gram - try: - grammar.add_production(prodname, syms, funcname, file, line) - except GrammarError as e: - errorlog.error('%s', e) - errors = True - - # Set the grammar start symbols - try: - if start is None: - grammar.set_start(pinfo.start) - else: - grammar.set_start(start) - except GrammarError as e: - errorlog.error(str(e)) - errors = True - - if errors: - raise YaccError('Unable to build parser') - - # Verify the grammar structure - undefined_symbols = grammar.undefined_symbols() - for sym, prod in undefined_symbols: - errorlog.error('%s:%d: Symbol %r used, but not defined as a token or a rule', prod.file, prod.line, sym) - errors = True - - unused_terminals = grammar.unused_terminals() - if unused_terminals: - debuglog.info('') - debuglog.info('Unused terminals:') - debuglog.info('') - for term in unused_terminals: - errorlog.warning('Token %r defined, but not used', term) - debuglog.info(' %s', term) - - # Print out all productions to the debug log - if debug: - debuglog.info('') - debuglog.info('Grammar') - debuglog.info('') - for n, p in enumerate(grammar.Productions): - debuglog.info('Rule %-5d %s', n, p) - - # Find unused non-terminals - unused_rules = grammar.unused_rules() - for prod in unused_rules: - errorlog.warning('%s:%d: Rule %r defined, but not used', prod.file, prod.line, prod.name) - - if len(unused_terminals) == 1: - errorlog.warning('There is 1 unused token') - if len(unused_terminals) > 1: - errorlog.warning('There are %d unused tokens', len(unused_terminals)) - - if len(unused_rules) == 1: - errorlog.warning('There is 1 unused rule') - if len(unused_rules) > 1: - errorlog.warning('There are %d unused rules', len(unused_rules)) - - if debug: - debuglog.info('') - debuglog.info('Terminals, with rules where they appear') - debuglog.info('') - terms = list(grammar.Terminals) - terms.sort() - for term in terms: - debuglog.info('%-20s : %s', term, ' '.join([str(s) for s in grammar.Terminals[term]])) - - debuglog.info('') - debuglog.info('Nonterminals, with rules where they appear') - debuglog.info('') - nonterms = list(grammar.Nonterminals) - nonterms.sort() - for nonterm in nonterms: - debuglog.info('%-20s : %s', nonterm, ' '.join([str(s) for s in grammar.Nonterminals[nonterm]])) - debuglog.info('') - - if check_recursion: - unreachable = grammar.find_unreachable() - for u in unreachable: - errorlog.warning('Symbol %r is unreachable', u) - - infinite = grammar.infinite_cycles() - for inf in infinite: - errorlog.error('Infinite recursion detected for symbol %r', inf) - errors = True - - unused_prec = grammar.unused_precedence() - for term, assoc in unused_prec: - errorlog.error('Precedence rule %r defined for unknown symbol %r', assoc, term) - errors = True - - if errors: - raise YaccError('Unable to build parser') - - # Run the LRGeneratedTable on the grammar - if debug: - errorlog.debug('Generating %s tables', method) - - lr = LRGeneratedTable(grammar, method, debuglog) - - if debug: - num_sr = len(lr.sr_conflicts) - - # Report shift/reduce and reduce/reduce conflicts - if num_sr == 1: - errorlog.warning('1 shift/reduce conflict') - elif num_sr > 1: - errorlog.warning('%d shift/reduce conflicts', num_sr) - - num_rr = len(lr.rr_conflicts) - if num_rr == 1: - errorlog.warning('1 reduce/reduce conflict') - elif num_rr > 1: - errorlog.warning('%d reduce/reduce conflicts', num_rr) - - # Write out conflicts to the output file - if debug and (lr.sr_conflicts or lr.rr_conflicts): - debuglog.warning('') - debuglog.warning('Conflicts:') - debuglog.warning('') - - for state, tok, resolution in lr.sr_conflicts: - debuglog.warning('shift/reduce conflict for %s in state %d resolved as %s', tok, state, resolution) - - already_reported = set() - for state, rule, rejected in lr.rr_conflicts: - if (state, id(rule), id(rejected)) in already_reported: - continue - debuglog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule) - debuglog.warning('rejected rule (%s) in state %d', rejected, state) - errorlog.warning('reduce/reduce conflict in state %d resolved using rule (%s)', state, rule) - errorlog.warning('rejected rule (%s) in state %d', rejected, state) - already_reported.add((state, id(rule), id(rejected))) - - warned_never = [] - for state, rule, rejected in lr.rr_conflicts: - if not rejected.reduced and (rejected not in warned_never): - debuglog.warning('Rule (%s) is never reduced', rejected) - errorlog.warning('Rule (%s) is never reduced', rejected) - warned_never.append(rejected) - - # Write the table file if requested - if write_tables: - try: - lr.write_table(tabmodule, outputdir, signature) - except IOError as e: - errorlog.warning("Couldn't create %r. %s" % (tabmodule, e)) - - # Write a pickled version of the tables - if picklefile: - try: - lr.pickle_table(picklefile, signature) - except IOError as e: - errorlog.warning("Couldn't create %r. %s" % (picklefile, e)) - - # Build the parser - lr.bind_callables(pinfo.pdict) - parser = LRParser(lr, pinfo.error_func) - - parse = parser.parse - return parser diff --git a/python/helpers/pydev/ply/ygen.py b/python/helpers/pydev/ply/ygen.py deleted file mode 100644 index acf5ca1a37b6..000000000000 --- a/python/helpers/pydev/ply/ygen.py +++ /dev/null @@ -1,74 +0,0 @@ -# ply: ygen.py -# -# This is a support program that auto-generates different versions of the YACC parsing -# function with different features removed for the purposes of performance. -# -# Users should edit the method LParser.parsedebug() in yacc.py. The source code -# for that method is then used to create the other methods. See the comments in -# yacc.py for further details. - -import os.path -import shutil - -def get_source_range(lines, tag): - srclines = enumerate(lines) - start_tag = '#--! %s-start' % tag - end_tag = '#--! %s-end' % tag - - for start_index, line in srclines: - if line.strip().startswith(start_tag): - break - - for end_index, line in srclines: - if line.strip().endswith(end_tag): - break - - return (start_index + 1, end_index) - -def filter_section(lines, tag): - filtered_lines = [] - include = True - tag_text = '#--! %s' % tag - for line in lines: - if line.strip().startswith(tag_text): - include = not include - elif include: - filtered_lines.append(line) - return filtered_lines - -def main(): - dirname = os.path.dirname(__file__) - shutil.copy2(os.path.join(dirname, 'yacc.py'), os.path.join(dirname, 'yacc.py.bak')) - with open(os.path.join(dirname, 'yacc.py'), 'r') as f: - lines = f.readlines() - - parse_start, parse_end = get_source_range(lines, 'parsedebug') - parseopt_start, parseopt_end = get_source_range(lines, 'parseopt') - parseopt_notrack_start, parseopt_notrack_end = get_source_range(lines, 'parseopt-notrack') - - # Get the original source - orig_lines = lines[parse_start:parse_end] - - # Filter the DEBUG sections out - parseopt_lines = filter_section(orig_lines, 'DEBUG') - - # Filter the TRACKING sections out - parseopt_notrack_lines = filter_section(parseopt_lines, 'TRACKING') - - # Replace the parser source sections with updated versions - lines[parseopt_notrack_start:parseopt_notrack_end] = parseopt_notrack_lines - lines[parseopt_start:parseopt_end] = parseopt_lines - - lines = [line.rstrip()+'\n' for line in lines] - with open(os.path.join(dirname, 'yacc.py'), 'w') as f: - f.writelines(lines) - - print('Updated yacc.py') - -if __name__ == '__main__': - main() - - - - - diff --git a/python/helpers/pydev/thriftpy/__init__.py b/python/helpers/pydev/thriftpy/__init__.py deleted file mode 100644 index 6a2377c9f296..000000000000 --- a/python/helpers/pydev/thriftpy/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys - -from .hook import install_import_hook, remove_import_hook -from .parser import load, load_module, load_fp - -__version__ = '0.3.8' -__python__ = sys.version_info -__all__ = ["install_import_hook", "remove_import_hook", "load", "load_module", - "load_fp"] diff --git a/python/helpers/pydev/thriftpy/_compat.py b/python/helpers/pydev/thriftpy/_compat.py deleted file mode 100644 index 43100ff8484e..000000000000 --- a/python/helpers/pydev/thriftpy/_compat.py +++ /dev/null @@ -1,126 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - thriftpy._compat - ~~~~~~~~~~~~~ - - py2/py3 compatibility support. -""" - -from __future__ import absolute_import - -import platform -import sys -import types - -PY3 = sys.version_info[0] == 3 -PYPY = "__pypy__" in sys.modules - -UNIX = platform.system() in ("Linux", "Darwin") -CYTHON = False # Cython always disabled in pypy and windows - -# only python2.7.9 and python 3.4 or above have true ssl context -MODERN_SSL = (2, 7, 9) <= sys.version_info < (3, 0, 0) or \ - sys.version_info >= (3, 4) - -if PY3: - text_type = str - string_types = (str,) - - def u(s): - return s -else: - text_type = unicode # noqa - string_types = (str, unicode) # noqa - - def u(s): - if not isinstance(s, text_type): - s = s.decode("utf-8") - return s - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass for py2 & py3 - - This code snippet is copied from six.""" - # This requires a bit of explanation: the basic idea is to make a - # dummy metaclass for one level of class instantiation that replaces - # itself with the actual metaclass. Because of internal type checks - # we also need to make sure that we downgrade the custom metaclass - # for one level to something closer to type (that's why __call__ and - # __init__ comes back from type etc.). - class metaclass(meta): - __call__ = type.__call__ - __init__ = type.__init__ - - def __new__(cls, name, this_bases, d): - if this_bases is None: - return type.__new__(cls, name, (), d) - return meta(name, bases, d) - return metaclass('temporary_class', None, {}) - - -def init_func_generator(spec): - """Generate `__init__` function based on TPayload.default_spec - - For example:: - - spec = [('name', 'Alice'), ('number', None)] - - will generate:: - - def __init__(self, name='Alice', number=None): - kwargs = locals() - kwargs.pop('self') - self.__dict__.update(kwargs) - - TODO: The `locals()` part may need refine. - """ - if not spec: - def __init__(self): - pass - return __init__ - - varnames, defaults = zip(*spec) - varnames = ('self', ) + varnames - - def init(self): - self.__dict__ = locals().copy() - del self.__dict__['self'] - - code = init.__code__ - if PY3: - new_code = types.CodeType(len(varnames), - 0, - len(varnames), - code.co_stacksize, - code.co_flags, - code.co_code, - code.co_consts, - code.co_names, - varnames, - code.co_filename, - "__init__", - code.co_firstlineno, - code.co_lnotab, - code.co_freevars, - code.co_cellvars) - else: - new_code = types.CodeType(len(varnames), - len(varnames), - code.co_stacksize, - code.co_flags, - code.co_code, - code.co_consts, - code.co_names, - varnames, - code.co_filename, - "__init__", - code.co_firstlineno, - code.co_lnotab, - code.co_freevars, - code.co_cellvars) - - return types.FunctionType(new_code, - {"__builtins__": __builtins__}, - argdefs=defaults) diff --git a/python/helpers/pydev/thriftpy/contrib/__init__.py b/python/helpers/pydev/thriftpy/contrib/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/helpers/pydev/thriftpy/contrib/tracking/__init__.py b/python/helpers/pydev/thriftpy/contrib/tracking/__init__.py deleted file mode 100644 index d412f08dcac3..000000000000 --- a/python/helpers/pydev/thriftpy/contrib/tracking/__init__.py +++ /dev/null @@ -1,201 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Tracking support similar to twitter finagle-thrift. - -Note: When using tracking, every client should have a corresponding -server processor. -""" - -from __future__ import absolute_import - -import os.path -import time - -from ...parser import load -from ...thrift import TClient, TApplicationException, TMessageType, \ - TProcessor, TType - -track_method = "__thriftpy_tracing_method_name__v2" -track_thrift = load(os.path.join(os.path.dirname(__file__), "tracking.thrift")) - - -__all__ = ["TTrackedClient", "TTrackedProcessor", "TrackerBase", - "ConsoleTracker"] - - -class RequestInfo(object): - def __init__(self, request_id, api, seq, client, server, status, start, - end, annotation, meta): - """Used to store call info. - - :request_id: used to identity a request - :api: api name - :seq: sequence number - :client: client name - :server: server name - :status: request status - :start: start timestamp - :end: end timestamp - :annotation: application-level key-value datas - """ - self.request_id = request_id - self.api = api - self.seq = seq - self.client = client - self.server = server - self.status = status - self.start = start - self.end = end - self.annotation = annotation - self.meta = meta - - -class TTrackedClient(TClient): - def __init__(self, tracker_handler, *args, **kwargs): - super(TTrackedClient, self).__init__(*args, **kwargs) - - self.tracker = tracker_handler - self._upgraded = False - - try: - self._negotiation() - self._upgraded = True - except TApplicationException as e: - if e.type != TApplicationException.UNKNOWN_METHOD: - raise - - def _negotiation(self): - self._oprot.write_message_begin(track_method, TMessageType.CALL, - self._seqid) - args = track_thrift.UpgradeArgs() - self.tracker.init_handshake_info(args) - args.write(self._oprot) - self._oprot.write_message_end() - self._oprot.trans.flush() - - api, msg_type, seqid = self._iprot.read_message_begin() - if msg_type == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(self._iprot) - self._iprot.read_message_end() - raise x - else: - result = track_thrift.UpgradeReply() - result.read(self._iprot) - self._iprot.read_message_end() - - def _send(self, _api, **kwargs): - if self._upgraded: - self._header = track_thrift.RequestHeader() - self.tracker.gen_header(self._header) - self._header.write(self._oprot) - - self.send_start = int(time.time() * 1000) - super(TTrackedClient, self)._send(_api, **kwargs) - - def _req(self, _api, *args, **kwargs): - if not self._upgraded: - return super(TTrackedClient, self)._req(_api, *args, **kwargs) - - exception = None - status = False - - try: - res = super(TTrackedClient, self)._req(_api, *args, **kwargs) - status = True - return res - except BaseException as e: - exception = e - raise - finally: - header_info = RequestInfo( - request_id=self._header.request_id, - seq=self._header.seq, - client=self.tracker.client, - server=self.tracker.server, - api=_api, - status=status, - start=self.send_start, - end=int(time.time() * 1000), - annotation=self.tracker.annotation, - meta=self._header.meta, - ) - self.tracker.record(header_info, exception) - - -class TTrackedProcessor(TProcessor): - def __init__(self, tracker_handler, *args, **kwargs): - super(TTrackedProcessor, self).__init__(*args, **kwargs) - - self.tracker = tracker_handler - self._upgraded = False - - def process(self, iprot, oprot): - if not self._upgraded: - res = self._try_upgrade(iprot) - else: - request_header = track_thrift.RequestHeader() - request_header.read(iprot) - self.tracker.handle(request_header) - res = super(TTrackedProcessor, self).process_in(iprot) - - self._do_process(iprot, oprot, *res) - - def _try_upgrade(self, iprot): - api, msg_type, seqid = iprot.read_message_begin() - if msg_type == TMessageType.CALL and api == track_method: - self._upgraded = True - - args = track_thrift.UpgradeArgs() - args.read(iprot) - self.tracker.handle_handshake_info(args) - result = track_thrift.UpgradeReply() - result.oneway = False - - def call(): - pass - iprot.read_message_end() - else: - result, call = self._process_in(api, iprot) - - return api, seqid, result, call - - def _process_in(self, api, iprot): - if api not in self._service.thrift_services: - iprot.skip(TType.STRUCT) - iprot.read_message_end() - return TApplicationException( - TApplicationException.UNKNOWN_METHOD), None - - args = getattr(self._service, api + "_args")() - args.read(iprot) - iprot.read_message_end() - result = getattr(self._service, api + "_result")() - - # convert kwargs to args - api_args = [args.thrift_spec[k][1] - for k in sorted(args.thrift_spec)] - - def call(): - return getattr(self._handler, api)( - *(args.__dict__[k] for k in api_args) - ) - - return result, call - - def _do_process(self, iprot, oprot, api, seqid, result, call): - if isinstance(result, TApplicationException): - return self.send_exception(oprot, api, result, seqid) - - try: - result.success = call() - except Exception as e: - # raise if api don't have throws - self.handle_exception(e, result) - - if not result.oneway: - self.send_result(oprot, api, result, seqid) - - -from .tracker import TrackerBase, ConsoleTracker # noqa diff --git a/python/helpers/pydev/thriftpy/contrib/tracking/tracker.py b/python/helpers/pydev/thriftpy/contrib/tracking/tracker.py deleted file mode 100644 index f5c31b958128..000000000000 --- a/python/helpers/pydev/thriftpy/contrib/tracking/tracker.py +++ /dev/null @@ -1,113 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import contextlib -import copy -import threading -import uuid - -ctx = threading.local() - - -class TrackerBase(object): - def __init__(self, client=None, server=None): - self.client = client - self.server = server - - def handle(self, header): - ctx.header = header - ctx.counter = 0 - - def gen_header(self, header): - header.request_id = self.get_request_id() - - if not hasattr(ctx, "counter"): - ctx.counter = 0 - - ctx.counter += 1 - - if hasattr(ctx, "header"): - header.seq = "{prev_seq}.{cur_counter}".format( - prev_seq=ctx.header.seq, cur_counter=ctx.counter) - header.meta = ctx.header.meta - else: - header.meta = {} - header.seq = str(ctx.counter) - - if hasattr(ctx, "meta"): - header.meta.update(ctx.meta) - - def record(self, header, exception): - pass - - @classmethod - @contextlib.contextmanager - def counter(cls, init=0): - """Context for manually setting counter of seq number. - - :init: init value - """ - if not hasattr(ctx, "counter"): - ctx.counter = 0 - - old = ctx.counter - ctx.counter = init - - try: - yield - finally: - ctx.counter = old - - @classmethod - @contextlib.contextmanager - def annotate(cls, **kwargs): - ctx.annotation = kwargs - try: - yield ctx.annotation - finally: - del ctx.annotation - - @classmethod - @contextlib.contextmanager - def add_meta(cls, **kwds): - if hasattr(ctx, 'meta'): - old_dict = copy.copy(ctx.meta) - ctx.meta.update(kwds) - try: - yield ctx.meta - finally: - ctx.meta = old_dict - else: - ctx.meta = kwds - try: - yield ctx.meta - finally: - del ctx.meta - - @property - def meta(self): - meta = ctx.header.meta if hasattr(ctx, "header") else {} - if hasattr(ctx, "meta"): - meta.update(ctx.meta) - return meta - - @property - def annotation(self): - return ctx.annotation if hasattr(ctx, "annotation") else {} - - def get_request_id(self): - if hasattr(ctx, "header"): - return ctx.header.request_id - return str(uuid.uuid4()) - - def init_handshake_info(self, handshake_obj): - pass - - def handle_handshake_info(self, handshake_obj): - pass - - -class ConsoleTracker(TrackerBase): - def record(self, header, exception): - print(header) diff --git a/python/helpers/pydev/thriftpy/contrib/tracking/tracking.thrift b/python/helpers/pydev/thriftpy/contrib/tracking/tracking.thrift deleted file mode 100644 index f4aaebc7ae60..000000000000 --- a/python/helpers/pydev/thriftpy/contrib/tracking/tracking.thrift +++ /dev/null @@ -1,16 +0,0 @@ -/* - * This is the structure used to send call info to server. - */ -struct RequestHeader { - 1: string request_id - 2: string seq - 3: map meta -} - -/** - * This is the struct that a successful upgrade will reply with. - */ -struct UpgradeReply {} -struct UpgradeArgs { - 1: string app_id -} diff --git a/python/helpers/pydev/thriftpy/hook.py b/python/helpers/pydev/thriftpy/hook.py deleted file mode 100644 index d11b63169feb..000000000000 --- a/python/helpers/pydev/thriftpy/hook.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import sys - -from .parser import load_module - - -class ThriftImporter(object): - def __init__(self, extension="_thrift"): - self.extension = extension - - def __eq__(self, other): - return self.__class__.__module__ == other.__class__.__module__ and \ - self.__class__.__name__ == other.__class__.__name__ and \ - self.extension == other.extension - - def find_module(self, fullname, path=None): - if fullname.endswith(self.extension): - return self - - def load_module(self, fullname): - return load_module(fullname) -_imp = ThriftImporter() - - -def install_import_hook(): - global _imp - sys.meta_path[:] = [x for x in sys.meta_path if _imp != x] + [_imp] - - -def remove_import_hook(): - global _imp - sys.meta_path[:] = [x for x in sys.meta_path if _imp != x] diff --git a/python/helpers/pydev/thriftpy/parser/__init__.py b/python/helpers/pydev/thriftpy/parser/__init__.py deleted file mode 100644 index 7aa7e2e527a3..000000000000 --- a/python/helpers/pydev/thriftpy/parser/__init__.py +++ /dev/null @@ -1,77 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - thriftpy.parser - ~~~~~~~~~~~~~~~ - - Thrift parser using ply -""" - -from __future__ import absolute_import - -import os -import sys - -from .parser import parse, parse_fp - - -def load(path, module_name=None, include_dirs=None, include_dir=None): - """Load thrift file as a module. - - The module loaded and objects inside may only be pickled if module_name - was provided. - - Note: `include_dir` will be depreacated in the future, use `include_dirs` - instead. If `include_dir` was provided (not None), it will be appended to - `include_dirs`. - """ - real_module = bool(module_name) - thrift = parse(path, module_name, include_dirs=include_dirs, - include_dir=include_dir) - - if real_module: - sys.modules[module_name] = thrift - return thrift - - -def load_fp(source, module_name): - """Load thrift file like object as a module. - """ - thrift = parse_fp(source, module_name) - sys.modules[module_name] = thrift - return thrift - - -def _import_module(import_name): - if '.' in import_name: - module, obj = import_name.rsplit('.', 1) - return getattr(__import__(module, None, None, [obj]), obj) - else: - return __import__(import_name) - - -def load_module(fullname): - """Load thrift_file by fullname, fullname should have '_thrift' as - suffix. - The loader will replace the '_thrift' with '.thrift' and use it as - filename to locate the real thrift file. - """ - if not fullname.endswith("_thrift"): - raise ImportError( - "ThriftPy can only load module with '_thrift' suffix") - - if fullname in sys.modules: - return sys.modules[fullname] - - if '.' in fullname: - module_name, thrift_module_name = fullname.rsplit('.', 1) - module = _import_module(module_name) - path_prefix = os.path.dirname(os.path.abspath(module.__file__)) - path = os.path.join(path_prefix, thrift_module_name) - else: - path = fullname - thrift_file = "{0}.thrift".format(path[:-7]) - - module = load(thrift_file, module_name=fullname) - sys.modules[fullname] = module - return sys.modules[fullname] diff --git a/python/helpers/pydev/thriftpy/parser/exc.py b/python/helpers/pydev/thriftpy/parser/exc.py deleted file mode 100644 index 98d697cd1d81..000000000000 --- a/python/helpers/pydev/thriftpy/parser/exc.py +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - - -class ThriftParserError(Exception): - pass - - -class ThriftLexerError(ThriftParserError): - pass - - -class ThriftGrammerError(ThriftParserError): - pass diff --git a/python/helpers/pydev/thriftpy/parser/lexer.py b/python/helpers/pydev/thriftpy/parser/lexer.py deleted file mode 100644 index 1c6e71cec600..000000000000 --- a/python/helpers/pydev/thriftpy/parser/lexer.py +++ /dev/null @@ -1,257 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from .exc import ThriftLexerError - -literals = ':;,=*{}()<>[]' - - -thrift_reserved_keywords = ( - 'BEGIN', - 'END', - '__CLASS__', - '__DIR__', - '__FILE__', - '__FUNCTION__', - '__LINE__', - '__METHOD__', - '__NAMESPACE__', - 'abstract', - 'alias', - 'and', - 'args', - 'as', - 'assert', - 'begin', - 'break', - 'case', - 'catch', - 'class', - 'clone', - 'continue', - 'declare', - 'def', - 'default', - 'del', - 'delete', - 'do', - 'dynamic', - 'elif', - 'else', - 'elseif', - 'elsif', - 'end', - 'enddeclare', - 'endfor', - 'endforeach', - 'endif', - 'endswitch', - 'endwhile', - 'ensure', - 'except', - 'exec', - 'finally', - 'float', - 'for', - 'foreach', - 'function', - 'global', - 'goto', - 'if', - 'implements', - 'import', - 'in', - 'inline', - 'instanceof', - 'interface', - 'is', - 'lambda', - 'module', - 'native', - 'new', - 'next', - 'nil', - 'not', - 'or', - 'pass', - 'public', - 'print', - 'private', - 'protected', - 'public', - 'raise', - 'redo', - 'rescue', - 'retry', - 'register', - 'return', - 'self', - 'sizeof', - 'static', - 'super', - 'switch', - 'synchronized', - 'then', - 'this', - 'throw', - 'transient', - 'try', - 'undef', - 'union', - 'unless', - 'unsigned', - 'until', - 'use', - 'var', - 'virtual', - 'volatile', - 'when', - 'while', - 'with', - 'xor', - 'yield' -) - - -keywords = ( - 'namespace', - 'include', - 'void', - 'bool', - 'byte', - 'i16', - 'i32', - 'i64', - 'double', - 'string', - 'binary', - 'map', - 'list', - 'set', - 'oneway', - 'typedef', - 'struct', - 'union', - 'exception', - 'extends', - 'throws', - 'service', - 'enum', - 'const', - 'required', - 'optional', -) - - -tokens = ( - 'BOOLCONSTANT', - 'INTCONSTANT', - 'DUBCONSTANT', - 'LITERAL', - 'IDENTIFIER', -) + tuple(map(lambda kw: kw.upper(), keywords)) - - -t_ignore = ' \t\r' # whitespace - - -def t_error(t): - raise ThriftLexerError('Illegal characher %r at line %d' % - (t.value[0], t.lineno)) - - -def t_newline(t): - r'\n+' - t.lexer.lineno += len(t.value) - - -def t_ignore_SILLYCOMM(t): - r'\/\*\**\*\/' - t.lexer.lineno += t.value.count('\n') - - -def t_ignore_MULTICOMM(t): - r'\/\*[^*]\/*([^*/]|[^*]\/|\*[^/])*\**\*\/' - t.lexer.lineno += t.value.count('\n') - - -def t_ignore_DOCTEXT(t): - r'\/\*\*([^*/]|[^*]\/|\*[^/])*\**\*\/' - t.lexer.lineno += t.value.count('\n') - - -def t_ignore_UNIXCOMMENT(t): - r'\#[^\n]*' - - -def t_ignore_COMMENT(t): - r'\/\/[^\n]*' - - -def t_BOOLCONSTANT(t): - r'true|false' - t.value = t.value == 'true' - return t - - -def t_DUBCONSTANT(t): - r'-?\d+\.\d*(e-?\d+)?' - t.value = float(t.value) - return t - - -def t_HEXCONSTANT(t): - r'0x[0-9A-Fa-f]+' - t.value = int(t.value, 16) - t.type = 'INTCONSTANT' - return t - - -def t_INTCONSTANT(t): - r'[+-]?[0-9]+' - t.value = int(t.value) - return t - - -def t_LITERAL(t): - r'(\"([^\\\n]|(\\.))*?\")|\'([^\\\n]|(\\.))*?\'' - s = t.value[1:-1] - maps = { - 't': '\t', - 'r': '\r', - 'n': '\n', - '\\': '\\', - '\'': '\'', - '"': '\"' - } - i = 0 - length = len(s) - val = '' - while i < length: - if s[i] == '\\': - i += 1 - if s[i] in maps: - val += maps[s[i]] - else: - msg = 'Unexcepted escaping characher: %s' % s[i] - raise ThriftLexerError(msg) - else: - val += s[i] - - i += 1 - - t.value = val - return t - - -def t_IDENTIFIER(t): - r'[a-zA-Z_](\.[a-zA-Z_0-9]|[a-zA-Z_0-9])*' - - if t.value in keywords: - t.type = t.value.upper() - return t - if t.value in thrift_reserved_keywords: - raise ThriftLexerError('Cannot use reserved language keyword: %r' - ' at line %d' % (t.value, t.lineno)) - return t diff --git a/python/helpers/pydev/thriftpy/parser/parser.py b/python/helpers/pydev/thriftpy/parser/parser.py deleted file mode 100644 index f93c3058bb94..000000000000 --- a/python/helpers/pydev/thriftpy/parser/parser.py +++ /dev/null @@ -1,820 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -IDL Ref: - https://thrift.apache.org/docs/idl -""" - -from __future__ import absolute_import - -import collections -import os -import sys -import types -from ply import lex, yacc -from .lexer import * # noqa -from .exc import ThriftParserError, ThriftGrammerError -from ..thrift import gen_init, TType, TPayload, TException - - -def p_error(p): - if p is None: - raise ThriftGrammerError('Grammer error at EOF') - raise ThriftGrammerError('Grammer error %r at line %d' % - (p.value, p.lineno)) - - -def p_start(p): - '''start : header definition''' - - -def p_header(p): - '''header : header_unit_ header - |''' - - -def p_header_unit_(p): - '''header_unit_ : header_unit ';' - | header_unit''' - - -def p_header_unit(p): - '''header_unit : include - | namespace''' - - -def p_include(p): - '''include : INCLUDE LITERAL''' - thrift = thrift_stack[-1] - - if thrift.__thrift_file__ is None: - raise ThriftParserError('Unexcepted include statement while loading' - 'from file like object.') - - for include_dir in include_dirs_: - path = os.path.join(include_dir, p[2]) - if os.path.exists(path): - child = parse(path) - setattr(thrift, child.__name__, child) - _add_thrift_meta('includes', child) - return - raise ThriftParserError(('Couldn\'t include thrift %s in any ' - 'directories provided') % p[2]) - - -def p_namespace(p): - '''namespace : NAMESPACE namespace_scope IDENTIFIER''' - # namespace is useless in thriftpy - # if p[2] == 'py' or p[2] == '*': - # setattr(thrift_stack[-1], '__name__', p[3]) - - -def p_namespace_scope(p): - '''namespace_scope : '*' - | IDENTIFIER''' - p[0] = p[1] - - -def p_sep(p): - '''sep : ',' - | ';' - ''' - - -def p_definition(p): - '''definition : definition definition_unit_ - |''' - - -def p_definition_unit_(p): - '''definition_unit_ : definition_unit ';' - | definition_unit''' - - -def p_definition_unit(p): - '''definition_unit : const - | ttype - ''' - - -def p_const(p): - '''const : CONST field_type IDENTIFIER '=' const_value - | CONST field_type IDENTIFIER '=' const_value sep''' - - try: - val = _cast(p[2])(p[5]) - except AssertionError: - raise ThriftParserError('Type error for constant %s at line %d' % - (p[3], p.lineno(3))) - setattr(thrift_stack[-1], p[3], val) - _add_thrift_meta('consts', val) - - -def p_const_value(p): - '''const_value : INTCONSTANT - | DUBCONSTANT - | LITERAL - | BOOLCONSTANT - | const_list - | const_map - | const_ref''' - p[0] = p[1] - - -def p_const_list(p): - '''const_list : '[' const_list_seq ']' ''' - p[0] = p[2] - - -def p_const_list_seq(p): - '''const_list_seq : const_value sep const_list_seq - | const_value const_list_seq - |''' - _parse_seq(p) - - -def p_const_map(p): - '''const_map : '{' const_map_seq '}' ''' - p[0] = dict(p[2]) - - -def p_const_map_seq(p): - '''const_map_seq : const_map_item sep const_map_seq - | const_map_item const_map_seq - |''' - _parse_seq(p) - - -def p_const_map_item(p): - '''const_map_item : const_value ':' const_value ''' - p[0] = [p[1], p[3]] - - -def p_const_ref(p): - '''const_ref : IDENTIFIER''' - child = thrift_stack[-1] - - for name in p[1].split('.'): - father = child - child = getattr(child, name, None) - if child is None: - raise ThriftParserError('Cann\'t find name %r at line %d' - % (p[1], p.lineno(1))) - - if _get_ttype(child) is None or _get_ttype(father) == TType.I32: - # child is a constant or enum value - p[0] = child - else: - raise ThriftParserError('No enum value or constant found ' - 'named %r' % p[1]) - - -def p_ttype(p): - '''ttype : typedef - | enum - | struct - | union - | exception - | service''' - - -def p_typedef(p): - '''typedef : TYPEDEF field_type IDENTIFIER''' - setattr(thrift_stack[-1], p[3], p[2]) - - -def p_enum(p): # noqa - '''enum : ENUM IDENTIFIER '{' enum_seq '}' ''' - val = _make_enum(p[2], p[4]) - setattr(thrift_stack[-1], p[2], val) - _add_thrift_meta('enums', val) - - -def p_enum_seq(p): - '''enum_seq : enum_item sep enum_seq - | enum_item enum_seq - |''' - _parse_seq(p) - - -def p_enum_item(p): - '''enum_item : IDENTIFIER '=' INTCONSTANT - | IDENTIFIER - |''' - if len(p) == 4: - p[0] = [p[1], p[3]] - elif len(p) == 2: - p[0] = [p[1], None] - - -def p_struct(p): - '''struct : seen_struct '{' field_seq '}' ''' - val = _fill_in_struct(p[1], p[3]) - _add_thrift_meta('structs', val) - - -def p_seen_struct(p): - '''seen_struct : STRUCT IDENTIFIER ''' - val = _make_empty_struct(p[2]) - setattr(thrift_stack[-1], p[2], val) - p[0] = val - - -def p_union(p): - '''union : seen_union '{' field_seq '}' ''' - val = _fill_in_struct(p[1], p[3]) - _add_thrift_meta('unions', val) - - -def p_seen_union(p): - '''seen_union : UNION IDENTIFIER ''' - val = _make_empty_struct(p[2]) - setattr(thrift_stack[-1], p[2], val) - p[0] = val - - -def p_exception(p): - '''exception : EXCEPTION IDENTIFIER '{' field_seq '}' ''' - val = _make_struct(p[2], p[4], base_cls=TException) - setattr(thrift_stack[-1], p[2], val) - _add_thrift_meta('exceptions', val) - - -def p_service(p): - '''service : SERVICE IDENTIFIER '{' function_seq '}' - | SERVICE IDENTIFIER EXTENDS IDENTIFIER '{' function_seq '}' - ''' - thrift = thrift_stack[-1] - - if len(p) == 8: - extends = thrift - for name in p[4].split('.'): - extends = getattr(extends, name, None) - if extends is None: - raise ThriftParserError('Can\'t find service %r for ' - 'service %r to extend' % - (p[4], p[2])) - - if not hasattr(extends, 'thrift_services'): - raise ThriftParserError('Can\'t extends %r, not a service' - % p[4]) - - else: - extends = None - - val = _make_service(p[2], p[len(p) - 2], extends) - setattr(thrift, p[2], val) - _add_thrift_meta('services', val) - - -def p_function(p): - '''function : ONEWAY function_type IDENTIFIER '(' field_seq ')' throws - | ONEWAY function_type IDENTIFIER '(' field_seq ')' - | function_type IDENTIFIER '(' field_seq ')' throws - | function_type IDENTIFIER '(' field_seq ')' ''' - - if p[1] == 'oneway': - oneway = True - base = 1 - else: - oneway = False - base = 0 - - if p[len(p) - 1] == ')': - throws = [] - else: - throws = p[len(p) - 1] - - p[0] = [oneway, p[base + 1], p[base + 2], p[base + 4], throws] - - -def p_function_seq(p): - '''function_seq : function sep function_seq - | function function_seq - |''' - _parse_seq(p) - - -def p_throws(p): - '''throws : THROWS '(' field_seq ')' ''' - p[0] = p[3] - - -def p_function_type(p): - '''function_type : field_type - | VOID''' - if p[1] == 'void': - p[0] = TType.VOID - else: - p[0] = p[1] - - -def p_field_seq(p): - '''field_seq : field sep field_seq - | field field_seq - |''' - _parse_seq(p) - - -def p_field(p): - '''field : field_id field_req field_type IDENTIFIER - | field_id field_req field_type IDENTIFIER '=' const_value''' - - if len(p) == 7: - try: - val = _cast(p[3])(p[6]) - except AssertionError: - raise ThriftParserError( - 'Type error for field %s ' - 'at line %d' % (p[4], p.lineno(4))) - else: - val = None - - p[0] = [p[1], p[2], p[3], p[4], val] - - -def p_field_id(p): - '''field_id : INTCONSTANT ':' ''' - p[0] = p[1] - - -def p_field_req(p): - '''field_req : REQUIRED - | OPTIONAL - |''' - if len(p) == 2: - p[0] = p[1] == 'required' - elif len(p) == 1: - p[0] = False # default: required=False - - -def p_field_type(p): - '''field_type : ref_type - | definition_type''' - p[0] = p[1] - - -def p_ref_type(p): - '''ref_type : IDENTIFIER''' - ref_type = thrift_stack[-1] - - for name in p[1].split('.'): - ref_type = getattr(ref_type, name, None) - if ref_type is None: - raise ThriftParserError('No type found: %r, at line %d' % - (p[1], p.lineno(1))) - - if hasattr(ref_type, '_ttype'): - p[0] = getattr(ref_type, '_ttype'), ref_type - else: - p[0] = ref_type - - -def p_base_type(p): # noqa - '''base_type : BOOL - | BYTE - | I16 - | I32 - | I64 - | DOUBLE - | STRING - | BINARY''' - if p[1] == 'bool': - p[0] = TType.BOOL - if p[1] == 'byte': - p[0] = TType.BYTE - if p[1] == 'i16': - p[0] = TType.I16 - if p[1] == 'i32': - p[0] = TType.I32 - if p[1] == 'i64': - p[0] = TType.I64 - if p[1] == 'double': - p[0] = TType.DOUBLE - if p[1] == 'string': - p[0] = TType.STRING - if p[1] == 'binary': - p[0] = TType.BINARY - - -def p_container_type(p): - '''container_type : map_type - | list_type - | set_type''' - p[0] = p[1] - - -def p_map_type(p): - '''map_type : MAP '<' field_type ',' field_type '>' ''' - p[0] = TType.MAP, (p[3], p[5]) - - -def p_list_type(p): - '''list_type : LIST '<' field_type '>' ''' - p[0] = TType.LIST, p[3] - - -def p_set_type(p): - '''set_type : SET '<' field_type '>' ''' - p[0] = TType.SET, p[3] - - -def p_definition_type(p): - '''definition_type : base_type - | container_type''' - p[0] = p[1] - - -thrift_stack = [] -include_dirs_ = ['.'] -thrift_cache = {} - - -def parse(path, module_name=None, include_dirs=None, include_dir=None, - lexer=None, parser=None, enable_cache=True): - """Parse a single thrift file to module object, e.g.:: - - >>> from thriftpy.parser.parser import parse - >>> note_thrift = parse("path/to/note.thrift") - - - :param path: file path to parse, should be a string ending with '.thrift'. - :param module_name: the name for parsed module, the default is the basename - without extension of `path`. - :param include_dirs: directories to find thrift files while processing - the `include` directive, by default: ['.']. - :param include_dir: directory to find child thrift files. Note this keyword - parameter will be deprecated in the future, it exists - for compatiable reason. If it's provided (not `None`), - it will be appended to `include_dirs`. - :param lexer: ply lexer to use, if not provided, `parse` will new one. - :param parser: ply parser to use, if not provided, `parse` will new one. - :param enable_cache: if this is set to be `True`, parsed module will be - cached, this is enabled by default. If `module_name` - is provided, use it as cache key, else use the `path`. - """ - if os.name == 'nt' and sys.version_info < (3, 2): - os.path.samefile = lambda f1, f2: os.stat(f1) == os.stat(f2) - - # dead include checking on current stack - for thrift in thrift_stack: - if thrift.__thrift_file__ is not None and \ - os.path.samefile(path, thrift.__thrift_file__): - raise ThriftParserError('Dead including on %s' % path) - - global thrift_cache - - cache_key = module_name or os.path.normpath(path) - - if enable_cache and cache_key in thrift_cache: - return thrift_cache[cache_key] - - if lexer is None: - lexer = lex.lex() - if parser is None: - parser = yacc.yacc(debug=False, write_tables=0) - - global include_dirs_ - - if include_dirs is not None: - include_dirs_ = include_dirs - if include_dir is not None: - include_dirs_.append(include_dir) - - if not path.endswith('.thrift'): - raise ThriftParserError('Path should end with .thrift') - - with open(path) as fh: - data = fh.read() - - if module_name is not None and not module_name.endswith('_thrift'): - raise ThriftParserError('ThriftPy can only generate module with ' - '\'_thrift\' suffix') - - if module_name is None: - basename = os.path.basename(path) - module_name = os.path.splitext(basename)[0] - - thrift = types.ModuleType(module_name) - setattr(thrift, '__thrift_file__', path) - thrift_stack.append(thrift) - lexer.lineno = 1 - parser.parse(data) - thrift_stack.pop() - - if enable_cache: - thrift_cache[cache_key] = thrift - return thrift - - -def parse_fp(source, module_name, lexer=None, parser=None, enable_cache=True): - """Parse a file-like object to thrift module object, e.g.:: - - >>> from thriftpy.parser.parser import parse_fp - >>> with open("path/to/note.thrift") as fp: - parse_fp(fp, "note_thrift") - - - :param source: file-like object, expected to have a method named `read`. - :param module_name: the name for parsed module, shoule be endswith - '_thrift'. - :param lexer: ply lexer to use, if not provided, `parse` will new one. - :param parser: ply parser to use, if not provided, `parse` will new one. - :param enable_cache: if this is set to be `True`, parsed module will be - cached by `module_name`, this is enabled by default. - """ - if not module_name.endswith('_thrift'): - raise ThriftParserError('ThriftPy can only generate module with ' - '\'_thrift\' suffix') - - if enable_cache and module_name in thrift_cache: - return thrift_cache[module_name] - - if not hasattr(source, 'read'): - raise ThriftParserError('Except `source` to be a file-like object with' - 'a method named \'read\'') - - if lexer is None: - lexer = lex.lex() - if parser is None: - parser = yacc.yacc(debug=False, write_tables=0) - - data = source.read() - - thrift = types.ModuleType(module_name) - setattr(thrift, '__thrift_file__', None) - thrift_stack.append(thrift) - lexer.lineno = 1 - parser.parse(data) - thrift_stack.pop() - - if enable_cache: - thrift_cache[module_name] = thrift - return thrift - - -def _add_thrift_meta(key, val): - thrift = thrift_stack[-1] - - if not hasattr(thrift, '__thrift_meta__'): - meta = collections.defaultdict(list) - setattr(thrift, '__thrift_meta__', meta) - else: - meta = getattr(thrift, '__thrift_meta__') - - meta[key].append(val) - - -def _parse_seq(p): - if len(p) == 4: - p[0] = [p[1]] + p[3] - elif len(p) == 3: - p[0] = [p[1]] + p[2] - elif len(p) == 1: - p[0] = [] - - -def _cast(t): # noqa - if t == TType.BOOL: - return _cast_bool - if t == TType.BYTE: - return _cast_byte - if t == TType.I16: - return _cast_i16 - if t == TType.I32: - return _cast_i32 - if t == TType.I64: - return _cast_i64 - if t == TType.DOUBLE: - return _cast_double - if t == TType.STRING: - return _cast_string - if t == TType.BINARY: - return _cast_binary - if t[0] == TType.LIST: - return _cast_list(t) - if t[0] == TType.SET: - return _cast_set(t) - if t[0] == TType.MAP: - return _cast_map(t) - if t[0] == TType.I32: - return _cast_enum(t) - if t[0] == TType.STRUCT: - return _cast_struct(t) - - -def _cast_bool(v): - assert isinstance(v, (bool, int)) - return bool(v) - - -def _cast_byte(v): - assert isinstance(v, str) - return v - - -def _cast_i16(v): - assert isinstance(v, int) - return v - - -def _cast_i32(v): - assert isinstance(v, int) - return v - - -def _cast_i64(v): - assert isinstance(v, int) - return v - - -def _cast_double(v): - assert isinstance(v, float) - return v - - -def _cast_string(v): - assert isinstance(v, str) - return v - - -def _cast_binary(v): - assert isinstance(v, str) - return v - - -def _cast_list(t): - assert t[0] == TType.LIST - - def __cast_list(v): - assert isinstance(v, list) - map(_cast(t[1]), v) - return v - return __cast_list - - -def _cast_set(t): - assert t[0] == TType.SET - - def __cast_set(v): - assert isinstance(v, (list, set)) - map(_cast(t[1]), v) - if not isinstance(v, set): - return set(v) - return v - return __cast_set - - -def _cast_map(t): - assert t[0] == TType.MAP - - def __cast_map(v): - assert isinstance(v, dict) - for key in v: - v[_cast(t[1][0])(key)] = \ - _cast(t[1][1])(v[key]) - return v - return __cast_map - - -def _cast_enum(t): - assert t[0] == TType.I32 - - def __cast_enum(v): - assert isinstance(v, int) - if v in t[1]._VALUES_TO_NAMES: - return v - raise ThriftParserError('Couldn\'t find a named value in enum ' - '%s for value %d' % (t[1].__name__, v)) - return __cast_enum - - -def _cast_struct(t): # struct/exception/union - assert t[0] == TType.STRUCT - - def __cast_struct(v): - if isinstance(v, t[1]): - return v # already cast - - assert isinstance(v, dict) - tspec = getattr(t[1], '_tspec') - - for key in tspec: # requirement check - if tspec[key][0] and key not in v: - raise ThriftParserError('Field %r was required to create ' - 'constant for type %r' % - (key, t[1].__name__)) - - for key in v: # cast values - if key not in tspec: - raise ThriftParserError('No field named %r was ' - 'found in struct of type %r' % - (key, t[1].__name__)) - v[key] = _cast(tspec[key][1])(v[key]) - return t[1](**v) - return __cast_struct - - -def _make_enum(name, kvs): - attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': TType.I32} - cls = type(name, (object, ), attrs) - - _values_to_names = {} - _names_to_values = {} - - if kvs: - val = kvs[0][1] - if val is None: - val = -1 - for item in kvs: - if item[1] is None: - item[1] = val + 1 - val = item[1] - for key, val in kvs: - setattr(cls, key, val) - _values_to_names[val] = key - _names_to_values[key] = val - setattr(cls, '_VALUES_TO_NAMES', _values_to_names) - setattr(cls, '_NAMES_TO_VALUES', _names_to_values) - return cls - - -def _make_empty_struct(name, ttype=TType.STRUCT, base_cls=TPayload): - attrs = {'__module__': thrift_stack[-1].__name__, '_ttype': ttype} - return type(name, (base_cls, ), attrs) - - -def _fill_in_struct(cls, fields, _gen_init=True): - thrift_spec = {} - default_spec = [] - _tspec = {} - - for field in fields: - if field[0] in thrift_spec or field[3] in _tspec: - raise ThriftGrammerError(('\'%d:%s\' field identifier/name has ' - 'already been used') % ( - field[0], field[3])) - ttype = field[2] - thrift_spec[field[0]] = _ttype_spec(ttype, field[3], field[1]) - default_spec.append((field[3], field[4])) - _tspec[field[3]] = field[1], ttype - setattr(cls, 'thrift_spec', thrift_spec) - setattr(cls, 'default_spec', default_spec) - setattr(cls, '_tspec', _tspec) - if _gen_init: - gen_init(cls, thrift_spec, default_spec) - return cls - - -def _make_struct(name, fields, ttype=TType.STRUCT, base_cls=TPayload, - _gen_init=True): - cls = _make_empty_struct(name, ttype=ttype, base_cls=base_cls) - return _fill_in_struct(cls, fields, _gen_init=_gen_init) - - -def _make_service(name, funcs, extends): - if extends is None: - extends = object - - attrs = {'__module__': thrift_stack[-1].__name__} - cls = type(name, (extends, ), attrs) - thrift_services = [] - - for func in funcs: - func_name = func[2] - # args payload cls - args_name = '%s_args' % func_name - args_fields = func[3] - args_cls = _make_struct(args_name, args_fields) - setattr(cls, args_name, args_cls) - # result payload cls - result_name = '%s_result' % func_name - result_type = func[1] - result_throws = func[4] - result_oneway = func[0] - result_cls = _make_struct(result_name, result_throws, - _gen_init=False) - setattr(result_cls, 'oneway', result_oneway) - if result_type != TType.VOID: - result_cls.thrift_spec[0] = _ttype_spec(result_type, 'success') - result_cls.default_spec.insert(0, ('success', None)) - gen_init(result_cls, result_cls.thrift_spec, result_cls.default_spec) - setattr(cls, result_name, result_cls) - thrift_services.append(func_name) - if extends is not None and hasattr(extends, 'thrift_services'): - thrift_services.extend(extends.thrift_services) - setattr(cls, 'thrift_services', thrift_services) - return cls - - -def _ttype_spec(ttype, name, required=False): - if isinstance(ttype, int): - return ttype, name, required - else: - return ttype[0], name, ttype[1], required - - -def _get_ttype(inst, default_ttype=None): - if hasattr(inst, '__dict__') and '_ttype' in inst.__dict__: - return inst.__dict__['_ttype'] - return default_ttype diff --git a/python/helpers/pydev/thriftpy/protocol/__init__.py b/python/helpers/pydev/thriftpy/protocol/__init__.py deleted file mode 100644 index 3ff89af8c39c..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from thriftpy._compat import PYPY, CYTHON -from .binary import TBinaryProtocol, TBinaryProtocolFactory -from .compact import TCompactProtocol, TCompactProtocolFactory -from .json import TJSONProtocol, TJSONProtocolFactory -from .multiplex import TMultiplexedProtocol, TMultiplexedProtocolFactory - -if not PYPY: - # enable cython binary by default for CPython. - if CYTHON: - from .cybin import TCyBinaryProtocol, TCyBinaryProtocolFactory - TBinaryProtocol = TCyBinaryProtocol # noqa - TBinaryProtocolFactory = TCyBinaryProtocolFactory # noqa -else: - # disable cython binary protocol for PYPY since it's slower. - TCyBinaryProtocol = TBinaryProtocol - TCyBinaryProtocolFactory = TBinaryProtocolFactory - -__all__ = ['TBinaryProtocol', 'TBinaryProtocolFactory', - 'TCyBinaryProtocol', 'TCyBinaryProtocolFactory', - 'TJSONProtocol', 'TJSONProtocolFactory', - 'TMultiplexedProtocol', 'TMultiplexedProtocolFactory', - 'TCompactProtocol', 'TCompactProtocolFactory'] diff --git a/python/helpers/pydev/thriftpy/protocol/binary.py b/python/helpers/pydev/thriftpy/protocol/binary.py deleted file mode 100644 index 036f076e7efa..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/binary.py +++ /dev/null @@ -1,401 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import struct - -from .exc import TProtocolException -from ..thrift import TType - -# VERSION_MASK = 0xffff0000 -VERSION_MASK = -65536 -# VERSION_1 = 0x80010000 -VERSION_1 = -2147418112 -TYPE_MASK = 0x000000ff - - -def pack_i8(byte): - return struct.pack("!b", byte) - - -def pack_i16(i16): - return struct.pack("!h", i16) - - -def pack_i32(i32): - return struct.pack("!i", i32) - - -def pack_i64(i64): - return struct.pack("!q", i64) - - -def pack_double(dub): - return struct.pack("!d", dub) - - -def pack_string(string): - return struct.pack("!i%ds" % len(string), len(string), string) - - -def unpack_i8(buf): - return struct.unpack("!b", buf)[0] - - -def unpack_i16(buf): - return struct.unpack("!h", buf)[0] - - -def unpack_i32(buf): - return struct.unpack("!i", buf)[0] - - -def unpack_i64(buf): - return struct.unpack("!q", buf)[0] - - -def unpack_double(buf): - return struct.unpack("!d", buf)[0] - - -def write_message_begin(outbuf, name, ttype, seqid, strict=True): - if strict: - outbuf.write(pack_i32(VERSION_1 | ttype)) - outbuf.write(pack_string(name.encode('utf-8'))) - else: - outbuf.write(pack_string(name.encode('utf-8'))) - outbuf.write(pack_i8(ttype)) - - outbuf.write(pack_i32(seqid)) - - -def write_field_begin(outbuf, ttype, fid): - outbuf.write(pack_i8(ttype) + pack_i16(fid)) - - -def write_field_stop(outbuf): - outbuf.write(pack_i8(TType.STOP)) - - -def write_list_begin(outbuf, etype, size): - outbuf.write(pack_i8(etype) + pack_i32(size)) - - -def write_map_begin(outbuf, ktype, vtype, size): - outbuf.write(pack_i8(ktype) + pack_i8(vtype) + pack_i32(size)) - - -def write_val(outbuf, ttype, val, spec=None): - if ttype == TType.BOOL: - if val: - outbuf.write(pack_i8(1)) - else: - outbuf.write(pack_i8(0)) - - elif ttype == TType.BYTE: - outbuf.write(pack_i8(val)) - - elif ttype == TType.I16: - outbuf.write(pack_i16(val)) - - elif ttype == TType.I32: - outbuf.write(pack_i32(val)) - - elif ttype == TType.I64: - outbuf.write(pack_i64(val)) - - elif ttype == TType.DOUBLE: - outbuf.write(pack_double(val)) - - elif ttype == TType.STRING: - if not isinstance(val, bytes): - val = val.encode('utf-8') - outbuf.write(pack_string(val)) - - elif ttype == TType.SET or ttype == TType.LIST: - if isinstance(spec, tuple): - e_type, t_spec = spec[0], spec[1] - else: - e_type, t_spec = spec, None - - val_len = len(val) - write_list_begin(outbuf, e_type, val_len) - for e_val in val: - write_val(outbuf, e_type, e_val, t_spec) - - elif ttype == TType.MAP: - if isinstance(spec[0], int): - k_type = spec[0] - k_spec = None - else: - k_type, k_spec = spec[0] - - if isinstance(spec[1], int): - v_type = spec[1] - v_spec = None - else: - v_type, v_spec = spec[1] - - write_map_begin(outbuf, k_type, v_type, len(val)) - for k in iter(val): - write_val(outbuf, k_type, k, k_spec) - write_val(outbuf, v_type, val[k], v_spec) - - elif ttype == TType.STRUCT: - for fid in iter(val.thrift_spec): - f_spec = val.thrift_spec[fid] - if len(f_spec) == 3: - f_type, f_name, f_req = f_spec - f_container_spec = None - else: - f_type, f_name, f_container_spec, f_req = f_spec - - v = getattr(val, f_name) - if v is None: - continue - - write_field_begin(outbuf, f_type, fid) - write_val(outbuf, f_type, v, f_container_spec) - write_field_stop(outbuf) - - -def read_message_begin(inbuf, strict=True): - sz = unpack_i32(inbuf.read(4)) - if sz < 0: - version = sz & VERSION_MASK - if version != VERSION_1: - raise TProtocolException( - type=TProtocolException.BAD_VERSION, - message='Bad version in read_message_begin: %d' % (sz)) - name_sz = unpack_i32(inbuf.read(4)) - name = inbuf.read(name_sz).decode('utf-8') - - type_ = sz & TYPE_MASK - else: - if strict: - raise TProtocolException(type=TProtocolException.BAD_VERSION, - message='No protocol version header') - - name = inbuf.read(sz).decode('utf-8') - type_ = unpack_i8(inbuf.read(1)) - - seqid = unpack_i32(inbuf.read(4)) - - return name, type_, seqid - - -def read_field_begin(inbuf): - f_type = unpack_i8(inbuf.read(1)) - if f_type == TType.STOP: - return f_type, 0 - - return f_type, unpack_i16(inbuf.read(2)) - - -def read_list_begin(inbuf): - e_type = unpack_i8(inbuf.read(1)) - sz = unpack_i32(inbuf.read(4)) - return e_type, sz - - -def read_map_begin(inbuf): - k_type, v_type = unpack_i8(inbuf.read(1)), unpack_i8(inbuf.read(1)) - sz = unpack_i32(inbuf.read(4)) - return k_type, v_type, sz - - -def read_val(inbuf, ttype, spec=None, decode_response=True): - if ttype == TType.BOOL: - return bool(unpack_i8(inbuf.read(1))) - - elif ttype == TType.BYTE: - return unpack_i8(inbuf.read(1)) - - elif ttype == TType.I16: - return unpack_i16(inbuf.read(2)) - - elif ttype == TType.I32: - return unpack_i32(inbuf.read(4)) - - elif ttype == TType.I64: - return unpack_i64(inbuf.read(8)) - - elif ttype == TType.DOUBLE: - return unpack_double(inbuf.read(8)) - - elif ttype == TType.STRING: - sz = unpack_i32(inbuf.read(4)) - byte_payload = inbuf.read(sz) - - # Since we cannot tell if we're getting STRING or BINARY - # if not asked not to decode, try both - if decode_response: - try: - return byte_payload.decode('utf-8') - except UnicodeDecodeError: - pass - return byte_payload - - elif ttype == TType.SET or ttype == TType.LIST: - if isinstance(spec, tuple): - v_type, v_spec = spec[0], spec[1] - else: - v_type, v_spec = spec, None - - result = [] - r_type, sz = read_list_begin(inbuf) - # the v_type is useless here since we already get it from spec - if r_type != v_type: - for _ in range(sz): - skip(inbuf, r_type) - return [] - - for i in range(sz): - result.append(read_val(inbuf, v_type, v_spec, decode_response)) - return result - - elif ttype == TType.MAP: - if isinstance(spec[0], int): - k_type = spec[0] - k_spec = None - else: - k_type, k_spec = spec[0] - - if isinstance(spec[1], int): - v_type = spec[1] - v_spec = None - else: - v_type, v_spec = spec[1] - - result = {} - sk_type, sv_type, sz = read_map_begin(inbuf) - if sk_type != k_type or sv_type != v_type: - for _ in range(sz): - skip(inbuf, sk_type) - skip(inbuf, sv_type) - return {} - - for i in range(sz): - k_val = read_val(inbuf, k_type, k_spec, decode_response) - v_val = read_val(inbuf, v_type, v_spec, decode_response) - result[k_val] = v_val - - return result - - elif ttype == TType.STRUCT: - obj = spec() - read_struct(inbuf, obj, decode_response) - return obj - - -def read_struct(inbuf, obj, decode_response=True): - while True: - f_type, fid = read_field_begin(inbuf) - if f_type == TType.STOP: - break - - if fid not in obj.thrift_spec: - skip(inbuf, f_type) - continue - - if len(obj.thrift_spec[fid]) == 3: - sf_type, f_name, f_req = obj.thrift_spec[fid] - f_container_spec = None - else: - sf_type, f_name, f_container_spec, f_req = obj.thrift_spec[fid] - - # it really should equal here. but since we already wasted - # space storing the duplicate info, let's check it. - if f_type != sf_type: - skip(inbuf, f_type) - continue - - setattr(obj, f_name, - read_val(inbuf, f_type, f_container_spec, decode_response)) - - -def skip(inbuf, ftype): - if ftype == TType.BOOL or ftype == TType.BYTE: - inbuf.read(1) - - elif ftype == TType.I16: - inbuf.read(2) - - elif ftype == TType.I32: - inbuf.read(4) - - elif ftype == TType.I64: - inbuf.read(8) - - elif ftype == TType.DOUBLE: - inbuf.read(8) - - elif ftype == TType.STRING: - inbuf.read(unpack_i32(inbuf.read(4))) - - elif ftype == TType.SET or ftype == TType.LIST: - v_type, sz = read_list_begin(inbuf) - for i in range(sz): - skip(inbuf, v_type) - - elif ftype == TType.MAP: - k_type, v_type, sz = read_map_begin(inbuf) - for i in range(sz): - skip(inbuf, k_type) - skip(inbuf, v_type) - - elif ftype == TType.STRUCT: - while True: - f_type, fid = read_field_begin(inbuf) - if f_type == TType.STOP: - break - skip(inbuf, f_type) - - -class TBinaryProtocol(object): - """Binary implementation of the Thrift protocol driver.""" - - def __init__(self, trans, - strict_read=True, strict_write=True, - decode_response=True): - self.trans = trans - self.strict_read = strict_read - self.strict_write = strict_write - self.decode_response = decode_response - - def skip(self, ttype): - skip(self.trans, ttype) - - def read_message_begin(self): - api, ttype, seqid = read_message_begin( - self.trans, strict=self.strict_read) - return api, ttype, seqid - - def read_message_end(self): - pass - - def write_message_begin(self, name, ttype, seqid): - write_message_begin(self.trans, name, ttype, seqid, - strict=self.strict_write) - - def write_message_end(self): - pass - - def read_struct(self, obj): - return read_struct(self.trans, obj, self.decode_response) - - def write_struct(self, obj): - write_val(self.trans, TType.STRUCT, obj) - - -class TBinaryProtocolFactory(object): - def __init__(self, strict_read=True, strict_write=True, - decode_response=True): - self.strict_read = strict_read - self.strict_write = strict_write - self.decode_response = decode_response - - def get_protocol(self, trans): - return TBinaryProtocol(trans, - self.strict_read, self.strict_write, - self.decode_response) diff --git a/python/helpers/pydev/thriftpy/protocol/compact.py b/python/helpers/pydev/thriftpy/protocol/compact.py deleted file mode 100644 index 47e33227e27b..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/compact.py +++ /dev/null @@ -1,565 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import array -from struct import pack, unpack - -from thriftpy._compat import PY3 -from .exc import TProtocolException -from ..thrift import TException -from ..thrift import TType - -CLEAR = 0 -FIELD_WRITE = 1 -VALUE_WRITE = 2 -CONTAINER_WRITE = 3 -BOOL_WRITE = 4 -FIELD_READ = 5 -CONTAINER_READ = 6 -VALUE_READ = 7 -BOOL_READ = 8 - - -def check_integer_limits(i, bits): - if bits == 8 and (i < -128 or i > 127): - raise TProtocolException(TProtocolException.INVALID_DATA, - "i8 requires -128 <= number <= 127") - elif bits == 16 and (i < -32768 or i > 32767): - raise TProtocolException(TProtocolException.INVALID_DATA, - "i16 requires -32768 <= number <= 32767") - elif bits == 32 and (i < -2147483648 or i > 2147483647): - raise TProtocolException( - TProtocolException.INVALID_DATA, - "i32 requires -2147483648 <= number <= 2147483647") - elif bits == 64 and (i < -9223372036854775808 or i > 9223372036854775807): - raise TProtocolException( - TProtocolException.INVALID_DATA, - "i64 requires -9223372036854775808 <= number <= \ - 9223372036854775807") - - -def make_zig_zag(n, bits): - check_integer_limits(n, bits) - return (n << 1) ^ (n >> (bits - 1)) - - -def from_zig_zag(n): - return (n >> 1) ^ -(n & 1) - - -def write_varint(trans, n): - out = [] - while True: - if n & ~0x7f == 0: - out.append(n) - break - else: - out.append((n & 0xff) | 0x80) - n = n >> 7 - data = array.array('B', out).tostring() - - if PY3: - trans.write(data) - else: - trans.write(bytes(data)) - - -def read_varint(trans): - result = 0 - shift = 0 - - while True: - x = trans.read(1) - byte = ord(x) - result |= (byte & 0x7f) << shift - if byte >> 7 == 0: - return result - shift += 7 - - -class CompactType(object): - STOP = 0x00 - TRUE = 0x01 - FALSE = 0x02 - BYTE = 0x03 - I16 = 0x04 - I32 = 0x05 - I64 = 0x06 - DOUBLE = 0x07 - BINARY = 0x08 - LIST = 0x09 - SET = 0x0A - MAP = 0x0B - STRUCT = 0x0C - - -CTYPES = { - TType.STOP: CompactType.STOP, - TType.BOOL: CompactType.TRUE, - TType.BYTE: CompactType.BYTE, - TType.I16: CompactType.I16, - TType.I32: CompactType.I32, - TType.I64: CompactType.I64, - TType.DOUBLE: CompactType.DOUBLE, - TType.STRING: CompactType.BINARY, - TType.STRUCT: CompactType.STRUCT, - TType.LIST: CompactType.LIST, - TType.SET: CompactType.SET, - TType.MAP: CompactType.MAP -} -TTYPES = dict((v, k) for k, v in CTYPES.items()) -TTYPES[CompactType.FALSE] = TType.BOOL - - -class TCompactProtocol(object): - """Compact implementation of the Thrift protocol driver.""" - PROTOCOL_ID = 0x82 - VERSION = 1 - VERSION_MASK = 0x1f - TYPE_MASK = 0xe0 - TYPE_BITS = 0x07 - TYPE_SHIFT_AMOUNT = 5 - - def __init__(self, trans, decode_response=True): - self.trans = trans - self._last_fid = 0 - self._bool_fid = None - self._bool_value = None - self._structs = [] - self.decode_response = decode_response - - def _get_ttype(self, byte): - return TTYPES[byte & 0x0f] - - def _read_size(self): - result = read_varint(self.trans) - if result < 0: - raise TException("Length < 0") - return result - - def read_message_begin(self): - proto_id = self.read_ubyte() - if proto_id != self.PROTOCOL_ID: - raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad protocol id in the message: %d' - % proto_id) - - ver_type = self.read_ubyte() - type = (ver_type >> self.TYPE_SHIFT_AMOUNT) & self.TYPE_BITS - version = ver_type & self.VERSION_MASK - if version != self.VERSION: - raise TProtocolException(TProtocolException.BAD_VERSION, - 'Bad version: %d (expect %d)' - % (version, self.VERSION)) - seqid = read_varint(self.trans) - name = self.read_string() - return name, type, seqid - - def read_message_end(self): - assert len(self._structs) == 0 - - def read_field_begin(self): - type = self.read_ubyte() - if type & 0x0f == TType.STOP: - return None, 0, 0 - - delta = type >> 4 - if delta == 0: - fid = from_zig_zag(read_varint(self.trans)) - else: - fid = self._last_fid + delta - self._last_fid = fid - - type = type & 0x0f - if type == CompactType.TRUE: - self._bool_value = True - elif type == CompactType.FALSE: - self._bool_value = False - - return None, self._get_ttype(type), fid - - def read_field_end(self): - pass - - def read_struct_begin(self): - self._structs.append(self._last_fid) - self._last_fid = 0 - - def read_struct_end(self): - self._last_fid = self._structs.pop() - - def read_map_begin(self): - size = self._read_size() - types = 0 - if size > 0: - types = self.read_ubyte() - vtype = self._get_ttype(types) - ktype = self._get_ttype(types >> 4) - return (ktype, vtype, size) - - def read_collection_begin(self): - size_type = self.read_ubyte() - size = size_type >> 4 - type = self._get_ttype(size_type) - if size == 15: - size = self._read_size() - return type, size - - def read_collection_end(self): - pass - - def read_byte(self): - result, = unpack('!b', self.trans.read(1)) - return result - - def read_ubyte(self): - result, = unpack('!B', self.trans.read(1)) - return result - - def read_int(self): - return from_zig_zag(read_varint(self.trans)) - - def read_double(self): - buff = self.trans.read(8) - val, = unpack(' self._last_fid \ - and self._bool_fid - self._last_fid <= 15: - if bool: - ctype = CompactType.TRUE - else: - ctype = CompactType.FALSE - self._write_field_header(ctype, self._bool_fid) - else: - if bool: - self.write_byte(CompactType.TRUE) - else: - self.write_byte(CompactType.FALSE) - - def write_i16(self, i16): - write_varint(self.trans, make_zig_zag(i16, 16)) - - def write_i32(self, i32): - write_varint(self.trans, make_zig_zag(i32, 32)) - - def write_i64(self, i64): - write_varint(self.trans, make_zig_zag(i64, 64)) - - def write_double(self, dub): - self.trans.write(pack('data)[0]) - - -cdef inline int32_t read_i32(CyTransportBase buf) except? -1: - cdef char data[4] - buf.c_read(4, data) - return be32toh((data)[0]) - - -cdef inline int64_t read_i64(CyTransportBase buf) except? -1: - cdef char data[8] - buf.c_read(8, data) - return be64toh((data)[0]) - - -cdef inline int write_i08(CyTransportBase buf, char val) except -1: - buf.c_write(&val, 1) - return 0 - - -cdef inline int write_i16(CyTransportBase buf, int16_t val) except -1: - val = htobe16(val) - buf.c_write((&val), 2) - return 0 - - -cdef inline int write_i32(CyTransportBase buf, int32_t val) except -1: - val = htobe32(val) - buf.c_write((&val), 4) - return 0 - - -cdef inline int write_i64(CyTransportBase buf, int64_t val) except -1: - val = htobe64(val) - buf.c_write((&val), 8) - return 0 - - -cdef inline int write_double(CyTransportBase buf, double val) except -1: - cdef int64_t v = htobe64(((&val))[0]) - buf.c_write((&v), 8) - return 0 - - -cdef inline write_list(CyTransportBase buf, object val, spec): - cdef TType e_type - cdef int val_len - - if isinstance(spec, int): - e_type = spec - e_spec = None - else: - e_type = spec[0] - e_spec = spec[1] - - val_len = len(val) - write_i08(buf, e_type) - write_i32(buf, val_len) - - for e_val in val: - c_write_val(buf, e_type, e_val, e_spec) - - -cdef inline write_string(CyTransportBase buf, bytes val): - cdef int val_len = len(val) - write_i32(buf, val_len) - - buf.c_write(val, val_len) - - -cdef inline write_dict(CyTransportBase buf, object val, spec): - cdef int val_len - cdef TType v_type, k_type - - key = spec[0] - if isinstance(key, int): - k_type = key - k_spec = None - else: - k_type = key[0] - k_spec = key[1] - - value = spec[1] - if isinstance(value, int): - v_type = value - v_spec = None - else: - v_type = value[0] - v_spec = value[1] - - val_len = len(val) - - write_i08(buf, k_type) - write_i08(buf, v_type) - write_i32(buf, val_len) - - for k, v in val.items(): - c_write_val(buf, k_type, k, k_spec) - c_write_val(buf, v_type, v, v_spec) - - -cdef inline read_struct(CyTransportBase buf, obj, decode_response=True): - cdef dict field_specs = obj.thrift_spec - cdef int fid - cdef TType field_type, ttype - cdef tuple field_spec - cdef str name - - while True: - field_type = read_i08(buf) - if field_type == T_STOP: - break - - fid = read_i16(buf) - if fid not in field_specs: - skip(buf, field_type) - continue - - field_spec = field_specs[fid] - ttype = field_spec[0] - if field_type != ttype: - skip(buf, field_type) - continue - - name = field_spec[1] - if len(field_spec) <= 3: - spec = None - else: - spec = field_spec[2] - - setattr(obj, name, c_read_val(buf, ttype, spec, decode_response)) - - return obj - - -cdef inline write_struct(CyTransportBase buf, obj): - cdef int fid - cdef TType f_type - cdef dict thrift_spec = obj.thrift_spec - cdef tuple field_spec - cdef str f_name - - for fid, field_spec in thrift_spec.items(): - f_type = field_spec[0] - f_name = field_spec[1] - if len(field_spec) <= 3: - container_spec = None - else: - container_spec = field_spec[2] - - v = getattr(obj, f_name) - if v is None: - continue - - write_i08(buf, f_type) - write_i16(buf, fid) - try: - c_write_val(buf, f_type, v, container_spec) - except (TypeError, AttributeError, AssertionError, OverflowError): - raise TDecodeException(obj.__class__.__name__, fid, f_name, v, - f_type, container_spec) - - write_i08(buf, T_STOP) - - -cdef inline c_read_binary(CyTransportBase buf, int32_t size): - cdef char string_val[STACK_STRING_LEN] - - if size > STACK_STRING_LEN: - data = malloc(size) - buf.c_read(size, data) - py_data = data[:size] - free(data) - else: - buf.c_read(size, string_val) - py_data = string_val[:size] - - return py_data - - -cdef inline c_read_string(CyTransportBase buf, int32_t size): - py_data = c_read_binary(buf, size) - try: - return py_data.decode("utf-8") - except: - return py_data - - -cdef c_read_val(CyTransportBase buf, TType ttype, spec=None, - decode_response=True): - cdef int size - cdef int64_t n - cdef TType v_type, k_type, orig_type, orig_key_type - - if ttype == T_BOOL: - return read_i08(buf) - - elif ttype == T_I08: - return read_i08(buf) - - elif ttype == T_I16: - return read_i16(buf) - - elif ttype == T_I32: - return read_i32(buf) - - elif ttype == T_I64: - return read_i64(buf) - - elif ttype == T_DOUBLE: - n = read_i64(buf) - return ((&n))[0] - - elif ttype == T_STRING: - size = read_i32(buf) - if decode_response: - return c_read_string(buf, size) - else: - return c_read_binary(buf, size) - - elif ttype == T_SET or ttype == T_LIST: - if isinstance(spec, int): - v_type = spec - v_spec = None - else: - v_type = spec[0] - v_spec = spec[1] - - orig_type = read_i08(buf) - size = read_i32(buf) - - if orig_type != v_type: - for _ in range(size): - skip(buf, orig_type) - return [] - - return [c_read_val(buf, v_type, v_spec, decode_response) - for _ in range(size)] - - elif ttype == T_MAP: - key = spec[0] - if isinstance(key, int): - k_type = key - k_spec = None - else: - k_type = key[0] - k_spec = key[1] - - value = spec[1] - if isinstance(value, int): - v_type = value - v_spec = None - else: - v_type = value[0] - v_spec = value[1] - - orig_key_type = read_i08(buf) - orig_type = read_i08(buf) - size = read_i32(buf) - - if orig_key_type != k_type or orig_type != v_type: - for _ in range(size): - skip(buf, orig_key_type) - skip(buf, orig_type) - return {} - - return {c_read_val(buf, k_type, k_spec, decode_response): c_read_val(buf, v_type, v_spec, decode_response) - for _ in range(size)} - - elif ttype == T_STRUCT: - return read_struct(buf, spec(), decode_response) - - -cdef c_write_val(CyTransportBase buf, TType ttype, val, spec=None): - if ttype == T_BOOL: - write_i08(buf, 1 if val else 0) - - elif ttype == T_I08: - write_i08(buf, val) - - elif ttype == T_I16: - write_i16(buf, val) - - elif ttype == T_I32: - write_i32(buf, val) - - elif ttype == T_I64: - write_i64(buf, val) - - elif ttype == T_DOUBLE: - write_double(buf, val) - - elif ttype == T_STRING: - if not isinstance(val, bytes): - try: - val = val.encode("utf-8") - except Exception: - pass - write_string(buf, val) - - elif ttype == T_SET or ttype == T_LIST: - write_list(buf, val, spec) - - elif ttype == T_MAP: - write_dict(buf, val, spec) - - elif ttype == T_STRUCT: - write_struct(buf, val) - - -cpdef skip(CyTransportBase buf, TType ttype): - cdef TType v_type, k_type, f_type - cdef int size - - if ttype == T_BOOL or ttype == T_I08: - read_i08(buf) - elif ttype == T_I16: - read_i16(buf) - elif ttype == T_I32: - read_i32(buf) - elif ttype == T_I64 or ttype == T_DOUBLE: - read_i64(buf) - elif ttype == T_STRING: - size = read_i32(buf) - c_read_binary(buf, size) - elif ttype == T_SET or ttype == T_LIST: - v_type = read_i08(buf) - size = read_i32(buf) - for _ in range(size): - skip(buf, v_type) - elif ttype == T_MAP: - k_type = read_i08(buf) - v_type = read_i08(buf) - size = read_i32(buf) - for _ in range(size): - skip(buf, k_type) - skip(buf, v_type) - elif ttype == T_STRUCT: - while 1: - f_type = read_i08(buf) - if f_type == T_STOP: - break - read_i16(buf) - skip(buf, f_type) - - -def read_val(CyTransportBase buf, TType ttype, decode_response=True): - return c_read_val(buf, ttype, None, decode_response) - - -def write_val(CyTransportBase buf, TType ttype, val, spec=None): - c_write_val(buf, ttype, val, spec) - - -cdef class TCyBinaryProtocol(object): - cdef public CyTransportBase trans - cdef public bool strict_read - cdef public bool strict_write - cdef public bool decode_response - - def __init__(self, trans, strict_read=True, strict_write=True, - decode_response=True): - self.trans = trans - self.strict_read = strict_read - self.strict_write = strict_write - self.decode_response = decode_response - - def skip(self, ttype): - skip(self.trans, (ttype)) - - def read_message_begin(self): - cdef int32_t size, version, seqid - cdef TType ttype - - size = read_i32(self.trans) - if size < 0: - version = size & VERSION_MASK - if version != VERSION_1: - raise ProtocolError('invalid version %d' % version) - - name = c_read_val(self.trans, T_STRING) - ttype = (size & TYPE_MASK) - else: - if self.strict_read: - raise ProtocolError('No protocol version header') - - name = c_read_string(self.trans, size) - ttype = (read_i08(self.trans)) - - seqid = read_i32(self.trans) - - return name, ttype, seqid - - def read_message_end(self): - pass - - def write_message_begin(self, name, TType ttype, int32_t seqid): - cdef int32_t version = VERSION_1 | ttype - if self.strict_write: - write_i32(self.trans, version) - c_write_val(self.trans, T_STRING, name) - else: - c_write_val(self.trans, T_STRING, name) - write_i08(self.trans, ttype) - - write_i32(self.trans, seqid) - - def write_message_end(self): - self.trans.c_flush() - - def read_struct(self, obj): - try: - return read_struct(self.trans, obj, self.decode_response) - except Exception: - self.trans.clean() - raise - - def write_struct(self, obj): - try: - write_struct(self.trans, obj) - except Exception: - self.trans.clean() - raise - - -class TCyBinaryProtocolFactory(object): - def __init__(self, strict_read=True, strict_write=True, - decode_response=True): - self.strict_read = strict_read - self.strict_write = strict_write - self.decode_response = decode_response - - def get_protocol(self, trans): - return TCyBinaryProtocol( - trans, self.strict_read, self.strict_write, self.decode_response) diff --git a/python/helpers/pydev/thriftpy/protocol/cybin/endian_port.h b/python/helpers/pydev/thriftpy/protocol/cybin/endian_port.h deleted file mode 100644 index be8299bd8bf2..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/cybin/endian_port.h +++ /dev/null @@ -1,42 +0,0 @@ - -#if defined(__APPLE__) - -#include - -#define htobe16(x) OSSwapHostToBigInt16(x) -#define htobe32(x) OSSwapHostToBigInt32(x) -#define htobe64(x) OSSwapHostToBigInt64(x) -#define be16toh(x) OSSwapBigToHostInt16(x) -#define be32toh(x) OSSwapBigToHostInt32(x) -#define be64toh(x) OSSwapBigToHostInt64(x) - -#else - -#include -#include - -#ifndef htobe16 -#define htobe16(x) bswap_16(x) -#endif - -#ifndef htobe32 -#define htobe32(x) bswap_32(x) -#endif - -#ifndef htobe64 -#define htobe64(x) bswap_64(x) -#endif - -#ifndef be16toh -#define be16toh(x) bswap_16(x) -#endif - -#ifndef be32toh -#define be32toh(x) bswap_32(x) -#endif - -#ifndef be64toh -#define be64toh(x) bswap_64(x) -#endif - -#endif diff --git a/python/helpers/pydev/thriftpy/protocol/exc.py b/python/helpers/pydev/thriftpy/protocol/exc.py deleted file mode 100644 index bc37a3059fa9..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/exc.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from ..thrift import TException - - -class TProtocolException(TException): - """Custom Protocol Exception class""" - - UNKNOWN = 0 - INVALID_DATA = 1 - NEGATIVE_SIZE = 2 - SIZE_LIMIT = 3 - BAD_VERSION = 4 - - def __init__(self, type=UNKNOWN, message=None): - TException.__init__(self, message) - self.type = type diff --git a/python/helpers/pydev/thriftpy/protocol/json.py b/python/helpers/pydev/thriftpy/protocol/json.py deleted file mode 100644 index 52d524ccd5d7..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/json.py +++ /dev/null @@ -1,213 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import struct - -import json -from thriftpy.thrift import TType -from .exc import TProtocolException - -INTEGER = (TType.BYTE, TType.I16, TType.I32, TType.I64) -FLOAT = (TType.DOUBLE,) - -VERSION = 1 - - -def json_value(ttype, val, spec=None): - if ttype in INTEGER or ttype in FLOAT or ttype == TType.STRING: - return val - - if ttype == TType.BOOL: - return True if val else False - - if ttype == TType.STRUCT: - return struct_to_json(val) - - if ttype in (TType.SET, TType.LIST): - return list_to_json(val, spec) - - if ttype == TType.MAP: - return map_to_json(val, spec) - - -def obj_value(ttype, val, spec=None): - if ttype in INTEGER: - return int(val) - - if ttype in FLOAT: - return float(val) - - if ttype in (TType.STRING, TType.BOOL): - return val - - if ttype == TType.STRUCT: - return struct_to_obj(val, spec()) - - if ttype in (TType.SET, TType.LIST): - return list_to_obj(val, spec) - - if ttype == TType.MAP: - return map_to_obj(val, spec) - - -def map_to_obj(val, spec): - res = {} - if isinstance(spec[0], int): - key_type, key_spec = spec[0], None - else: - key_type, key_spec = spec[0] - - if isinstance(spec[1], int): - value_type, value_spec = spec[1], None - else: - value_type, value_spec = spec[1] - - for v in val: - res[obj_value(key_type, v["key"], key_spec)] = obj_value( - value_type, v["value"], value_spec) - - return res - - -def map_to_json(val, spec): - res = [] - if isinstance(spec[0], int): - key_type = spec[0] - key_spec = None - else: - key_type, key_spec = spec[0] - - if isinstance(spec[1], int): - value_type = spec[1] - value_spec = None - else: - value_type, value_spec = spec[1] - - for k, v in val.items(): - res.append({"key": json_value(key_type, k, key_spec), - "value": json_value(value_type, v, value_spec)}) - - return res - - -def list_to_obj(val, spec): - if isinstance(spec, tuple): - elem_type, type_spec = spec - else: - elem_type, type_spec = spec, None - - return [obj_value(elem_type, i, type_spec) for i in val] - - -def list_to_json(val, spec): - if isinstance(spec, tuple): - elem_type, type_spec = spec - else: - elem_type, type_spec = spec, None - - return [json_value(elem_type, i, type_spec) for i in val] - - -def struct_to_json(val): - outobj = {} - for fid, field_spec in val.thrift_spec.items(): - field_type, field_name = field_spec[:2] - - if len(field_spec) <= 3: - field_type_spec = None - else: - field_type_spec = field_spec[2] - - v = getattr(val, field_name) - if v is None: - continue - - outobj[field_name] = json_value(field_type, v, field_type_spec) - - return outobj - - -def struct_to_obj(val, obj): - for fid, field_spec in obj.thrift_spec.items(): - field_type, field_name = field_spec[:2] - - if len(field_spec) <= 3: - field_type_spec = None - else: - field_type_spec = field_spec[2] - - if field_name in val: - setattr(obj, field_name, - obj_value(field_type, val[field_name], field_type_spec)) - - return obj - - -class TJSONProtocol(object): - """A JSON protocol. - - The message in the transport are encoded as this: 4 bytes represents - the length of the json object and immediately followed by the json object. - - '\x00\x00\x00+' '{"payload": {}, "metadata": {"version": 1}}' - - the 4 bytes are the bytes representation of an integer and is encoded in - big-endian. - """ - def __init__(self, trans): - self.trans = trans - self._meta = {"version": VERSION} - self._data = None - - def _write_len(self, x): - self.trans.write(struct.pack('!I', int(x))) - - def _read_len(self): - l = self.trans.read(4) - return struct.unpack('!I', l)[0] - - def read_message_begin(self): - size = self._read_len() - self._data = json.loads(self.trans.read(size).decode("utf-8")) - metadata = self._data["metadata"] - - version = int(metadata["version"]) - if version != VERSION: - raise TProtocolException( - type=TProtocolException.BAD_VERSION, - message="Bad version in read_message_begin:{}".format(version)) - - return metadata["name"], metadata["ttype"], metadata["seqid"] - - def read_message_end(self): - pass - - def write_message_begin(self, name, ttype, seqid): - self._meta.update({"name": name, "ttype": ttype, "seqid": seqid}) - - def write_message_end(self): - pass - - def read_struct(self, obj): - if not self._data: - size = self._read_len() - self._data = json.loads(self.trans.read(size).decode("utf-8")) - - res = struct_to_obj(self._data["payload"], obj) - self._data = None - return res - - def write_struct(self, obj): - data = json.dumps({ - "metadata": self._meta, - "payload": struct_to_json(obj) - }) - - self._write_len(len(data)) - self.trans.write(data.encode("utf-8")) - - -class TJSONProtocolFactory(object): - def get_protocol(self, trans): - return TJSONProtocol(trans) diff --git a/python/helpers/pydev/thriftpy/protocol/multiplex.py b/python/helpers/pydev/thriftpy/protocol/multiplex.py deleted file mode 100644 index 58b573cb2662..000000000000 --- a/python/helpers/pydev/thriftpy/protocol/multiplex.py +++ /dev/null @@ -1,34 +0,0 @@ -# -*- coding: utf-8 -*- - -from thriftpy.thrift import TMultiplexedProcessor, TMessageType - - -class TMultiplexedProtocol(object): - """Multiplex the protocol by prepend service name to api for every api call. - Can be used together with all original protocols. - """ - - def __init__(self, proto, service_name): - self.service_name = service_name - self._proto = proto - - def __getattr__(self, name): - return getattr(self._proto, name) - - def write_message_begin(self, name, ttype, seqid): - if ttype in (TMessageType.CALL, TMessageType.ONEWAY): - self._proto.write_message_begin( - self.service_name + TMultiplexedProcessor.SEPARATOR + name, - ttype, seqid) - else: - self._proto.write_message_begin(name, ttype, seqid) - - -class TMultiplexedProtocolFactory(object): - def __init__(self, proto_factory, service_name): - self._proto_factory = proto_factory - self.service_name = service_name - - def get_protocol(self, trans): - proto = self._proto_factory.get_protocol(trans) - return TMultiplexedProtocol(proto, self.service_name) diff --git a/python/helpers/pydev/thriftpy/rpc.py b/python/helpers/pydev/thriftpy/rpc.py deleted file mode 100644 index d229e896e9de..000000000000 --- a/python/helpers/pydev/thriftpy/rpc.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import contextlib -import warnings - -from thriftpy.protocol import TBinaryProtocolFactory -from thriftpy.server import TThreadedServer -from thriftpy.thrift import TProcessor, TClient -from thriftpy.transport import ( - TBufferedTransportFactory, - TServerSocket, - TSocket, -) - - -def make_client(service, host="localhost", port=9090, unix_socket=None, - proto_factory=TBinaryProtocolFactory(), - trans_factory=TBufferedTransportFactory(), - timeout=None): - if unix_socket: - socket = TSocket(unix_socket=unix_socket) - elif host and port: - socket = TSocket(host, port, socket_timeout=timeout) - else: - raise ValueError("Either host/port or unix_socket must be provided.") - - transport = trans_factory.get_transport(socket) - protocol = proto_factory.get_protocol(transport) - transport.open() - return TClient(service, protocol) - - -def make_server(service, handler, - host="localhost", port=9090, unix_socket=None, - proto_factory=TBinaryProtocolFactory(), - trans_factory=TBufferedTransportFactory()): - processor = TProcessor(service, handler) - if unix_socket: - server_socket = TServerSocket(unix_socket=unix_socket) - elif host and port: - server_socket = TServerSocket(host=host, port=port) - else: - raise ValueError("Either host/port or unix_socket must be provided.") - - server = TThreadedServer(processor, server_socket, - iprot_factory=proto_factory, - itrans_factory=trans_factory) - return server - - -@contextlib.contextmanager -def client_context(service, host="localhost", port=9090, unix_socket=None, - proto_factory=TBinaryProtocolFactory(), - trans_factory=TBufferedTransportFactory(), - timeout=3000, socket_timeout=3000, connect_timeout=None): - if timeout: - warnings.warn("`timeout` deprecated, use `socket_timeout` and " - "`connect_timeout` instead.") - socket_timeout = connect_timeout = timeout - - if unix_socket: - socket = TSocket(unix_socket=unix_socket, - connect_timeout=connect_timeout, - socket_timeout=socket_timeout) - elif host and port: - socket = TSocket(host, port, - connect_timeout=connect_timeout, - socket_timeout=socket_timeout) - else: - raise ValueError("Either host/port or unix_socket must be provided.") - - try: - transport = trans_factory.get_transport(socket) - protocol = proto_factory.get_protocol(transport) - transport.open() - yield TClient(service, protocol) - - finally: - transport.close() diff --git a/python/helpers/pydev/thriftpy/server.py b/python/helpers/pydev/thriftpy/server.py deleted file mode 100644 index 33a1d378f0d6..000000000000 --- a/python/helpers/pydev/thriftpy/server.py +++ /dev/null @@ -1,104 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import logging -import threading - -from thriftpy.protocol import TBinaryProtocolFactory -from thriftpy.transport import ( - TBufferedTransportFactory, - TTransportException -) - -logger = logging.getLogger(__name__) - - -class TServer(object): - def __init__(self, processor, trans, - itrans_factory=None, iprot_factory=None, - otrans_factory=None, oprot_factory=None): - self.processor = processor - self.trans = trans - - self.itrans_factory = itrans_factory or TBufferedTransportFactory() - self.iprot_factory = iprot_factory or TBinaryProtocolFactory() - self.otrans_factory = otrans_factory or self.itrans_factory - self.oprot_factory = oprot_factory or self.iprot_factory - - def serve(self): - pass - - def close(self): - pass - - -class TSimpleServer(TServer): - """Simple single-threaded server that just pumps around one transport.""" - - def __init__(self, *args): - TServer.__init__(self, *args) - self.closed = False - - def serve(self): - self.trans.listen() - while True: - client = self.trans.accept() - itrans = self.itrans_factory.get_transport(client) - otrans = self.otrans_factory.get_transport(client) - iprot = self.iprot_factory.get_protocol(itrans) - oprot = self.oprot_factory.get_protocol(otrans) - try: - while not self.closed: - self.processor.process(iprot, oprot) - except TTransportException: - pass - except Exception as x: - logger.exception(x) - - itrans.close() - otrans.close() - - def close(self): - self.closed = True - - -class TThreadedServer(TServer): - """Threaded server that spawns a new thread per each connection.""" - - def __init__(self, *args, **kwargs): - self.daemon = kwargs.pop("daemon", False) - TServer.__init__(self, *args, **kwargs) - self.closed = False - - def serve(self): - self.trans.listen() - while not self.closed: - try: - client = self.trans.accept() - t = threading.Thread(target=self.handle, args=(client,)) - t.setDaemon(self.daemon) - t.start() - except KeyboardInterrupt: - raise - except Exception as x: - logger.exception(x) - - def handle(self, client): - itrans = self.itrans_factory.get_transport(client) - otrans = self.otrans_factory.get_transport(client) - iprot = self.iprot_factory.get_protocol(itrans) - oprot = self.oprot_factory.get_protocol(otrans) - try: - while True: - self.processor.process(iprot, oprot) - except TTransportException: - pass - except Exception as x: - logger.exception(x) - - itrans.close() - otrans.close() - - def close(self): - self.closed = True diff --git a/python/helpers/pydev/thriftpy/thrift.py b/python/helpers/pydev/thriftpy/thrift.py deleted file mode 100644 index 0cf087729c29..000000000000 --- a/python/helpers/pydev/thriftpy/thrift.py +++ /dev/null @@ -1,393 +0,0 @@ -# -*- coding: utf-8 -*- - -""" - thriftpy.thrift - ~~~~~~~~~~~~~~~~~~ - - Thrift simplified. -""" - -from __future__ import absolute_import - -import functools - -from ._compat import init_func_generator, with_metaclass - - -def args2kwargs(thrift_spec, *args): - arg_names = [item[1][1] for item in sorted(thrift_spec.items())] - return dict(zip(arg_names, args)) - - -def parse_spec(ttype, spec=None): - name_map = TType._VALUES_TO_NAMES - - def _type(s): - return parse_spec(*s) if isinstance(s, tuple) else name_map[s] - - if spec is None: - return name_map[ttype] - - if ttype == TType.STRUCT: - return spec.__name__ - - if ttype in (TType.LIST, TType.SET): - return "%s<%s>" % (name_map[ttype], _type(spec)) - - if ttype == TType.MAP: - return "MAP<%s, %s>" % (_type(spec[0]), _type(spec[1])) - - -class TType(object): - STOP = 0 - VOID = 1 - BOOL = 2 - BYTE = 3 - I08 = 3 - DOUBLE = 4 - I16 = 6 - I32 = 8 - I64 = 10 - STRING = 11 - UTF7 = 11 - BINARY = 11 # This here just for parsing. For all purposes, it's a string - STRUCT = 12 - MAP = 13 - SET = 14 - LIST = 15 - UTF8 = 16 - UTF16 = 17 - - _VALUES_TO_NAMES = { - STOP: 'STOP', - VOID: 'VOID', - BOOL: 'BOOL', - BYTE: 'BYTE', - I08: 'BYTE', - DOUBLE: 'DOUBLE', - I16: 'I16', - I32: 'I32', - I64: 'I64', - STRING: 'STRING', - UTF7: 'STRING', - BINARY: 'STRING', - STRUCT: 'STRUCT', - MAP: 'MAP', - SET: 'SET', - LIST: 'LIST', - UTF8: 'UTF8', - UTF16: 'UTF16' - } - - -class TMessageType(object): - CALL = 1 - REPLY = 2 - EXCEPTION = 3 - ONEWAY = 4 - - -class TPayloadMeta(type): - - def __new__(cls, name, bases, attrs): - if "default_spec" in attrs: - attrs["__init__"] = init_func_generator(attrs.pop("default_spec")) - return super(TPayloadMeta, cls).__new__(cls, name, bases, attrs) - - -def gen_init(cls, thrift_spec=None, default_spec=None): - if thrift_spec is not None: - cls.thrift_spec = thrift_spec - - if default_spec is not None: - cls.__init__ = init_func_generator(default_spec) - return cls - - -class TPayload(with_metaclass(TPayloadMeta, object)): - - __hash__ = None - - def read(self, iprot): - iprot.read_struct(self) - - def write(self, oprot): - oprot.write_struct(self) - - def __repr__(self): - l = ['%s=%r' % (key, value) for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(l)) - - def __str__(self): - return repr(self) - - def __eq__(self, other): - return isinstance(other, self.__class__) and \ - self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not self.__eq__(other) - - -class TClient(object): - - def __init__(self, service, iprot, oprot=None): - self._service = service - self._iprot = self._oprot = iprot - if oprot is not None: - self._oprot = oprot - self._seqid = 0 - - def __getattr__(self, _api): - if _api in self._service.thrift_services: - return functools.partial(self._req, _api) - - raise AttributeError("{} instance has no attribute '{}'".format( - self.__class__.__name__, _api)) - - def __dir__(self): - return self._service.thrift_services - - def _req(self, _api, *args, **kwargs): - _kw = args2kwargs(getattr(self._service, _api + "_args").thrift_spec, - *args) - kwargs.update(_kw) - result_cls = getattr(self._service, _api + "_result") - - self._send(_api, **kwargs) - # wait result only if non-oneway - if not getattr(result_cls, "oneway"): - return self._recv(_api) - - def _send(self, _api, **kwargs): - self._oprot.write_message_begin(_api, TMessageType.CALL, self._seqid) - args = getattr(self._service, _api + "_args")() - for k, v in kwargs.items(): - setattr(args, k, v) - args.write(self._oprot) - self._oprot.write_message_end() - self._oprot.trans.flush() - - def _recv(self, _api): - fname, mtype, rseqid = self._iprot.read_message_begin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(self._iprot) - self._iprot.read_message_end() - raise x - result = getattr(self._service, _api + "_result")() - result.read(self._iprot) - self._iprot.read_message_end() - - if hasattr(result, "success") and result.success is not None: - return result.success - - # void api without throws - if len(result.thrift_spec) == 0: - return - - # check throws - for k, v in result.__dict__.items(): - if k != "success" and v: - raise v - - # no throws & not void api - if hasattr(result, "success"): - raise TApplicationException(TApplicationException.MISSING_RESULT) - - def close(self): - self._iprot.trans.close() - if self._iprot != self._oprot: - self._oprot.trans.close() - - -class TProcessor(object): - """Base class for procsessor, which works on two streams.""" - - def __init__(self, service, handler): - self._service = service - self._handler = handler - - def process_in(self, iprot): - api, type, seqid = iprot.read_message_begin() - if api not in self._service.thrift_services: - iprot.skip(TType.STRUCT) - iprot.read_message_end() - return api, seqid, TApplicationException(TApplicationException.UNKNOWN_METHOD), None # noqa - - args = getattr(self._service, api + "_args")() - args.read(iprot) - iprot.read_message_end() - result = getattr(self._service, api + "_result")() - - # convert kwargs to args - api_args = [args.thrift_spec[k][1] for k in sorted(args.thrift_spec)] - - def call(): - f = getattr(self._handler, api) - return f(*(args.__dict__[k] for k in api_args)) - - return api, seqid, result, call - - def send_exception(self, oprot, api, exc, seqid): - oprot.write_message_begin(api, TMessageType.EXCEPTION, seqid) - exc.write(oprot) - oprot.write_message_end() - oprot.trans.flush() - - def send_result(self, oprot, api, result, seqid): - oprot.write_message_begin(api, TMessageType.REPLY, seqid) - result.write(oprot) - oprot.write_message_end() - oprot.trans.flush() - - def handle_exception(self, e, result): - for k in sorted(result.thrift_spec): - if result.thrift_spec[k][1] == "success": - continue - - _, exc_name, exc_cls, _ = result.thrift_spec[k] - if isinstance(e, exc_cls): - setattr(result, exc_name, e) - break - else: - raise e - - def process(self, iprot, oprot): - api, seqid, result, call = self.process_in(iprot) - - if isinstance(result, TApplicationException): - return self.send_exception(oprot, api, result, seqid) - - try: - result.success = call() - except Exception as e: - # raise if api don't have throws - self.handle_exception(e, result) - - if not result.oneway: - self.send_result(oprot, api, result, seqid) - - -class TMultiplexedProcessor(TProcessor): - SEPARATOR = ":" - - def __init__(self): - self.processors = {} - - def register_processor(self, service_name, processor): - if service_name in self.processors: - raise TApplicationException( - type=TApplicationException.INTERNAL_ERROR, - message='processor for `{0}` already registered' - .format(service_name)) - self.processors[service_name] = processor - - def process_in(self, iprot): - api, type, seqid = iprot.read_message_begin() - if type not in (TMessageType.CALL, TMessageType.ONEWAY): - raise TException("TMultiplex protocol only supports CALL & ONEWAY") - if TMultiplexedProcessor.SEPARATOR not in api: - raise TException("Service name not found in message. " - "You should use TMultiplexedProtocol in client.") - - service_name, api = api.split(TMultiplexedProcessor.SEPARATOR) - if service_name not in self.processors: - iprot.skip(TType.STRUCT) - iprot.read_message_end() - e = TApplicationException(TApplicationException.UNKNOWN_METHOD) - return api, seqid, e, None - - proc = self.processors[service_name] - args = getattr(proc._service, api + "_args")() - args.read(iprot) - iprot.read_message_end() - result = getattr(proc._service, api + "_result")() - - # convert kwargs to args - api_args = [args.thrift_spec[k][1] for k in sorted(args.thrift_spec)] - - def call(): - f = getattr(proc._handler, api) - return f(*(args.__dict__[k] for k in api_args)) - - return api, seqid, result, call - - -class TProcessorFactory(object): - - def __init__(self, processor_class, *args, **kwargs): - self.args = args - self.kwargs = kwargs - - self.processor_class = processor_class - - def get_processor(self): - return self.processor_class(*self.args, **self.kwargs) - - -class TException(TPayload, Exception): - """Base class for all thrift exceptions.""" - - def __hash__(self): - return id(self) - - def __eq__(self, other): - return id(self) == id(other) - - -class TDecodeException(TException): - def __init__(self, name, fid, field, value, ttype, spec=None): - self.struct_name = name - self.fid = fid - self.field = field - self.value = value - - self.type_repr = parse_spec(ttype, spec) - - def __str__(self): - return ( - "Field '%s(%s)' of '%s' needs type '%s', " - "but the value is `%r`" - ) % (self.field, self.fid, self.struct_name, self.type_repr, - self.value) - - -class TApplicationException(TException): - """Application level thrift exceptions.""" - - thrift_spec = { - 1: (TType.STRING, 'message', False), - 2: (TType.I32, 'type', False), - } - - UNKNOWN = 0 - UNKNOWN_METHOD = 1 - INVALID_MESSAGE_TYPE = 2 - WRONG_METHOD_NAME = 3 - BAD_SEQUENCE_ID = 4 - MISSING_RESULT = 5 - INTERNAL_ERROR = 6 - PROTOCOL_ERROR = 7 - - def __init__(self, type=UNKNOWN, message=None): - super(TApplicationException, self).__init__() - self.type = type - self.message = message - - def __str__(self): - if self.message: - return self.message - - if self.type == self.UNKNOWN_METHOD: - return 'Unknown method' - elif self.type == self.INVALID_MESSAGE_TYPE: - return 'Invalid message type' - elif self.type == self.WRONG_METHOD_NAME: - return 'Wrong method name' - elif self.type == self.BAD_SEQUENCE_ID: - return 'Bad sequence ID' - elif self.type == self.MISSING_RESULT: - return 'Missing result' - else: - return 'Default (unknown) TApplicationException' diff --git a/python/helpers/pydev/thriftpy/tornado.py b/python/helpers/pydev/thriftpy/tornado.py deleted file mode 100644 index cc138ced6b70..000000000000 --- a/python/helpers/pydev/thriftpy/tornado.py +++ /dev/null @@ -1,231 +0,0 @@ -# -*- coding: utf-8 -*- - -""" ->>> pingpong = thriftpy.load("pingpong.thrift") ->>> ->>> class Dispatcher(object): ->>> def ping(self): ->>> return "pong" - ->>> server = make_server(pingpong.PingPong, Dispatcher()) ->>> server.listen(6000) ->>> client = ioloop.IOLoop.current().run_sync( - lambda: make_client(pingpong.PingPong, '127.0.0.1', 6000)) ->>> ioloop.IOLoop.current().run_sync(client.ping) -'pong' -""" - -from __future__ import absolute_import - -import logging -import socket -import struct -import toro -from contextlib import contextmanager -from datetime import timedelta -from io import BytesIO -from tornado import tcpserver, ioloop, iostream, gen - -# TODO need TCyTornadoStreamTransport to work with cython binary protocol -from .protocol.binary import TBinaryProtocolFactory -from .thrift import TApplicationException, TProcessor, TClient -from .transport import TTransportException, TTransportBase -from .transport.memory import TMemoryBuffer - -logger = logging.getLogger(__name__) - - -class TTornadoStreamTransport(TTransportBase): - """a framed, buffered transport over a Tornado stream""" - DEFAULT_CONNECT_TIMEOUT = timedelta(seconds=1) - DEFAULT_READ_TIMEOUT = timedelta(seconds=1) - - def __init__(self, host, port, stream=None, io_loop=None, ssl_options=None, - read_timeout=DEFAULT_READ_TIMEOUT): - self.host = host - self.port = port - self.io_loop = io_loop or ioloop.IOLoop.current() - self.read_timeout = read_timeout - self.is_queuing_reads = False - self.read_queue = [] - self.__wbuf = BytesIO() - self._read_lock = toro.Lock() - self.ssl_options = ssl_options - - # servers provide a ready-to-go stream - self.stream = stream - if self.stream is not None: - self._set_close_callback() - - def with_timeout(self, timeout, future): - return gen.with_timeout(timeout, future, self.io_loop) - - @gen.coroutine - def open(self, timeout=DEFAULT_CONNECT_TIMEOUT): - logger.debug('socket connecting') - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) - if self.ssl_options is None: - self.stream = iostream.IOStream(sock) - else: - self.stream = iostream.SSLIOStream(sock, ssl_options=self.ssl_options) - - try: - yield self.with_timeout(timeout, self.stream.connect( - (self.host, self.port))) - except (socket.error, OSError, IOError): - message = 'could not connect to {}:{}'.format(self.host, self.port) - raise TTransportException( - type=TTransportException.NOT_OPEN, - message=message) - - self._set_close_callback() - raise gen.Return(self) - - def _set_close_callback(self): - self.stream.set_close_callback(self.close) - - def close(self): - # don't raise if we intend to close - self.stream.set_close_callback(None) - self.stream.close() - - def read(self, _): - # The generated code for Tornado shouldn't do individual reads -- only - # frames at a time - assert False, "you're doing it wrong" - - @contextmanager - def io_exception_context(self): - try: - yield - except (socket.error, OSError, IOError) as e: - raise TTransportException( - type=TTransportException.END_OF_FILE, - message=str(e)) - except iostream.StreamBufferFullError as e: - raise TTransportException( - type=TTransportException.UNKNOWN, - message=str(e)) - except gen.TimeoutError as e: - raise TTransportException( - type=TTransportException.TIMED_OUT, - message=str(e)) - - @gen.coroutine - def read_frame(self): - # IOStream processes reads one at a time - with (yield self._read_lock.acquire()): - with self.io_exception_context(): - frame_header = yield self._read_bytes(4) - if len(frame_header) == 0: - raise iostream.StreamClosedError( - 'Read zero bytes from stream') - frame_length, = struct.unpack('!i', frame_header) - logger.debug('received frame header, frame length = %d', - frame_length) - frame = yield self._read_bytes(frame_length) - logger.debug('received frame payload: %r', frame) - raise gen.Return(frame) - - def _read_bytes(self, n): - return self.with_timeout(self.read_timeout, self.stream.read_bytes(n)) - - def write(self, buf): - self.__wbuf.write(buf) - - def flush(self): - frame = self.__wbuf.getvalue() - # reset wbuf before write/flush to preserve state on underlying failure - frame_length = struct.pack('!i', len(frame)) - self.__wbuf = BytesIO() - with self.io_exception_context(): - return self.stream.write(frame_length + frame) - - -class TTornadoServer(tcpserver.TCPServer): - def __init__(self, processor, iprot_factory, oprot_factory=None, - transport_read_timeout=TTornadoStreamTransport.DEFAULT_READ_TIMEOUT, # noqa - *args, **kwargs): - super(TTornadoServer, self).__init__(*args, **kwargs) - - self._processor = processor - self._iprot_factory = iprot_factory - self._oprot_factory = (oprot_factory if oprot_factory is not None - else iprot_factory) - self.transport_read_timeout = transport_read_timeout - - @gen.coroutine - def handle_stream(self, stream, address): - host, port = address - trans = TTornadoStreamTransport( - host=host, port=port, stream=stream, - io_loop=self.io_loop, read_timeout=self.transport_read_timeout) - try: - oprot = self._oprot_factory.get_protocol(trans) - iprot = self._iprot_factory.get_protocol(TMemoryBuffer()) - - while not trans.stream.closed(): - # TODO: maybe read multiple frames in advance for concurrency - try: - frame = yield trans.read_frame() - except TTransportException as e: - if e.type == TTransportException.END_OF_FILE: - break - else: - raise - - iprot.trans.setvalue(frame) - api, seqid, result, call = self._processor.process_in(iprot) - if isinstance(result, TApplicationException): - self._processor.send_exception(oprot, api, result, seqid) - else: - try: - result.success = yield gen.maybe_future(call()) - except Exception as e: - # raise if api don't have throws - self._processor.handle_exception(e, result) - - self._processor.send_result(oprot, api, result, seqid) - except Exception: - logger.exception('thrift exception in handle_stream') - trans.close() - - logger.info('client disconnected %s:%d', host, port) - - -class TTornadoClient(TClient): - @gen.coroutine - def _recv(self, api): - frame = yield self._oprot.trans.read_frame() - self._iprot.trans.setvalue(frame) - result = super(TTornadoClient, self)._recv(api) - raise gen.Return(result) - - def close(self): - self._oprot.trans.close() - - -def make_server( - service, handler, proto_factory=TBinaryProtocolFactory(), - io_loop=None, ssl_options=None, - transport_read_timeout=TTornadoStreamTransport.DEFAULT_READ_TIMEOUT): - processor = TProcessor(service, handler) - server = TTornadoServer(processor, iprot_factory=proto_factory, - transport_read_timeout=transport_read_timeout, - io_loop=io_loop, ssl_options=ssl_options) - return server - - -@gen.coroutine -def make_client( - service, host, port, proto_factory=TBinaryProtocolFactory(), - io_loop=None, ssl_options=None, - connect_timeout=TTornadoStreamTransport.DEFAULT_CONNECT_TIMEOUT, - read_timeout=TTornadoStreamTransport.DEFAULT_READ_TIMEOUT): - transport = TTornadoStreamTransport(host, port, io_loop=io_loop, ssl_options=ssl_options, - read_timeout=read_timeout) - iprot = proto_factory.get_protocol(TMemoryBuffer()) - oprot = proto_factory.get_protocol(transport) - yield transport.open(connect_timeout) - client = TTornadoClient(service, iprot, oprot) - raise gen.Return(client) diff --git a/python/helpers/pydev/thriftpy/transport/__init__.py b/python/helpers/pydev/thriftpy/transport/__init__.py deleted file mode 100644 index 9948d760a077..000000000000 --- a/python/helpers/pydev/thriftpy/transport/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from thriftpy._compat import CYTHON -from ..thrift import TType, TException - - -def readall(read_fn, sz): - buff = b'' - have = 0 - while have < sz: - chunk = read_fn(sz - have) - have += len(chunk) - buff += chunk - - if len(chunk) == 0: - raise TTransportException(TTransportException.END_OF_FILE, - "End of file reading from transport") - - return buff - - -class TTransportBase(object): - """Base class for Thrift transport layer.""" - - def _read(self, sz): - raise NotImplementedError - - def read(self, sz): - return readall(self._read, sz) - - -class TTransportException(TException): - """Custom Transport Exception class""" - - thrift_spec = { - 1: (TType.STRING, 'message'), - 2: (TType.I32, 'type'), - } - - UNKNOWN = 0 - NOT_OPEN = 1 - ALREADY_OPEN = 2 - TIMED_OUT = 3 - END_OF_FILE = 4 - - def __init__(self, type=UNKNOWN, message=None): - super(TTransportException, self).__init__() - self.type = type - self.message = message - - -# Avoid recursive import -from .socket import TSocket, TServerSocket # noqa -from .sslsocket import TSSLSocket, TSSLServerSocket # noqa -from ._ssl import create_thriftpy_context # noqa -from .buffered import TBufferedTransport, TBufferedTransportFactory # noqa -from .framed import TFramedTransport, TFramedTransportFactory # noqa -from .memory import TMemoryBuffer # noqa - -if CYTHON: - from .buffered import TCyBufferedTransport, TCyBufferedTransportFactory - from .framed import TCyFramedTransport, TCyFramedTransportFactory - from .memory import TCyMemoryBuffer - - # enable cython binary by default for CPython. - TMemoryBuffer = TCyMemoryBuffer # noqa - TBufferedTransport = TCyBufferedTransport # noqa - TBufferedTransportFactory = TCyBufferedTransportFactory # noqa - TFramedTransport = TCyFramedTransport # noqa - TFramedTransportFactory = TCyFramedTransportFactory # noqa -else: - # disable cython binary protocol for PYPY since it's slower. - TCyMemoryBuffer = TMemoryBuffer - TCyBufferedTransport = TBufferedTransport - TCyBufferedTransportFactory = TBufferedTransportFactory - TCyFramedTransport = TFramedTransport - TCyFramedTransportFactory = TFramedTransportFactory - -__all__ = [ - "TSocket", "TServerSocket", - "TSSLSocket", "TSSLServerSocket", "create_thriftpy_context", - "TTransportBase", "TTransportException", - "TMemoryBuffer", "TFramedTransport", "TFramedTransportFactory", - "TBufferedTransport", "TBufferedTransportFactory", "TCyMemoryBuffer", - "TCyBufferedTransport", "TCyBufferedTransportFactory", - "TCyFramedTransport", "TCyFramedTransportFactory" -] diff --git a/python/helpers/pydev/thriftpy/transport/_ssl.py b/python/helpers/pydev/thriftpy/transport/_ssl.py deleted file mode 100644 index f0bf1fcfa78f..000000000000 --- a/python/helpers/pydev/thriftpy/transport/_ssl.py +++ /dev/null @@ -1,174 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -The codes in this ssl compat lib were inspired by urllib3.utils.ssl_ module. -""" - -import ssl -import warnings - -from .._compat import MODERN_SSL - -try: - from ssl import ( - OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, - OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE, OP_SINGLE_ECDH_USE - ) -except ImportError: - OP_NO_SSLv2 = 0x1000000 - OP_NO_SSLv3 = 0x2000000 - OP_NO_COMPRESSION = 0x20000 - OP_CIPHER_SERVER_PREFERENCE = 0x400000 - OP_SINGLE_DH_USE = 0x100000 - OP_SINGLE_ECDH_USE = 0x80000 - - -# Disable weak or insecure ciphers by default -# (OpenSSL's default setting is 'DEFAULT:!aNULL:!eNULL') -# Enable a better set of ciphers by default -# This list has been explicitly chosen to: -# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) -# * Prefer ECDHE over DHE for better performance -# * Prefer any AES-GCM over any AES-CBC for better performance and security -# * Then Use HIGH cipher suites as a fallback -# * Then Use 3DES as fallback which is secure but slow -# * Disable NULL authentication, NULL encryption, and MD5 MACs for security -# reasons -DEFAULT_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5' -) - -# Restricted and more secure ciphers for the server side -# This list has been explicitly chosen to: -# * Prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE) -# * Prefer ECDHE over DHE for better performance -# * Prefer any AES-GCM over any AES-CBC for better performance and security -# * Then Use HIGH cipher suites as a fallback -# * Then Use 3DES as fallback which is secure but slow -# * Disable NULL authentication, NULL encryption, MD5 MACs, DSS, and RC4 for -# security reasons -RESTRICTED_SERVER_CIPHERS = ( - 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+HIGH:' - 'DH+HIGH:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+HIGH:RSA+3DES:!aNULL:' - '!eNULL:!MD5:!DSS:!RC4' -) - - -class InsecurePlatformWarning(Warning): - """Warned when certain SSL configuration is not available on a platform. - """ - pass - - -try: - from ssl import SSLContext -except ImportError: - import sys - - class SSLContext(object): - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) - - def __init__(self, protocol_version): - self.protocol = protocol_version - # Use default values from a real SSLContext - self.check_hostname = False - self.verify_mode = ssl.CERT_NONE - self.ca_certs = None - self.options = 0 - self.certfile = None - self.keyfile = None - self.ciphers = None - - def load_cert_chain(self, certfile=None, keyfile=None): - self.certfile = certfile - self.keyfile = keyfile - - def load_verify_locations(self, cafile=None, capath=None): - if capath is not None: - raise OSError("CA directories not supported in older Pythons") - self.ca_certs = cafile - - def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - "Your version of Python does not support setting " - "a custom cipher suite. Please upgrade to Python " - "2.7, 3.2, or later if you need this functionality." - ) - self.ciphers = cipher_suite - - def wrap_socket(self, socket, server_hostname=None, server_side=False): - warnings.warn( - "A true SSLContext object is not available. This prevents " - "urllib3 from configuring SSL appropriately and may cause " - "certain SSL connections to fail.", - InsecurePlatformWarning - ) - kwargs = { - "keyfile": self.keyfile, - "certfile": self.certfile, - "ca_certs": self.ca_certs, - "cert_reqs": self.verify_mode, - "ssl_version": self.protocol, - "server_side": server_side, - } - - if self.supports_set_ciphers: - # Platform-specific: Python 2.7+ - return ssl.wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: - # Platform-specific: Python 2.6 - return ssl.wrap_socket(socket, **kwargs) - - -def create_thriftpy_context(server_side=False, ciphers=None): - """Backport create_default_context for older python versions. - - The SSLContext has some default security options, you can disable them - manually, for example:: - - from thriftpy.transport import _ssl - context = _ssl.create_thriftpy_context() - context.options &= ~_ssl.OP_NO_SSLv3 - - You can do the same to enable compression. - """ - if MODERN_SSL: - if server_side: - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - else: - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH) - - if ciphers: - context.set_ciphers(ciphers) - - else: - context = SSLContext(ssl.PROTOCOL_SSLv23) - context.options |= OP_NO_SSLv2 - context.options |= OP_NO_SSLv3 - context.options |= OP_NO_COMPRESSION - - # server/client default options - if server_side: - context.options |= OP_CIPHER_SERVER_PREFERENCE - context.options |= OP_SINGLE_DH_USE - context.options |= OP_SINGLE_ECDH_USE - else: - context.verify_mode = ssl.CERT_REQUIRED - # context.check_hostname = True - warnings.warn( - "ssl check hostname support disabled, upgrade your python", - InsecurePlatformWarning) - - # Platform-specific: Python 2.6 - if getattr(context, 'supports_set_ciphers', True): - if ciphers: - context.set_ciphers(ciphers) - else: - warnings.warn("ssl ciphers support disabled, upgrade your python", - InsecurePlatformWarning) - - return context diff --git a/python/helpers/pydev/thriftpy/transport/buffered/__init__.py b/python/helpers/pydev/thriftpy/transport/buffered/__init__.py deleted file mode 100644 index f53904ca5df2..000000000000 --- a/python/helpers/pydev/thriftpy/transport/buffered/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from io import BytesIO - -from thriftpy._compat import CYTHON -from .. import TTransportBase - - -class TBufferedTransport(TTransportBase): - """Class that wraps another transport and buffers its I/O. - - The implementation uses a (configurable) fixed-size read buffer - but buffers all writes until a flush is performed. - """ - DEFAULT_BUFFER = 4096 - - def __init__(self, trans, buf_size=DEFAULT_BUFFER): - self._trans = trans - self._wbuf = BytesIO() - self._rbuf = BytesIO(b"") - self._buf_size = buf_size - - def is_open(self): - return self._trans.is_open() - - def open(self): - return self._trans.open() - - def close(self): - return self._trans.close() - - def _read(self, sz): - ret = self._rbuf.read(sz) - if len(ret) != 0: - return ret - - self._rbuf = BytesIO(self._trans.read(max(sz, self._buf_size))) - return self._rbuf.read(sz) - - def write(self, buf): - self._wbuf.write(buf) - - def flush(self): - out = self._wbuf.getvalue() - # reset wbuf before write/flush to preserve state on underlying failure - self._wbuf = BytesIO() - self._trans.write(out) - self._trans.flush() - - def getvalue(self): - return self._trans.getvalue() - - -class TBufferedTransportFactory(object): - def get_transport(self, trans): - return TBufferedTransport(trans) - - -if CYTHON: - from .cybuffered import TCyBufferedTransport, TCyBufferedTransportFactory # noqa diff --git a/python/helpers/pydev/thriftpy/transport/buffered/cybuffered.pyx b/python/helpers/pydev/thriftpy/transport/buffered/cybuffered.pyx deleted file mode 100644 index 22645228ec74..000000000000 --- a/python/helpers/pydev/thriftpy/transport/buffered/cybuffered.pyx +++ /dev/null @@ -1,90 +0,0 @@ -from thriftpy.transport.cybase cimport ( - TCyBuffer, - CyTransportBase, - DEFAULT_BUFFER -) - -from .. import TTransportException - -DEF MIN_BUFFER_SIZE = 1024 - - -cdef class TCyBufferedTransport(CyTransportBase): - """binary reader/writer""" - - cdef: - TCyBuffer rbuf, wbuf - - def __init__(self, trans, int buf_size=DEFAULT_BUFFER): - if buf_size < MIN_BUFFER_SIZE: - raise Exception("buffer too small") - - self.trans = trans - self.rbuf = TCyBuffer(buf_size) - self.wbuf = TCyBuffer(buf_size) - - def clean(self): - self.rbuf.clean() - self.wbuf.clean() - - def is_open(self): - return self.trans.is_open() - - def open(self): - return self.trans.open() - - def close(self): - return self.trans.close() - - def write(self, bytes data): - cdef int sz = len(data) - return self.c_write(data, sz) - - def read(self, int sz): - return self.get_string(sz) - - def flush(self): - return self.c_flush() - - cdef c_write(self, const char *data, int sz): - cdef: - int cap = self.wbuf.buf_size - self.wbuf.data_size - int r - - if cap < sz: - self.c_flush() - - r = self.wbuf.write(sz, data) - if r == -1: - raise MemoryError("Write to buffer error") - - cdef c_read(self, int sz, char* out): - if sz <= 0: - return 0 - - self.read_trans(sz, out) - return sz - - cdef read_trans(self, int sz, char *out): - cdef int i = self.rbuf.read_trans(self.trans, sz, out) - if i == -1: - raise TTransportException(TTransportException.END_OF_FILE, - "End of file reading from transport") - elif i == -2: - raise MemoryError("grow read buffer fail") - - cdef c_flush(self): - cdef bytes data - if self.wbuf.data_size > 0: - data = self.wbuf.buf[:self.wbuf.data_size] - self.trans.write(data) - self.trans.flush() - self.wbuf.clean() - - def getvalue(self): - return self.trans.getvalue() - - -class TCyBufferedTransportFactory(object): - def get_transport(self, trans): - return TCyBufferedTransport(trans) diff --git a/python/helpers/pydev/thriftpy/transport/cybase.pxd b/python/helpers/pydev/thriftpy/transport/cybase.pxd deleted file mode 100644 index a960be90369d..000000000000 --- a/python/helpers/pydev/thriftpy/transport/cybase.pxd +++ /dev/null @@ -1,24 +0,0 @@ -cdef enum: - DEFAULT_BUFFER = 4096 - STACK_STRING_LEN = 4096 - -cdef class TCyBuffer(object): - cdef: - char *buf - int cur, buf_size, data_size - - void move_to_start(self) - void clean(self) - int write(self, int sz, const char *value) - int grow(self, int min_size) - read_trans(self, trans, int sz, char *out) - - -cdef class CyTransportBase(object): - cdef object trans - - cdef c_read(self, int sz, char* out) - cdef c_write(self, char* data, int sz) - cdef c_flush(self) - - cdef get_string(self, int sz) diff --git a/python/helpers/pydev/thriftpy/transport/cybase.pyx b/python/helpers/pydev/thriftpy/transport/cybase.pyx deleted file mode 100644 index c05560f86c88..000000000000 --- a/python/helpers/pydev/thriftpy/transport/cybase.pyx +++ /dev/null @@ -1,138 +0,0 @@ -from libc.stdlib cimport malloc, free -from libc.string cimport memcpy, memmove - - -cdef class TCyBuffer(object): - def __cinit__(self, buf_size): - self.buf = malloc(buf_size) - self.buf_size = buf_size - self.cur = 0 - self.data_size = 0 - - def __dealloc__(self): - if self.buf != NULL: - free(self.buf) - self.buf = NULL - - cdef void move_to_start(self): - memmove(self.buf, self.buf + self.cur, self.data_size) - self.cur = 0 - - cdef void clean(self): - self.cur = 0 - self.data_size = 0 - - cdef int write(self, int sz, const char *value): - cdef: - int cap = self.buf_size - self.data_size - int remain = cap - self.cur - - if sz <= 0: - return 0 - - if remain < sz: - self.move_to_start() - - # recompute remain spaces - remain = cap - self.cur - - if remain < sz: - if self.grow(sz - remain + self.buf_size) != 0: - return -1 - - memcpy(self.buf + self.cur + self.data_size, value, sz) - self.data_size += sz - - return sz - - cdef read_trans(self, trans, int sz, char *out): - cdef int cap, new_data_len - - if sz <= 0: - return 0 - - if self.data_size < sz: - if self.buf_size < sz: - if self.grow(sz) != 0: - return -2 # grow buffer error - - cap = self.buf_size - self.data_size - - new_data = trans.read(cap) - new_data_len = len(new_data) - - while new_data_len + self.data_size < sz: - more = trans.read(cap - new_data_len) - more_len = len(more) - if more_len <= 0: - return -1 # end of file error - - new_data += more - new_data_len += more_len - - if cap - self.cur < new_data_len: - self.move_to_start() - - memcpy(self.buf + self.cur + self.data_size, new_data, - new_data_len) - self.data_size += new_data_len - - memcpy(out, self.buf + self.cur, sz) - self.cur += sz - self.data_size -= sz - - return sz - - cdef int grow(self, int min_size): - if min_size <= self.buf_size: - return 0 - - cdef int multiples = min_size / self.buf_size - if min_size % self.buf_size != 0: - multiples += 1 - - cdef int new_size = self.buf_size * multiples - cdef char *new_buf = malloc(new_size) - if new_buf == NULL: - return -1 - memcpy(new_buf + self.cur, self.buf + self.cur, self.data_size) - free(self.buf) - self.buf_size = new_size - self.buf = new_buf - return 0 - - -cdef class CyTransportBase(object): - cdef c_read(self, int sz, char* out): - pass - - cdef c_write(self, char* data, int sz): - pass - - cdef c_flush(self): - pass - - def clean(self): - pass - - @property - def sock(self): - if not self.trans: - return - return getattr(self.trans, 'sock', None) - - cdef get_string(self, int sz): - cdef: - char out[STACK_STRING_LEN] - char *dy_out - - if sz > STACK_STRING_LEN: - dy_out = malloc(sz) - try: - size = self.c_read(sz, dy_out) - return dy_out[:size] - finally: - free(dy_out) - else: - size = self.c_read(sz, out) - return out[:size] diff --git a/python/helpers/pydev/thriftpy/transport/framed/__init__.py b/python/helpers/pydev/thriftpy/transport/framed/__init__.py deleted file mode 100644 index cf42e111e400..000000000000 --- a/python/helpers/pydev/thriftpy/transport/framed/__init__.py +++ /dev/null @@ -1,74 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import struct -from io import BytesIO - -from thriftpy._compat import CYTHON -from .. import TTransportBase, readall -from ..buffered import TBufferedTransport - - -class TFramedTransport(TTransportBase): - """Class that wraps another transport and frames its I/O when writing.""" - def __init__(self, trans): - self._trans = trans - self._rbuf = BytesIO() - self._wbuf = BytesIO() - - def is_open(self): - return self._trans.is_open() - - def open(self): - return self._trans.open() - - def close(self): - return self._trans.close() - - def read(self, sz): - # Important: don't attempt to read the next frame if the caller - # doesn't actually need any data. - if sz == 0: - return b'' - - ret = self._rbuf.read(sz) - if len(ret) != 0: - return ret - - self.read_frame() - return self._rbuf.read(sz) - - def read_frame(self): - buff = readall(self._trans.read, 4) - sz, = struct.unpack('!i', buff) - frame = readall(self._trans.read, sz) - self._rbuf = BytesIO(frame) - - def write(self, buf): - self._wbuf.write(buf) - - def flush(self): - # reset wbuf before write/flush to preserve state on underlying failure - out = self._wbuf.getvalue() - self._wbuf = BytesIO() - - # N.B.: Doing this string concatenation is WAY cheaper than making - # two separate calls to the underlying socket object. Socket writes in - # Python turn out to be REALLY expensive, but it seems to do a pretty - # good job of managing string buffer operations without excessive - # copies - self._trans.write(struct.pack("!i", len(out)) + out) - self._trans.flush() - - def getvalue(self): - return self._trans.getvalue() - - -class TFramedTransportFactory(object): - def get_transport(self, trans): - return TBufferedTransport(TFramedTransport(trans)) - - -if CYTHON: - from .cyframed import TCyFramedTransport, TCyFramedTransportFactory # noqa diff --git a/python/helpers/pydev/thriftpy/transport/framed/cyframed.pyx b/python/helpers/pydev/thriftpy/transport/framed/cyframed.pyx deleted file mode 100644 index 04b488680b53..000000000000 --- a/python/helpers/pydev/thriftpy/transport/framed/cyframed.pyx +++ /dev/null @@ -1,133 +0,0 @@ -from libc.stdint cimport - -int32_t -from libc.stdlib cimport - -malloc, free -from libc.string cimport - -memcpy - -from thriftpy.transport.cybase cimport - -( - TCyBuffer, - CyTransportBase, - DEFAULT_BUFFER, - STACK_STRING_LEN -) -from .. import TTransportException - - -cdef extern from "../../protocol/cybin/endian_port.h": - int32_t be32toh(int32_t n) - int32_t htobe32(int32_t n) - - -cdef class TCyFramedTransport(CyTransportBase): - cdef: - TCyBuffer rbuf, rframe_buf, wframe_buf - - def __init__(self, trans, int buf_size=DEFAULT_BUFFER): - self.trans = trans - self.rbuf = TCyBuffer(buf_size) - self.rframe_buf = TCyBuffer(buf_size) - self.wframe_buf = TCyBuffer(buf_size) - - cdef read_trans(self, int sz, char *out): - cdef int i = self.rbuf.read_trans(self.trans, sz, out) - if i == -1: - raise TTransportException(TTransportException.END_OF_FILE, - "End of file reading from transport") - elif i == -2: - raise MemoryError("grow buffer fail") - - cdef write_rframe_buffer(self, const char *data, int sz): - cdef int r = self.rframe_buf.write(sz, data) - if r == -1: - raise MemoryError("Write to buffer error") - - cdef c_read(self, int sz, char *out): - if sz <= 0: - return 0 - - while self.rframe_buf.data_size < sz: - self.read_frame() - - memcpy(out, self.rframe_buf.buf + self.rframe_buf.cur, sz) - self.rframe_buf.cur += sz - self.rframe_buf.data_size -= sz - - return sz - - cdef c_write(self, const char *data, int sz): - cdef int r = self.wframe_buf.write(sz, data) - if r == -1: - raise MemoryError("Write to buffer error") - - cdef read_frame(self): - cdef: - char frame_len[4] - char stack_frame[STACK_STRING_LEN] - char *dy_frame - int32_t frame_size - - self.read_trans(4, frame_len) - frame_size = be32toh((frame_len)[0]) - - if frame_size <= 0: - raise TTransportException("No frame.", TTransportException.UNKNOWN) - - if frame_size <= STACK_STRING_LEN: - self.read_trans(frame_size, stack_frame) - self.write_rframe_buffer(stack_frame, frame_size) - else: - dy_frame = malloc(frame_size) - try: - self.read_trans(frame_size, dy_frame) - self.write_rframe_buffer(dy_frame, frame_size) - finally: - free(dy_frame) - - cdef c_flush(self): - cdef: - bytes data - char *size_str - - if self.wframe_buf.data_size > 0: - data = self.wframe_buf.buf[:self.wframe_buf.data_size] - size = htobe32(self.wframe_buf.data_size) - size_str = (&size) - - self.trans.write(size_str[:4] + data) - self.trans.flush() - self.wframe_buf.clean() - - def read(self, int sz): - return self.get_string(sz) - - def write(self, bytes data): - cdef int sz = len(data) - self.c_write(data, sz) - - def flush(self): - self.c_flush() - - def is_open(self): - return self.trans.is_open() - - def open(self): - return self.trans.open() - - def close(self): - return self.trans.close() - - def clean(self): - self.rbuf.clean() - self.rframe_buf.clean() - self.wframe_buf.clean() - - -class TCyFramedTransportFactory(object): - def get_transport(self, trans): - return TCyFramedTransport(trans) diff --git a/python/helpers/pydev/thriftpy/transport/memory/__init__.py b/python/helpers/pydev/thriftpy/transport/memory/__init__.py deleted file mode 100644 index 7e83fba51de1..000000000000 --- a/python/helpers/pydev/thriftpy/transport/memory/__init__.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -from io import BytesIO - -from thriftpy._compat import CYTHON -from .. import TTransportBase - - -class TMemoryBuffer(TTransportBase): - """Wraps a BytesIO object as a TTransport.""" - - def __init__(self, value=None): - """value -- a value as the initial value in the BytesIO object. - - If value is set, the transport can be read first. - """ - self._buffer = BytesIO(value) if value is not None else BytesIO() - self._pos = 0 - - def is_open(self): - return not self._buffer.closed - - def open(self): - pass - - def close(self): - self._buffer.close() - - def read(self, sz): - return self._read(sz) - - def _read(self, sz): - orig_pos = self._buffer.tell() - self._buffer.seek(self._pos) - res = self._buffer.read(sz) - self._buffer.seek(orig_pos) - self._pos += len(res) - return res - - def write(self, buf): - self._buffer.write(buf) - - def flush(self): - pass - - def getvalue(self): - return self._buffer.getvalue() - - def setvalue(self, value): - self._buffer = BytesIO(value) - self._pos = 0 - - -if CYTHON: - from .cymemory import TCyMemoryBuffer # noqa diff --git a/python/helpers/pydev/thriftpy/transport/memory/cymemory.pyx b/python/helpers/pydev/thriftpy/transport/memory/cymemory.pyx deleted file mode 100644 index a2c43019185c..000000000000 --- a/python/helpers/pydev/thriftpy/transport/memory/cymemory.pyx +++ /dev/null @@ -1,98 +0,0 @@ -from libc.stdlib cimport - -malloc, free -from libc.string cimport - -memcpy - -from thriftpy.transport.cybase cimport - -( - TCyBuffer, - CyTransportBase, - DEFAULT_BUFFER, -) - -def to_bytes(s): - try: - return s.encode("utf-8") - except Exception: - return s - - -cdef class TCyMemoryBuffer(CyTransportBase): - cdef TCyBuffer buf - - def __init__(self, value=b'', int buf_size=DEFAULT_BUFFER): - self.trans = None - self.buf = TCyBuffer(buf_size) - - if value: - self.setvalue(value) - - cdef c_read(self, int sz, char* out): - if self.buf.data_size < sz: - sz = self.buf.data_size - - if sz <= 0: - out[0] = '\0' - else: - memcpy(out, self.buf.buf + self.buf.cur, sz) - self.buf.cur += sz - self.buf.data_size -= sz - - return sz - - cdef c_write(self, const char* data, int sz): - cdef int r = self.buf.write(sz, data) - if r == -1: - raise MemoryError("Write to memory error") - - cdef _getvalue(self): - cdef char *out - cdef int size = self.buf.data_size - - if size <= 0: - return b'' - - out = malloc(size) - try: - memcpy(out, self.buf.buf + self.buf.cur, size) - return out[:size] - finally: - free(out) - - cdef _setvalue(self, int sz, const char *value): - self.buf.clean() - self.buf.write(sz, value) - - def read(self, sz): - return self.get_string(sz) - - def write(self, data): - data = to_bytes(data) - - cdef int sz = len(data) - return self.c_write(data, sz) - - def is_open(self): - return True - - def open(self): - pass - - def close(self): - pass - - def flush(self): - pass - - def clean(self): - self.buf.clean() - - def getvalue(self): - return self._getvalue() - - def setvalue(self, value): - value = to_bytes(value) - self._setvalue(len(value), value) diff --git a/python/helpers/pydev/thriftpy/transport/socket.py b/python/helpers/pydev/thriftpy/transport/socket.py deleted file mode 100644 index 8e7b068853e5..000000000000 --- a/python/helpers/pydev/thriftpy/transport/socket.py +++ /dev/null @@ -1,217 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import, division - -import errno -import os -import socket -import struct -import sys - -from . import TTransportException - - -class TSocket(object): - """Socket implementation for client side.""" - - def __init__(self, host=None, port=None, unix_socket=None, - sock=None, socket_family=socket.AF_INET, - socket_timeout=3000, connect_timeout=None): - """Initialize a TSocket - - TSocket can be initialized in 3 ways: - * host + port. can configure to use AF_INET/AF_INET6 - * unix_socket - * socket. should pass already opened socket here. - - @param host(str) The host to connect to. - @param port(int) The (TCP) port to connect to. - @param unix_socket(str) The filename of a unix socket to connect to. - @param sock(socket) Initialize with opened socket directly. - If this param used, the host, port and unix_socket params will - be ignored. - @param socket_family(str) socket.AF_INET or socket.AF_INET6. only - take effect when using host/port - @param socket_timeout socket timeout in ms - @param connect_timeout connect timeout in ms, only used in - connection, will be set to socket_timeout if not set. - """ - if sock: - self.sock = sock - elif unix_socket: - self.unix_socket = unix_socket - self.host = None - self.port = None - self.sock = None - else: - self.unix_socket = None - self.host = host - self.port = port - self.sock = None - - self.socket_family = socket_family - self.socket_timeout = socket_timeout / 1000 if socket_timeout else None - self.connect_timeout = connect_timeout / 1000 if connect_timeout \ - else self.socket_timeout - - def _init_sock(self): - if self.unix_socket: - _sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - else: - _sock = socket.socket(self.socket_family, socket.SOCK_STREAM) - _sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - # socket options - linger = struct.pack('ii', 0, 0) - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger) - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - - self.sock = _sock - - def set_handle(self, sock): - self.sock = sock - - def set_timeout(self, ms): - """Backward compat api, will bind the timeout to both connect_timeout - and socket_timeout. - """ - self.socket_timeout = ms / 1000 if (ms and ms > 0) else None - self.connect_timeout = self.socket_timeout - - if self.sock is not None: - self.sock.settimeout(self.socket_timeout) - - def is_open(self): - return bool(self.sock) - - def open(self): - self._init_sock() - - addr = self.unix_socket or (self.host, self.port) - - try: - if self.connect_timeout: - self.sock.settimeout(self.connect_timeout) - - self.sock.connect(addr) - - if self.socket_timeout: - self.sock.settimeout(self.socket_timeout) - - except (socket.error, OSError): - raise TTransportException( - type=TTransportException.NOT_OPEN, - message="Could not connect to %s" % str(addr)) - - def read(self, sz): - try: - buff = self.sock.recv(sz) - except socket.error as e: - if (e.args[0] == errno.ECONNRESET and - (sys.platform == 'darwin' or - sys.platform.startswith('freebsd'))): - # freebsd and Mach don't follow POSIX semantic of recv - # and fail with ECONNRESET if peer performed shutdown. - # See corresponding comment and code in TSocket::read() - # in lib/cpp/src/transport/TSocket.cpp. - self.close() - # Trigger the check to raise the END_OF_FILE exception below. - buff = '' - else: - raise - - if len(buff) == 0: - raise TTransportException(type=TTransportException.END_OF_FILE, - message='TSocket read 0 bytes') - return buff - - def write(self, buff): - self.sock.sendall(buff) - - def flush(self): - pass - - def close(self): - if not self.sock: - return - - try: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - except (socket.error, OSError): - pass - - -class TServerSocket(object): - """Socket implementation for server side.""" - - def __init__(self, host=None, port=None, unix_socket=None, - socket_family=socket.AF_INET, client_timeout=3000, - backlog=128): - """Initialize a TServerSocket - - TSocket can be initialized in 2 ways: - * host + port. can configure to use AF_INET/AF_INET6 - * unix_socket - - @param host(str) The host to connect to - @param port(int) The (TCP) port to connect to - @param unix_socket(str) The filename of a unix socket to connect to - @param socket_family(str) socket.AF_INET or socket.AF_INET6. only - take effect when using host/port - @param client_timeout client socket timeout - @param backlog backlog for server socket - """ - - if unix_socket: - self.unix_socket = unix_socket - self.host = None - self.port = None - else: - self.unix_socket = None - self.host = host - self.port = port - - self.socket_family = socket_family - self.client_timeout = client_timeout / 1000 if client_timeout else None - self.backlog = backlog - - def _init_sock(self): - if self.unix_socket: - # try remove the sock file it already exists - _sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - _sock.connect(self.unix_socket) - except (socket.error, OSError) as err: - if err.args[0] == errno.ECONNREFUSED: - os.unlink(self.unix_socket) - else: - _sock = socket.socket(self.socket_family, socket.SOCK_STREAM) - - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if hasattr(socket, "SO_REUSEPORT"): - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - _sock.settimeout(None) - self.sock = _sock - - def listen(self): - self._init_sock() - - addr = self.unix_socket or (self.host, self.port) - self.sock.bind(addr) - self.sock.listen(self.backlog) - - def accept(self): - client, _ = self.sock.accept() - client.settimeout(self.client_timeout) - return TSocket(sock=client) - - def close(self): - if not self.sock: - return - - try: - self.sock.shutdown(socket.SHUT_RDWR) - self.sock.close() - except (socket.error, OSError): - pass diff --git a/python/helpers/pydev/thriftpy/transport/sslsocket.py b/python/helpers/pydev/thriftpy/transport/sslsocket.py deleted file mode 100644 index f00f0be94467..000000000000 --- a/python/helpers/pydev/thriftpy/transport/sslsocket.py +++ /dev/null @@ -1,122 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import os -import socket -import ssl -import struct - -from ._ssl import ( - create_thriftpy_context, - RESTRICTED_SERVER_CIPHERS, - DEFAULT_CIPHERS -) -from .socket import TSocket, TServerSocket - - -class TSSLSocket(TSocket): - """SSL socket implementation for client side - """ - - def __init__(self, host, port, socket_family=socket.AF_INET, - socket_timeout=3000, connect_timeout=None, - ssl_context=None, validate=True, - cafile=None, capath=None, certfile=None, keyfile=None, - ciphers=DEFAULT_CIPHERS): - """Initialize a TSSLSocket - - @param validate(bool) Set to False to disable SSL certificate - validation and hostname validation. Default enabled. - @param cafile(str) Path to a file of concatenated CA - certificates in PEM format. - @param capath(str) path to a directory containing several CA - certificates in PEM format, following an OpenSSL specific layout. - @param certfile(str) The certfile string must be the path to a - single file in PEM format containing the certificate as well as - any number of CA certificates needed to establish the - certificate’s authenticity. - @param keyfile(str) The keyfile string, if not present, - the private key will be taken from certfile as well. - @param ciphers(list) The cipher suites to allow - @param ssl_context(SSLContext) Customize the SSLContext, can be used - to persist SSLContext object. Caution it's easy to get wrong, only - use if you know what you're doing. - - The `host` must be the same with server if validate enabled. - """ - super(TSSLSocket, self).__init__( - host=host, port=port, socket_family=socket_family, - connect_timeout=connect_timeout, socket_timeout=socket_timeout) - - if ssl_context: - self.ssl_context = ssl_context - else: - self.ssl_context = create_thriftpy_context(server_side=False, - ciphers=ciphers) - - if cafile or capath: - self.ssl_context.load_verify_locations(cafile=cafile, - capath=capath) - - if certfile: - self.ssl_context.load_cert_chain(certfile, keyfile=keyfile) - - if not validate: - self.ssl_context.check_hostname = False - self.ssl_context.verify_mode = ssl.CERT_NONE - - def _init_sock(self): - _sock = socket.socket(self.socket_family, socket.SOCK_STREAM) - _sock = self.ssl_context.wrap_socket(_sock, - server_hostname=self.host) - # socket options - linger = struct.pack('ii', 0, 0) - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger) - _sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) - _sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - self.sock = _sock - - -class TSSLServerSocket(TServerSocket): - """SSL implementation of TServerSocket - """ - - def __init__(self, host, port, socket_family=socket.AF_INET, - client_timeout=3000, backlog=128, - ssl_context=None, certfile='cert.pem', - ciphers=RESTRICTED_SERVER_CIPHERS): - """Initialize a TSSLServerSocket - - @param certfile(str) The server cert pem filename - @param ciphers(list) The cipher suites to allow - @param ssl_context(SSLContext) Customize the SSLContext, can be used - to persist SSLContext object. Caution it's easy to get wrong, only - use if you know what you're doing. - """ - super(TSSLServerSocket, self).__init__( - host=host, port=port, socket_family=socket_family, - client_timeout=client_timeout, backlog=backlog) - - if ssl_context: - self.ssl_context = ssl_context - else: - if not os.access(certfile, os.R_OK): - raise IOError('No such certfile found: %s' % certfile) - - self.ssl_context = create_thriftpy_context(server_side=True, - ciphers=ciphers) - self.ssl_context.load_cert_chain(certfile=certfile) - - def accept(self): - sock, _ = self.sock.accept() - try: - ssl_sock = self.ssl_context.wrap_socket(sock, server_side=True) - except ssl.SSLError: - # failed handshake/ssl wrap, close socket to client - sock.shutdown(socket.SHUT_RDWR) - sock.close() - raise - else: - ssl_sock.settimeout(self.client_timeout) - return TSocket(sock=ssl_sock) diff --git a/python/helpers/pydev/thriftpy/utils.py b/python/helpers/pydev/thriftpy/utils.py deleted file mode 100644 index 8271a8602192..000000000000 --- a/python/helpers/pydev/thriftpy/utils.py +++ /dev/null @@ -1,37 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import absolute_import - -import binascii - -from .protocol.binary import TBinaryProtocolFactory -from .transport import TMemoryBuffer - - -def serialize(thrift_object, proto_factory=TBinaryProtocolFactory()): - transport = TMemoryBuffer() - protocol = proto_factory.get_protocol(transport) - thrift_object.write(protocol) - protocol.write_message_end() - return transport.getvalue() - - -def deserialize(thrift_object, buf, proto_factory=TBinaryProtocolFactory()): - transport = TMemoryBuffer(buf) - protocol = proto_factory.get_protocol(transport) - thrift_object.read(protocol) - return thrift_object - - -def hexlify(byte_array, delimeter=' '): - s = binascii.hexlify(byte_array).decode('utf-8') - return delimeter.join(a+b for a, b in zip(s[::2], s[1::2])) - - -def hexprint(byte_array, delimeter=' ', count=10): - print("Bytes:") - print(byte_array) - - print("\nHex:") - g = hexlify(byte_array, delimeter).split(delimeter) - print('\n'.join(' '.join(g[i:i+10]) for i in range(0, len(g), 10))) diff --git a/python/helpers/third_party/thriftpy/README.md b/python/helpers/third_party/thriftpy/README.md new file mode 100644 index 000000000000..be7d61d47b51 --- /dev/null +++ b/python/helpers/third_party/thriftpy/README.md @@ -0,0 +1 @@ +#Thriftpy with dependencies \ No newline at end of file diff --git a/python/helpers/profiler/ply/__init__.py b/python/helpers/third_party/thriftpy/ply/__init__.py similarity index 100% rename from python/helpers/profiler/ply/__init__.py rename to python/helpers/third_party/thriftpy/ply/__init__.py diff --git a/python/helpers/profiler/ply/cpp.py b/python/helpers/third_party/thriftpy/ply/cpp.py similarity index 100% rename from python/helpers/profiler/ply/cpp.py rename to python/helpers/third_party/thriftpy/ply/cpp.py diff --git a/python/helpers/profiler/ply/ctokens.py b/python/helpers/third_party/thriftpy/ply/ctokens.py similarity index 100% rename from python/helpers/profiler/ply/ctokens.py rename to python/helpers/third_party/thriftpy/ply/ctokens.py diff --git a/python/helpers/profiler/ply/lex.py b/python/helpers/third_party/thriftpy/ply/lex.py similarity index 100% rename from python/helpers/profiler/ply/lex.py rename to python/helpers/third_party/thriftpy/ply/lex.py diff --git a/python/helpers/profiler/ply/yacc.py b/python/helpers/third_party/thriftpy/ply/yacc.py similarity index 100% rename from python/helpers/profiler/ply/yacc.py rename to python/helpers/third_party/thriftpy/ply/yacc.py diff --git a/python/helpers/profiler/ply/ygen.py b/python/helpers/third_party/thriftpy/ply/ygen.py similarity index 100% rename from python/helpers/profiler/ply/ygen.py rename to python/helpers/third_party/thriftpy/ply/ygen.py diff --git a/python/helpers/profiler/thriftpy/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/__init__.py diff --git a/python/helpers/profiler/thriftpy/_compat.py b/python/helpers/third_party/thriftpy/thriftpy/_compat.py similarity index 100% rename from python/helpers/profiler/thriftpy/_compat.py rename to python/helpers/third_party/thriftpy/thriftpy/_compat.py diff --git a/python/helpers/profiler/thriftpy/contrib/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/contrib/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/contrib/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/contrib/__init__.py diff --git a/python/helpers/profiler/thriftpy/contrib/tracking/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/contrib/tracking/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/__init__.py diff --git a/python/helpers/profiler/thriftpy/contrib/tracking/tracker.py b/python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/tracker.py similarity index 100% rename from python/helpers/profiler/thriftpy/contrib/tracking/tracker.py rename to python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/tracker.py diff --git a/python/helpers/profiler/thriftpy/contrib/tracking/tracking.thrift b/python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/tracking.thrift similarity index 100% rename from python/helpers/profiler/thriftpy/contrib/tracking/tracking.thrift rename to python/helpers/third_party/thriftpy/thriftpy/contrib/tracking/tracking.thrift diff --git a/python/helpers/profiler/thriftpy/hook.py b/python/helpers/third_party/thriftpy/thriftpy/hook.py similarity index 100% rename from python/helpers/profiler/thriftpy/hook.py rename to python/helpers/third_party/thriftpy/thriftpy/hook.py diff --git a/python/helpers/profiler/thriftpy/parser/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/parser/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/parser/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/parser/__init__.py diff --git a/python/helpers/profiler/thriftpy/parser/exc.py b/python/helpers/third_party/thriftpy/thriftpy/parser/exc.py similarity index 100% rename from python/helpers/profiler/thriftpy/parser/exc.py rename to python/helpers/third_party/thriftpy/thriftpy/parser/exc.py diff --git a/python/helpers/profiler/thriftpy/parser/lexer.py b/python/helpers/third_party/thriftpy/thriftpy/parser/lexer.py similarity index 100% rename from python/helpers/profiler/thriftpy/parser/lexer.py rename to python/helpers/third_party/thriftpy/thriftpy/parser/lexer.py diff --git a/python/helpers/profiler/thriftpy/parser/parser.py b/python/helpers/third_party/thriftpy/thriftpy/parser/parser.py similarity index 100% rename from python/helpers/profiler/thriftpy/parser/parser.py rename to python/helpers/third_party/thriftpy/thriftpy/parser/parser.py diff --git a/python/helpers/profiler/thriftpy/protocol/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/__init__.py diff --git a/python/helpers/profiler/thriftpy/protocol/binary.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/binary.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/binary.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/binary.py diff --git a/python/helpers/profiler/thriftpy/protocol/compact.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/compact.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/compact.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/compact.py diff --git a/python/helpers/profiler/thriftpy/protocol/cybin/cybin.pyx b/python/helpers/third_party/thriftpy/thriftpy/protocol/cybin/cybin.pyx similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/cybin/cybin.pyx rename to python/helpers/third_party/thriftpy/thriftpy/protocol/cybin/cybin.pyx diff --git a/python/helpers/profiler/thriftpy/protocol/cybin/endian_port.h b/python/helpers/third_party/thriftpy/thriftpy/protocol/cybin/endian_port.h similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/cybin/endian_port.h rename to python/helpers/third_party/thriftpy/thriftpy/protocol/cybin/endian_port.h diff --git a/python/helpers/profiler/thriftpy/protocol/exc.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/exc.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/exc.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/exc.py diff --git a/python/helpers/profiler/thriftpy/protocol/json.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/json.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/json.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/json.py diff --git a/python/helpers/profiler/thriftpy/protocol/multiplex.py b/python/helpers/third_party/thriftpy/thriftpy/protocol/multiplex.py similarity index 100% rename from python/helpers/profiler/thriftpy/protocol/multiplex.py rename to python/helpers/third_party/thriftpy/thriftpy/protocol/multiplex.py diff --git a/python/helpers/profiler/thriftpy/rpc.py b/python/helpers/third_party/thriftpy/thriftpy/rpc.py similarity index 100% rename from python/helpers/profiler/thriftpy/rpc.py rename to python/helpers/third_party/thriftpy/thriftpy/rpc.py diff --git a/python/helpers/profiler/thriftpy/server.py b/python/helpers/third_party/thriftpy/thriftpy/server.py similarity index 100% rename from python/helpers/profiler/thriftpy/server.py rename to python/helpers/third_party/thriftpy/thriftpy/server.py diff --git a/python/helpers/profiler/thriftpy/thrift.py b/python/helpers/third_party/thriftpy/thriftpy/thrift.py similarity index 100% rename from python/helpers/profiler/thriftpy/thrift.py rename to python/helpers/third_party/thriftpy/thriftpy/thrift.py diff --git a/python/helpers/profiler/thriftpy/tornado.py b/python/helpers/third_party/thriftpy/thriftpy/tornado.py similarity index 100% rename from python/helpers/profiler/thriftpy/tornado.py rename to python/helpers/third_party/thriftpy/thriftpy/tornado.py diff --git a/python/helpers/profiler/thriftpy/transport/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/transport/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/__init__.py diff --git a/python/helpers/profiler/thriftpy/transport/_ssl.py b/python/helpers/third_party/thriftpy/thriftpy/transport/_ssl.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/_ssl.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/_ssl.py diff --git a/python/helpers/profiler/thriftpy/transport/buffered/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/transport/buffered/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/buffered/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/buffered/__init__.py diff --git a/python/helpers/profiler/thriftpy/transport/buffered/cybuffered.pyx b/python/helpers/third_party/thriftpy/thriftpy/transport/buffered/cybuffered.pyx similarity index 100% rename from python/helpers/profiler/thriftpy/transport/buffered/cybuffered.pyx rename to python/helpers/third_party/thriftpy/thriftpy/transport/buffered/cybuffered.pyx diff --git a/python/helpers/profiler/thriftpy/transport/cybase.pxd b/python/helpers/third_party/thriftpy/thriftpy/transport/cybase.pxd similarity index 100% rename from python/helpers/profiler/thriftpy/transport/cybase.pxd rename to python/helpers/third_party/thriftpy/thriftpy/transport/cybase.pxd diff --git a/python/helpers/profiler/thriftpy/transport/cybase.pyx b/python/helpers/third_party/thriftpy/thriftpy/transport/cybase.pyx similarity index 100% rename from python/helpers/profiler/thriftpy/transport/cybase.pyx rename to python/helpers/third_party/thriftpy/thriftpy/transport/cybase.pyx diff --git a/python/helpers/profiler/thriftpy/transport/framed/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/transport/framed/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/framed/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/framed/__init__.py diff --git a/python/helpers/profiler/thriftpy/transport/framed/cyframed.pyx b/python/helpers/third_party/thriftpy/thriftpy/transport/framed/cyframed.pyx similarity index 100% rename from python/helpers/profiler/thriftpy/transport/framed/cyframed.pyx rename to python/helpers/third_party/thriftpy/thriftpy/transport/framed/cyframed.pyx diff --git a/python/helpers/profiler/thriftpy/transport/memory/__init__.py b/python/helpers/third_party/thriftpy/thriftpy/transport/memory/__init__.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/memory/__init__.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/memory/__init__.py diff --git a/python/helpers/profiler/thriftpy/transport/memory/cymemory.pyx b/python/helpers/third_party/thriftpy/thriftpy/transport/memory/cymemory.pyx similarity index 100% rename from python/helpers/profiler/thriftpy/transport/memory/cymemory.pyx rename to python/helpers/third_party/thriftpy/thriftpy/transport/memory/cymemory.pyx diff --git a/python/helpers/profiler/thriftpy/transport/socket.py b/python/helpers/third_party/thriftpy/thriftpy/transport/socket.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/socket.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/socket.py diff --git a/python/helpers/profiler/thriftpy/transport/sslsocket.py b/python/helpers/third_party/thriftpy/thriftpy/transport/sslsocket.py similarity index 100% rename from python/helpers/profiler/thriftpy/transport/sslsocket.py rename to python/helpers/third_party/thriftpy/thriftpy/transport/sslsocket.py diff --git a/python/helpers/profiler/thriftpy/utils.py b/python/helpers/third_party/thriftpy/thriftpy/utils.py similarity index 100% rename from python/helpers/profiler/thriftpy/utils.py rename to python/helpers/third_party/thriftpy/thriftpy/utils.py diff --git a/python/intellij.python.helpers.iml b/python/intellij.python.helpers.iml index e36f4c1a1055..09aac628ee5d 100644 --- a/python/intellij.python.helpers.iml +++ b/python/intellij.python.helpers.iml @@ -5,6 +5,7 @@ + diff --git a/python/src/com/jetbrains/python/PythonHelper.java b/python/src/com/jetbrains/python/PythonHelper.java index 3c93cb4fe245..4c2dc6adff96 100644 --- a/python/src/com/jetbrains/python/PythonHelper.java +++ b/python/src/com/jetbrains/python/PythonHelper.java @@ -20,12 +20,14 @@ import com.intellij.execution.configurations.GeneralCommandLine; import com.intellij.execution.configurations.ParamsGroup; import com.intellij.openapi.projectRoots.Sdk; import com.intellij.openapi.util.io.FileUtil; +import com.intellij.util.containers.ContainerUtil; import com.jetbrains.python.psi.LanguageLevel; import com.jetbrains.python.sdk.PythonEnvUtil; import com.jetbrains.python.sdk.PythonSdkType; import org.jetbrains.annotations.NotNull; import java.io.File; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -41,12 +43,12 @@ public enum PythonHelper implements HelperPackage { COVERAGEPY("coveragepy", ""), COVERAGE("coverage_runner", "run_coverage"), DEBUGGER("pydev", "pydevd"), - + ATTACH_DEBUGGER("pydev/pydevd_attach_to_process/attach_pydevd.py"), - CONSOLE("pydev", "pydevconsole"), + CONSOLE("pydev", "pydevconsole", HelperDependency.THRIFTPY), RUN_IN_CONSOLE("pydev", "pydev_run_in_console"), - PROFILER("profiler", "run_profiler"), + PROFILER("profiler", "run_profiler", HelperDependency.THRIFTPY), LOAD_ENTRY_POINT("pycharm", "pycharm_load_entry_point"), @@ -91,41 +93,49 @@ public enum PythonHelper implements HelperPackage { public static final String PY2_HELPER_DEPENDENCIES_DIR = "py2only"; @NotNull - private static PathHelperPackage findModule(String moduleEntryPoint, String path, boolean asModule) { + private static PathHelperPackage findModule(String moduleEntryPoint, String path, boolean asModule, String[] thirdPartyDependencies) { + List dependencies = HelperDependency.findThirdPartyDependencies(thirdPartyDependencies); + if (getHelperFile(path + ".zip").isFile()) { - return new ModuleHelperPackage(moduleEntryPoint, path + ".zip"); + return new ModuleHelperPackage(moduleEntryPoint, path + ".zip", dependencies); } if (!asModule && new File(getHelperFile(path), moduleEntryPoint + ".py").isFile()) { - return new ScriptPythonHelper(moduleEntryPoint + ".py", getHelperFile(path)); + return new ScriptPythonHelper(moduleEntryPoint + ".py", getHelperFile(path), dependencies); } - return new ModuleHelperPackage(moduleEntryPoint, path); + return new ModuleHelperPackage(moduleEntryPoint, path, dependencies); } private final PathHelperPackage myModule; - PythonHelper(String pythonPath, String moduleName) { - this(pythonPath, moduleName, false); + PythonHelper(String pythonPath, String moduleName, String... dependencies) { + this(pythonPath, moduleName, false, dependencies); } - PythonHelper(String pythonPath, String moduleName, boolean asModule) { - myModule = findModule(moduleName, pythonPath, asModule); + PythonHelper(String pythonPath, String moduleName, boolean asModule, String... dependencies) { + myModule = findModule(moduleName, pythonPath, asModule, dependencies); } PythonHelper(String helperScript) { - myModule = new ScriptPythonHelper(helperScript, getHelpersRoot()); + myModule = new ScriptPythonHelper(helperScript, getHelpersRoot(), Collections.emptyList()); } public abstract static class PathHelperPackage implements HelperPackage { protected final File myPath; + @NotNull + protected final List myDependencies; - PathHelperPackage(String path) { + PathHelperPackage(String path, @NotNull List dependencies) { myPath = new File(path); + myDependencies = dependencies; } @Override public void addToPythonPath(@NotNull Map environment) { + // at first add dependencies + myDependencies.forEach(dependency -> dependency.addToPythonPath(environment)); + // then add helper script PythonEnvUtil.addToPythonPath(environment, getPythonPathEntry()); } @@ -174,8 +184,8 @@ public enum PythonHelper implements HelperPackage { public static class ModuleHelperPackage extends PathHelperPackage { private final String myModuleName; - public ModuleHelperPackage(String moduleName, String relativePath) { - super(getHelperFile(relativePath).getAbsolutePath()); + public ModuleHelperPackage(String moduleName, String relativePath, @NotNull List dependencies) { + super(getHelperFile(relativePath).getAbsolutePath(), dependencies); this.myModuleName = moduleName; } @@ -200,8 +210,8 @@ public enum PythonHelper implements HelperPackage { public static class ScriptPythonHelper extends PathHelperPackage { private final String myPythonPath; - public ScriptPythonHelper(String script, File pythonPath) { - super(new File(pythonPath, script).getAbsolutePath()); + public ScriptPythonHelper(String script, File pythonPath, @NotNull List dependencies) { + super(new File(pythonPath, script).getAbsolutePath(), dependencies); myPythonPath = pythonPath.getAbsolutePath(); } @@ -218,7 +228,38 @@ public enum PythonHelper implements HelperPackage { } } - + private static class HelperDependency { + private static final String THRIFTPY = "thriftpy"; + + @NotNull + private final String myPythonPath; + + private HelperDependency(@NotNull String pythonPath) {myPythonPath = pythonPath;} + + public void addToPythonPath(@NotNull Map environment) { + PythonEnvUtil.addToPythonPath(environment, myPythonPath); + } + + @NotNull + private static List findThirdPartyDependencies(String... dependencies) { + if (dependencies == null) { + return Collections.emptyList(); + } + return ContainerUtil.map(dependencies, s -> getThirdPartyDependency(s)); + } + + @NotNull + private static HelperDependency getThirdPartyDependency(@NotNull String name) { + String path = new File(getHelpersThirdPartyDir(), name).getAbsolutePath(); + return new HelperDependency(path); + } + + @NotNull + private static File getHelpersThirdPartyDir() { + return getHelperFile("third_party"); + } + } + @NotNull @Override public String getPythonPathEntry() { @@ -252,5 +293,4 @@ public enum PythonHelper implements HelperPackage { public GeneralCommandLine newCommandLine(@NotNull Sdk pythonSdk, @NotNull List parameters) { return myModule.newCommandLine(pythonSdk, parameters); } - }