diff --git a/python/helpers/profiler/__init__.py b/python/helpers/profiler/__init__.py new file mode 100644 index 000000000000..595e3818d699 --- /dev/null +++ b/python/helpers/profiler/__init__.py @@ -0,0 +1 @@ +__author__ = 'traff' diff --git a/python/helpers/profiler/prof_io.py b/python/helpers/profiler/prof_io.py new file mode 100644 index 000000000000..6f7848babacc --- /dev/null +++ b/python/helpers/profiler/prof_io.py @@ -0,0 +1,132 @@ +from socket import SHUT_WR +import sys +import time +import traceback + +from profiler_protocol_pb2 import ProfilerRequest + + +__author__ = 'traff' + +from prof_util import ProfDaemonThread + +import struct + +from pydev_imports import _queue +import pydevd_vm_type + + +def send_message(sock, message): + """ Send a serialized message (protobuf Message interface) + to a socket, prepended by its length packed in 4 + bytes (big endian). + """ + s = message.SerializeToString() + packed_len = struct.pack('>L', len(s)) + sock.sendall(packed_len + s) + + +def get_message(sock, msgtype): + """ Read a message from a socket. msgtype is a subclass of + of protobuf Message. + """ + len_buf = socket_read_n(sock, 4) + msg_len = struct.unpack('>L', len_buf)[0] + msg_buf = socket_read_n(sock, msg_len) + + msg = msgtype() + msg.ParseFromString(msg_buf) + return msg + + +def socket_read_n(sock, n): + """ Read exactly n bytes from the socket. + Raise RuntimeError if the connection closed before + n bytes were read. + """ + buf = '' + while n > 0: + data = sock.recv(n) + if data == '': + raise RuntimeError('unexpected connection close') + buf += data + n -= len(data) + return buf + + +class ProfWriter(ProfDaemonThread): + """ writer thread writes out the commands in an infinite loop """ + def __init__(self, sock): + ProfDaemonThread.__init__(self) + self.sock = sock + self.setName("profiler.Writer") + self.messageQueue = _queue.Queue() + if pydevd_vm_type.GetVmType() == 'python': + self.timeout = 0 + else: + self.timeout = 0.1 + + def addCommand(self, message): + """ message is NetCommand """ + if not self.killReceived: #we don't take new data after everybody die + self.messageQueue.put(message) + + def OnRun(self): + """ just loop and write responses """ + + get_has_timeout = sys.hexversion >= 0x02030000 # 2.3 onwards have it. + try: + while True: + try: + try: + if get_has_timeout: + message = self.messageQueue.get(1, 0.1) + else: + time.sleep(.01) + message = self.messageQueue.get(0) + except _queue.Empty: + if self.killReceived: + try: + self.sock.shutdown(SHUT_WR) + self.sock.close() + except: + pass + + return #break if queue is empty and killReceived + else: + continue + except: + return + + send_message(self.sock, message) + + time.sleep(self.timeout) + except Exception: + traceback.print_exc() + + +class ProfReader(ProfDaemonThread): + """ reader thread reads and dispatches commands in an infinite loop """ + + def __init__(self, sock, message_processor): + ProfDaemonThread.__init__(self) + self.sock = sock + self.processor = message_processor + self.setName("profiler.Reader") + + def OnRun(self): + try: + while not self.killReceived: + try: + message = get_message(self.sock, ProfilerRequest) + except: + traceback.print_exc() + return # Finished communication. + + try: + self.processor.process(message) + except: + traceback.print_exc() + + except: + traceback.print_exc() \ No newline at end of file diff --git a/python/helpers/profiler/prof_util.py b/python/helpers/profiler/prof_util.py new file mode 100644 index 000000000000..03c15317112c --- /dev/null +++ b/python/helpers/profiler/prof_util.py @@ -0,0 +1,15 @@ +__author__ = 'traff' + +import threading + +class ProfDaemonThread(threading.Thread): + def __init__(self): + super(ProfDaemonThread, self).__init__() + self.setDaemon(True) + self.killReceived = False + + def run(self): + self.OnRun() + + def OnRun(self): + pass \ No newline at end of file diff --git a/python/helpers/profiler/profiler_protocol_pb2.py b/python/helpers/profiler/profiler_protocol_pb2.py new file mode 100644 index 000000000000..382cd52f1641 --- /dev/null +++ b/python/helpers/profiler/profiler_protocol_pb2.py @@ -0,0 +1,333 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: profiler_protocol.proto + +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +# from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='profiler_protocol.proto', + package='com.jetbrains.python.profiler', + serialized_pb='\n\x17profiler_protocol.proto\x12\x1d\x63om.jetbrains.python.profiler\"\xad\x02\n\x06YStats\x12\x43\n\nfunc_stats\x18\x01 \x03(\x0b\x32/.com.jetbrains.python.profiler.YStats.FuncStats\x12G\n\x0cthread_stats\x18\x02 \x03(\x0b\x32\x31.com.jetbrains.python.profiler.YStats.ThreadStats\x1a\x36\n\tFuncStats\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05ncall\x18\x02 \x02(\x05\x12\x0c\n\x04ttot\x18\x03 \x02(\x05\x1a]\n\x0bThreadStats\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\n\n\x02id\x18\x02 \x02(\t\x12\x11\n\tlast_func\x18\x03 \x02(\t\x12\x0c\n\x04ttot\x18\x04 \x02(\x05\x12\x13\n\x0bsched_count\x18\x05 \x02(\x05\"\x0c\n\nYStats_Req\"\x12\n\x10YStatsString_Req\"\xa0\x01\n\x0fProfilerRequest\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x39\n\x06ystats\x18\x02 \x01(\x0b\x32).com.jetbrains.python.profiler.YStats_Req\x12\x46\n\rystats_string\x18\x03 \x01(\x0b\x32/.com.jetbrains.python.profiler.YStatsString_Req\"l\n\x10ProfilerResponse\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x35\n\x06ystats\x18\x02 \x01(\x0b\x32%.com.jetbrains.python.profiler.YStats\x12\x15\n\rystats_string\x18\x03 \x01(\t') + + + + +_YSTATS_FUNCSTATS = _descriptor.Descriptor( + name='FuncStats', + full_name='com.jetbrains.python.profiler.YStats.FuncStats', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='com.jetbrains.python.profiler.YStats.FuncStats.name', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ncall', full_name='com.jetbrains.python.profiler.YStats.FuncStats.ncall', index=1, + number=2, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ttot', full_name='com.jetbrains.python.profiler.YStats.FuncStats.ttot', index=2, + number=3, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=211, + serialized_end=265, +) + +_YSTATS_THREADSTATS = _descriptor.Descriptor( + name='ThreadStats', + full_name='com.jetbrains.python.profiler.YStats.ThreadStats', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.name', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='id', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.id', index=1, + number=2, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='last_func', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.last_func', index=2, + number=3, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ttot', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.ttot', index=3, + number=4, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sched_count', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.sched_count', index=4, + number=5, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=267, + serialized_end=360, +) + +_YSTATS = _descriptor.Descriptor( + name='YStats', + full_name='com.jetbrains.python.profiler.YStats', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='func_stats', full_name='com.jetbrains.python.profiler.YStats.func_stats', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='thread_stats', full_name='com.jetbrains.python.profiler.YStats.thread_stats', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_YSTATS_FUNCSTATS, _YSTATS_THREADSTATS, ], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=59, + serialized_end=360, +) + + +_YSTATS_REQ = _descriptor.Descriptor( + name='YStats_Req', + full_name='com.jetbrains.python.profiler.YStats_Req', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=362, + serialized_end=374, +) + + +_YSTATSSTRING_REQ = _descriptor.Descriptor( + name='YStatsString_Req', + full_name='com.jetbrains.python.profiler.YStatsString_Req', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=376, + serialized_end=394, +) + + +_PROFILERREQUEST = _descriptor.Descriptor( + name='ProfilerRequest', + full_name='com.jetbrains.python.profiler.ProfilerRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='com.jetbrains.python.profiler.ProfilerRequest.id', index=0, + number=1, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ystats', full_name='com.jetbrains.python.profiler.ProfilerRequest.ystats', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ystats_string', full_name='com.jetbrains.python.profiler.ProfilerRequest.ystats_string', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=397, + serialized_end=557, +) + + +_PROFILERRESPONSE = _descriptor.Descriptor( + name='ProfilerResponse', + full_name='com.jetbrains.python.profiler.ProfilerResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='id', full_name='com.jetbrains.python.profiler.ProfilerResponse.id', index=0, + number=1, type=5, cpp_type=1, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ystats', full_name='com.jetbrains.python.profiler.ProfilerResponse.ystats', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='ystats_string', full_name='com.jetbrains.python.profiler.ProfilerResponse.ystats_string', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=559, + serialized_end=667, +) + +_YSTATS_FUNCSTATS.containing_type = _YSTATS; +_YSTATS_THREADSTATS.containing_type = _YSTATS; +_YSTATS.fields_by_name['func_stats'].message_type = _YSTATS_FUNCSTATS +_YSTATS.fields_by_name['thread_stats'].message_type = _YSTATS_THREADSTATS +_PROFILERREQUEST.fields_by_name['ystats'].message_type = _YSTATS_REQ +_PROFILERREQUEST.fields_by_name['ystats_string'].message_type = _YSTATSSTRING_REQ +_PROFILERRESPONSE.fields_by_name['ystats'].message_type = _YSTATS +DESCRIPTOR.message_types_by_name['YStats'] = _YSTATS +DESCRIPTOR.message_types_by_name['YStats_Req'] = _YSTATS_REQ +DESCRIPTOR.message_types_by_name['YStatsString_Req'] = _YSTATSSTRING_REQ +DESCRIPTOR.message_types_by_name['ProfilerRequest'] = _PROFILERREQUEST +DESCRIPTOR.message_types_by_name['ProfilerResponse'] = _PROFILERRESPONSE + +class YStats(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + + class FuncStats(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _YSTATS_FUNCSTATS + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats.FuncStats) + + class ThreadStats(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _YSTATS_THREADSTATS + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats.ThreadStats) + DESCRIPTOR = _YSTATS + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats) + +class YStats_Req(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _YSTATS_REQ + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats_Req) + +class YStatsString_Req(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _YSTATSSTRING_REQ + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStatsString_Req) + +class ProfilerRequest(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _PROFILERREQUEST + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.ProfilerRequest) + +class ProfilerResponse(_message.Message): + __metaclass__ = _reflection.GeneratedProtocolMessageType + DESCRIPTOR = _PROFILERRESPONSE + + # @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.ProfilerResponse) + + +# @@protoc_insertion_point(module_scope) diff --git a/python/helpers/profiler/run_profiler.py b/python/helpers/profiler/run_profiler.py new file mode 100644 index 000000000000..ddd3c9acebea --- /dev/null +++ b/python/helpers/profiler/run_profiler.py @@ -0,0 +1,115 @@ +from socket import AF_INET +from socket import SOCK_STREAM +from socket import socket +import time + +import yappi + +from prof_io import ProfWriter, ProfReader +from pydevd_utils import save_main_module +import pydev_imports +from profiler_protocol_pb2 import ProfilerResponse + + +__author__ = 'traff' + +import sys +import traceback + + +def StartClient(host, port): + """ connects to a host/port """ + + s = socket(AF_INET, SOCK_STREAM) + + MAX_TRIES = 100 + i = 0 + while i