Python profiler support prototype: yappi and PyCharm bound together with protobuf

This commit is contained in:
Dmitry Trofimov
2015-01-20 15:38:40 +01:00
parent f59e542201
commit 556d72686a
5 changed files with 596 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
__author__ = 'traff'
+132
View File
@@ -0,0 +1,132 @@
from socket import SHUT_WR
import sys
import time
import traceback
from profiler_protocol_pb2 import ProfilerRequest
__author__ = 'traff'
from prof_util import ProfDaemonThread
import struct
from pydev_imports import _queue
import pydevd_vm_type
def send_message(sock, message):
""" Send a serialized message (protobuf Message interface)
to a socket, prepended by its length packed in 4
bytes (big endian).
"""
s = message.SerializeToString()
packed_len = struct.pack('>L', len(s))
sock.sendall(packed_len + s)
def get_message(sock, msgtype):
""" Read a message from a socket. msgtype is a subclass of
of protobuf Message.
"""
len_buf = socket_read_n(sock, 4)
msg_len = struct.unpack('>L', len_buf)[0]
msg_buf = socket_read_n(sock, msg_len)
msg = msgtype()
msg.ParseFromString(msg_buf)
return msg
def socket_read_n(sock, n):
""" Read exactly n bytes from the socket.
Raise RuntimeError if the connection closed before
n bytes were read.
"""
buf = ''
while n > 0:
data = sock.recv(n)
if data == '':
raise RuntimeError('unexpected connection close')
buf += data
n -= len(data)
return buf
class ProfWriter(ProfDaemonThread):
""" writer thread writes out the commands in an infinite loop """
def __init__(self, sock):
ProfDaemonThread.__init__(self)
self.sock = sock
self.setName("profiler.Writer")
self.messageQueue = _queue.Queue()
if pydevd_vm_type.GetVmType() == 'python':
self.timeout = 0
else:
self.timeout = 0.1
def addCommand(self, message):
""" message is NetCommand """
if not self.killReceived: #we don't take new data after everybody die
self.messageQueue.put(message)
def OnRun(self):
""" just loop and write responses """
get_has_timeout = sys.hexversion >= 0x02030000 # 2.3 onwards have it.
try:
while True:
try:
try:
if get_has_timeout:
message = self.messageQueue.get(1, 0.1)
else:
time.sleep(.01)
message = self.messageQueue.get(0)
except _queue.Empty:
if self.killReceived:
try:
self.sock.shutdown(SHUT_WR)
self.sock.close()
except:
pass
return #break if queue is empty and killReceived
else:
continue
except:
return
send_message(self.sock, message)
time.sleep(self.timeout)
except Exception:
traceback.print_exc()
class ProfReader(ProfDaemonThread):
""" reader thread reads and dispatches commands in an infinite loop """
def __init__(self, sock, message_processor):
ProfDaemonThread.__init__(self)
self.sock = sock
self.processor = message_processor
self.setName("profiler.Reader")
def OnRun(self):
try:
while not self.killReceived:
try:
message = get_message(self.sock, ProfilerRequest)
except:
traceback.print_exc()
return # Finished communication.
try:
self.processor.process(message)
except:
traceback.print_exc()
except:
traceback.print_exc()
+15
View File
@@ -0,0 +1,15 @@
__author__ = 'traff'
import threading
class ProfDaemonThread(threading.Thread):
def __init__(self):
super(ProfDaemonThread, self).__init__()
self.setDaemon(True)
self.killReceived = False
def run(self):
self.OnRun()
def OnRun(self):
pass
@@ -0,0 +1,333 @@
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: profiler_protocol.proto
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
# from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
DESCRIPTOR = _descriptor.FileDescriptor(
name='profiler_protocol.proto',
package='com.jetbrains.python.profiler',
serialized_pb='\n\x17profiler_protocol.proto\x12\x1d\x63om.jetbrains.python.profiler\"\xad\x02\n\x06YStats\x12\x43\n\nfunc_stats\x18\x01 \x03(\x0b\x32/.com.jetbrains.python.profiler.YStats.FuncStats\x12G\n\x0cthread_stats\x18\x02 \x03(\x0b\x32\x31.com.jetbrains.python.profiler.YStats.ThreadStats\x1a\x36\n\tFuncStats\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\r\n\x05ncall\x18\x02 \x02(\x05\x12\x0c\n\x04ttot\x18\x03 \x02(\x05\x1a]\n\x0bThreadStats\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\n\n\x02id\x18\x02 \x02(\t\x12\x11\n\tlast_func\x18\x03 \x02(\t\x12\x0c\n\x04ttot\x18\x04 \x02(\x05\x12\x13\n\x0bsched_count\x18\x05 \x02(\x05\"\x0c\n\nYStats_Req\"\x12\n\x10YStatsString_Req\"\xa0\x01\n\x0fProfilerRequest\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x39\n\x06ystats\x18\x02 \x01(\x0b\x32).com.jetbrains.python.profiler.YStats_Req\x12\x46\n\rystats_string\x18\x03 \x01(\x0b\x32/.com.jetbrains.python.profiler.YStatsString_Req\"l\n\x10ProfilerResponse\x12\n\n\x02id\x18\x01 \x02(\x05\x12\x35\n\x06ystats\x18\x02 \x01(\x0b\x32%.com.jetbrains.python.profiler.YStats\x12\x15\n\rystats_string\x18\x03 \x01(\t')
_YSTATS_FUNCSTATS = _descriptor.Descriptor(
name='FuncStats',
full_name='com.jetbrains.python.profiler.YStats.FuncStats',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='com.jetbrains.python.profiler.YStats.FuncStats.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=unicode("", "utf-8"),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ncall', full_name='com.jetbrains.python.profiler.YStats.FuncStats.ncall', index=1,
number=2, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ttot', full_name='com.jetbrains.python.profiler.YStats.FuncStats.ttot', index=2,
number=3, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=211,
serialized_end=265,
)
_YSTATS_THREADSTATS = _descriptor.Descriptor(
name='ThreadStats',
full_name='com.jetbrains.python.profiler.YStats.ThreadStats',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='name', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.name', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=unicode("", "utf-8"),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='id', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.id', index=1,
number=2, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=unicode("", "utf-8"),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='last_func', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.last_func', index=2,
number=3, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=unicode("", "utf-8"),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ttot', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.ttot', index=3,
number=4, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='sched_count', full_name='com.jetbrains.python.profiler.YStats.ThreadStats.sched_count', index=4,
number=5, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=267,
serialized_end=360,
)
_YSTATS = _descriptor.Descriptor(
name='YStats',
full_name='com.jetbrains.python.profiler.YStats',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='func_stats', full_name='com.jetbrains.python.profiler.YStats.func_stats', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='thread_stats', full_name='com.jetbrains.python.profiler.YStats.thread_stats', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[_YSTATS_FUNCSTATS, _YSTATS_THREADSTATS, ],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=59,
serialized_end=360,
)
_YSTATS_REQ = _descriptor.Descriptor(
name='YStats_Req',
full_name='com.jetbrains.python.profiler.YStats_Req',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=362,
serialized_end=374,
)
_YSTATSSTRING_REQ = _descriptor.Descriptor(
name='YStatsString_Req',
full_name='com.jetbrains.python.profiler.YStatsString_Req',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=376,
serialized_end=394,
)
_PROFILERREQUEST = _descriptor.Descriptor(
name='ProfilerRequest',
full_name='com.jetbrains.python.profiler.ProfilerRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='com.jetbrains.python.profiler.ProfilerRequest.id', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ystats', full_name='com.jetbrains.python.profiler.ProfilerRequest.ystats', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ystats_string', full_name='com.jetbrains.python.profiler.ProfilerRequest.ystats_string', index=2,
number=3, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=397,
serialized_end=557,
)
_PROFILERRESPONSE = _descriptor.Descriptor(
name='ProfilerResponse',
full_name='com.jetbrains.python.profiler.ProfilerResponse',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='com.jetbrains.python.profiler.ProfilerResponse.id', index=0,
number=1, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ystats', full_name='com.jetbrains.python.profiler.ProfilerResponse.ystats', index=1,
number=2, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
_descriptor.FieldDescriptor(
name='ystats_string', full_name='com.jetbrains.python.profiler.ProfilerResponse.ystats_string', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=unicode("", "utf-8"),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
is_extendable=False,
extension_ranges=[],
serialized_start=559,
serialized_end=667,
)
_YSTATS_FUNCSTATS.containing_type = _YSTATS;
_YSTATS_THREADSTATS.containing_type = _YSTATS;
_YSTATS.fields_by_name['func_stats'].message_type = _YSTATS_FUNCSTATS
_YSTATS.fields_by_name['thread_stats'].message_type = _YSTATS_THREADSTATS
_PROFILERREQUEST.fields_by_name['ystats'].message_type = _YSTATS_REQ
_PROFILERREQUEST.fields_by_name['ystats_string'].message_type = _YSTATSSTRING_REQ
_PROFILERRESPONSE.fields_by_name['ystats'].message_type = _YSTATS
DESCRIPTOR.message_types_by_name['YStats'] = _YSTATS
DESCRIPTOR.message_types_by_name['YStats_Req'] = _YSTATS_REQ
DESCRIPTOR.message_types_by_name['YStatsString_Req'] = _YSTATSSTRING_REQ
DESCRIPTOR.message_types_by_name['ProfilerRequest'] = _PROFILERREQUEST
DESCRIPTOR.message_types_by_name['ProfilerResponse'] = _PROFILERRESPONSE
class YStats(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
class FuncStats(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _YSTATS_FUNCSTATS
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats.FuncStats)
class ThreadStats(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _YSTATS_THREADSTATS
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats.ThreadStats)
DESCRIPTOR = _YSTATS
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats)
class YStats_Req(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _YSTATS_REQ
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStats_Req)
class YStatsString_Req(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _YSTATSSTRING_REQ
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.YStatsString_Req)
class ProfilerRequest(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _PROFILERREQUEST
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.ProfilerRequest)
class ProfilerResponse(_message.Message):
__metaclass__ = _reflection.GeneratedProtocolMessageType
DESCRIPTOR = _PROFILERRESPONSE
# @@protoc_insertion_point(class_scope:com.jetbrains.python.profiler.ProfilerResponse)
# @@protoc_insertion_point(module_scope)
+115
View File
@@ -0,0 +1,115 @@
from socket import AF_INET
from socket import SOCK_STREAM
from socket import socket
import time
import yappi
from prof_io import ProfWriter, ProfReader
from pydevd_utils import save_main_module
import pydev_imports
from profiler_protocol_pb2 import ProfilerResponse
__author__ = 'traff'
import sys
import traceback
def StartClient(host, port):
""" connects to a host/port """
s = socket(AF_INET, SOCK_STREAM)
MAX_TRIES = 100
i = 0
while i<MAX_TRIES:
try:
s.connect((host, port))
except:
i+=1
time.sleep(0.2)
continue
return s
sys.stderr.write("Could not connect to %s: %s\n" % (host, port))
sys.stderr.flush()
traceback.print_exc()
sys.exit(1) #TODO: is it safe?
class Profiler(object):
def __init__(self):
pass
def connect(self, host, port):
s = StartClient(host, port)
self.initializeNetwork(s)
def initializeNetwork(self, sock):
try:
sock.settimeout(None) # infinite, no timeouts from now on - jython does not have it
except:
pass
self.writer = ProfWriter(sock)
self.reader = ProfReader(sock, self)
self.writer.start()
self.reader.start()
time.sleep(0.1) # give threads time to start
def process(self, message):
if message.HasField('ystats_string'):
self.stats_string(message.id)
def run(self, file):
m = save_main_module(file, 'run_profiler')
globals = m.__dict__
try:
globals['__builtins__'] = __builtins__
except NameError:
pass # Not there on Jython...
self.start_profiling()
pydev_imports.execfile(file, globals, globals) # execute the script
# self.stats_string()
time.sleep(10)
def start_profiling(self):
yappi.start()
def stats_string(self, id):
import StringIO
output = StringIO.StringIO()
yappi.get_func_stats().print_all(out=output)
m = ProfilerResponse()
m.id = id
m.ystats_string = output.getvalue()
self.writer.addCommand(m)
if __name__ == '__main__':
host = sys.argv[1]
port = int(sys.argv[2])
file = sys.argv[3]
profiler = Profiler()
try:
profiler.connect(host, port)
except:
sys.stderr.write("Could not connect to %s: %s\n" % (host, port))
traceback.print_exc()
sys.exit(1)
profiler.run(file)