PY-18029 Finish Python Console improvements

python/helpers/pydev/pydev_console/io.py

@@ -0,0 +1,76 @@
+import threading
+
+
+class PipeIO(object):
+    """Thread-safe pipe with blocking reads and writes
+    """
+    MAX_BUFFER_SIZE = 4096
+
+    def __init__(self):
+        self.lock = threading.RLock()
+        self.bytes_consumed = threading.Condition(self.lock)
+        self.bytes_produced = threading.Condition(self.lock)
+
+        self.buffer = bytearray()
+        self.read_pos = 0
+
+    def _bytes_available(self):
+        return self.read_pos < len(self.buffer)
+
+    def _reset_buffer(self):
+        self.buffer = bytearray()
+        self.read_pos = 0
+
+    def read(self, sz):
+        """Reads `sz` bytes at most
+
+        Blocks until some data is available in buffer.
+
+        :param sz: the maximum count of bytes to read
+        :return: bytes read
+        """
+        self.lock.acquire()
+        try:
+            while not self._bytes_available():
+                self.bytes_produced.wait()
+
+            read_until_pos = min(self.read_pos + sz, len(self.buffer))
+            result = bytes(self.buffer[self.read_pos:read_until_pos])
+
+            self.read_pos = read_until_pos
+            self.bytes_consumed.notifyAll()
+
+            return result
+        finally:
+            self.lock.release()
+
+    def write(self, buf):
+        """Writes `buf` content
+
+        Blocks until all `buf` written.
+
+        :param buf: bytes to write
+        :return: None
+        """
+        self.lock.acquire()
+        try:
+            buf_pos = 0
+            while True:
+                if buf_pos == len(buf):
+                    break
+
+                if len(self.buffer) == self.MAX_BUFFER_SIZE:
+                    while self.read_pos < self.MAX_BUFFER_SIZE:
+                        self.bytes_consumed.wait()
+
+                    self._reset_buffer()
+
+                bytes_to_write = min(len(buf) - buf_pos, self.MAX_BUFFER_SIZE - len(self.buffer))
+                new_buf_pos = buf_pos + bytes_to_write
+
+                self.buffer.extend(buf[buf_pos:new_buf_pos])
+                self.bytes_produced.notifyAll()
+
+                buf_pos = new_buf_pos
+        finally:
+            self.lock.release()

python/helpers/pydev/pydev_console/thrift_rpc.py

@@ -1,20 +1,18 @@
+import socket
 import threading
 
-from pydev_console.thrift_transport import TBidirectionalClientTransport, TSyncClient
+from pydev_console.thrift_transport import TSyncClient, open_transports_as_client, _create_client_server_transports
 from thriftpy.protocol import TBinaryProtocolFactory
 from thriftpy.server import TThreadedServer
 from thriftpy.thrift import TProcessor
 
 
 def make_rpc_client(client_service, host, port, proto_factory=TBinaryProtocolFactory()):
-    # instantiate client
-    transport = TBidirectionalClientTransport(host, port)
-    protocol = proto_factory.get_protocol(transport)
-    transport.open()
+    client_transport, server_transport = open_transports_as_client((host, port))
 
-    client = TSyncClient(client_service, protocol)
+    client_protocol = proto_factory.get_protocol(client_transport)
 
-    server_transport = transport.get_server_transport()
+    client = TSyncClient(client_service, client_protocol)
 
     return client, server_transport
 
@@ -33,3 +31,42 @@ def start_rpc_server(server_transport, server_service, server_handler, proto_fac
     t.start()
 
     return server
+
+
+def start_rpc_server_and_make_client(host, port, server_service, client_service, server_handler,
+                                     proto_factory=TBinaryProtocolFactory(strict_read=False, strict_write=False)):
+    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+    server_socket.bind((host, port))
+    server_socket.listen(1)
+
+    t = threading.Thread(target=_rpc_server, args=(server_socket, server_service, client_service, server_handler, proto_factory))
+    # t.setDaemon(self.daemon)
+    t.start()
+
+    return server_socket
+
+
+def _rpc_server(server_socket, server_service, client_service, server_handler, proto_factory):
+    client_socket, address = server_socket.accept()
+
+    client_transport, server_transport = _create_client_server_transports(client_socket)
+
+    # setup server
+    processor = TProcessor(server_service, server_handler)
+
+    # todo as `server.serve()` is excessive we may want to get rid of `server` as `TThreadedServer`
+    server = TThreadedServer(processor, server_transport, iprot_factory=proto_factory)
+
+    client = server.trans.accept()
+    t = threading.Thread(target=server.handle, args=(client,))
+    # t.setDaemon(self.daemon)
+    t.start()
+
+    client_protocol = proto_factory.get_protocol(client_transport)
+
+    client = TSyncClient(client_service, client_protocol)
+
+    # todo fix broken encapsulation
+    server_handler.rpc_client = client

python/helpers/pydev/pydev_console/thrift_transport.py

@@ -2,8 +2,8 @@ import socket
 import struct
 import sys
 import threading
-from io import BytesIO, SEEK_CUR
 
+from pydev_console.io import PipeIO
 from thriftpy.thrift import TClient
 from thriftpy.transport import TTransportBase, readall
 
@@ -11,28 +11,13 @@ REQUEST = 0
 RESPONSE = 1
 
 
-def _read_anything(read_fn, sz):
-    buff = b''
-    have = 0
-    while have == 0:
-        chunk = read_fn(sz - have)
-
-        have += len(chunk)
-        buff += chunk
-
-    return buff
-
-
 class MultiplexedSocketReader(object):
 
     def __init__(self, s):
         self._socket = s
 
-        self._request_buffer = BytesIO()
-        self._response_buffer = BytesIO()
-
-        self._request_buffer_lock = threading.RLock()
-        self._response_buffer_lock = threading.RLock()
+        self._request_pipe = PipeIO()
+        self._response_pipe = PipeIO()
 
         self._read_socket_lock = threading.RLock()
 
@@ -40,19 +25,13 @@ class MultiplexedSocketReader(object):
         """
         Invoked form server-side of the bidirectional transport.
         """
-        return _read_anything(self._read_request_buffer, sz)
+        return self._request_pipe.read(sz)
 
     def read_response(self, sz):
         """
         Invoked form client-side of the bidirectional transport.
         """
-        return _read_anything(self._read_response_buffer, sz)
-
-    def _read_request_buffer(self, sz):
-        return self._request_buffer.read(sz)
-
-    def _read_response_buffer(self, sz):
-        return self._response_buffer.read(sz)
+        return self._response_pipe.read(sz)
 
     # noinspection PyUnusedLocal
     def _try_fill_buffer(self, sz):
@@ -63,13 +42,9 @@ class MultiplexedSocketReader(object):
             direction, frame = self._read_frame()
 
         if direction == REQUEST:
-            with self._request_buffer_lock:
-                self._request_buffer.write(frame)
-                self._request_buffer.seek(-len(frame), SEEK_CUR)
+            self._request_pipe.write(frame)
         elif direction == RESPONSE:
-            with self._response_buffer_lock:
-                self._response_buffer.write(frame)
-                self._response_buffer.seek(-len(frame), SEEK_CUR)
+            self._response_pipe.write(frame)
 
     def _read_frame(self):
         # todo introduce sz argument
@@ -110,7 +85,7 @@ class FramedWriter(object):
     MAX_BUFFER_SIZE = 4096
 
     def __init__(self):
-        self._buffer = BytesIO()
+        self._buffer = bytearray()
 
     def _get_writer(self):
         raise NotImplementedError
@@ -130,21 +105,21 @@ class FramedWriter(object):
             if buffer_size + bytes_to_write > self.MAX_BUFFER_SIZE:
                 write_till_byte = bytes_written + (self.MAX_BUFFER_SIZE - buffer_size)
 
-                self._buffer.write(buf[bytes_written:write_till_byte])
+                self._buffer.extend(buf[bytes_written:write_till_byte])
                 self.flush()
 
                 bytes_written = write_till_byte
             else:
                 # the whole buffer processed
-                self._buffer.write(buf[bytes_written:])
+                self._buffer.extend(buf[bytes_written:])
                 bytes_written = buf_len
 
     def flush(self):
         # reset wbuf before write/flush to preserve state on underlying failure
-        out = self._buffer.getvalue()
+        out = bytes(self._buffer)
         # prepend the message with the direction byte
         out = struct.pack("b", self._get_write_direction()) + out
-        self._buffer = BytesIO()
+        self._buffer = bytearray()
 
         # N.B.: Doing this string concatenation is WAY cheaper than making
         # two separate calls to the underlying socket object. Socket writes in
@@ -154,21 +129,18 @@ class FramedWriter(object):
         self._get_writer().write(struct.pack("!i", len(out)) + out)
 
     def close(self):
-        self._buffer.close()
+        self._buffer = bytearray()
+        pass
 
 
 class TBidirectionalClientTransport(TTransportBase, FramedWriter):
-    def __init__(self, host, port):
+    def __init__(self, client_socket, reader, writer):
         super(TBidirectionalClientTransport, self).__init__()
 
-        self.host = host
-        self.port = port
-
         # the following properties will be initialized in `open()`
-        self._client_socket = None
-        self._reader = None
-        self._writer = None
-        self._server_transport = None
+        self._client_socket = client_socket
+        self._reader = reader
+        self._writer = writer
 
     def _get_writer(self):
         return self._writer
@@ -182,29 +154,10 @@ class TBidirectionalClientTransport(TTransportBase, FramedWriter):
         """
         return self._reader.read_response(sz)
 
-    def get_server_transport(self):
-        if not self._server_transport:
-            raise Exception
-
-        return self._server_transport
-
     def is_open(self):
         # todo we may try to monitor reads and writes and put a flag if they fail
         return self._client_socket
 
-    def open(self):
-        client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        client_socket.connect((self.host, self.port))
-
-        self._client_socket = client_socket
-
-        self._reader = MultiplexedSocketReader(self._client_socket)
-        self._reader.start_reading()
-
-        self._writer = SocketWriter(self._client_socket)
-
-        self._server_transport = TReversedServerTransport(self._reader.read_request, self._writer)
-
     def close(self):
         # todo should we do something with buffer of the multiplexed reader
         self._client_socket.shutdown(socket.SHUT_RDWR)
@@ -268,3 +221,34 @@ class TSyncClient(TClient):
     def _req(self, _api, *args, **kwargs):
         with self._lock:
             return super(TSyncClient, self)._req(_api, *args, **kwargs)
+
+
+def open_transports_as_client(addr):
+    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    client_socket.connect(addr)
+
+    return _create_client_server_transports(client_socket)
+
+
+def open_transports_as_server(addr):
+    server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+    server_socket.bind(addr)
+    server_socket.listen(1)
+
+    client_socket, address = server_socket.accept()
+
+    raise _create_client_server_transports(client_socket)
+
+
+def _create_client_server_transports(sock):
+    reader = MultiplexedSocketReader(sock)
+    reader.start_reading()
+
+    writer = SocketWriter(sock)
+
+    client_transport = TBidirectionalClientTransport(sock, reader, writer)
+    server_transport = TReversedServerTransport(client_transport._reader.read_request, client_transport._writer)
+
+    return client_transport, server_transport

python/helpers/pydev/pydevconsole.py

@@ -1,8 +1,9 @@
 '''
 Entry point module to start the interactive console.
 '''
+from _pydev_bundle._pydev_getopt import gnu_getopt
 from _pydev_imps._pydev_saved_modules import thread
-from pydev_console.thrift_rpc import make_rpc_client, start_rpc_server
+from pydev_console.thrift_rpc import make_rpc_client, start_rpc_server, start_rpc_server_and_make_client
 
 start_new_thread = thread.start_new_thread
 
@@ -348,9 +349,8 @@ def enable_thrift_logging():
     logger.addHandler(ch)
 
 
-def start_server(host, port, client_port, client_host = None):
-    if not client_host:
-        client_host = host
+def start_server():
+    # 0. General stuff
 
     #replace exit (see comments on method)
     #note that this does not work in jython!!! (sys method can't be replaced).
@@ -360,9 +360,42 @@ def start_server(host, port, client_port, client_host = None):
 
     enable_thrift_logging()
 
+    server_service = console_thrift.PythonConsole
+    client_service = console_thrift.IDE
+
+    # 1. Start Python console server
+
+
+    interpreter = InterpreterInterface(threading.currentThread(), None, None)
+
+    # `InterpreterInterface` implements all methods required for the handler
+    server_handler = interpreter
+
+    server_socket = start_rpc_server_and_make_client('', 0, server_service, client_service, server_handler)
+
+
+    # 2. Print server port for the IDE
+
+    _, server_port = server_socket.getsockname()
+    print(server_port)
+
+    # 3. Wait for IDE to connect to the server
+
+    process_exec_queue(interpreter)
+
+
+def start_client(host, port):
+    #replace exit (see comments on method)
+    #note that this does not work in jython!!! (sys method can't be replaced).
+    sys.exit = do_exit
+
+    from pydev_console.thrift_communication import console_thrift
+
+    enable_thrift_logging()
+
     client_service = console_thrift.IDE
 
-    client, server_transport = make_rpc_client(client_service, client_host, client_port)
+    client, server_transport = make_rpc_client(client_service, host, port)
 
     interpreter = InterpreterInterface(threading.currentThread(), None, client)
 
@@ -545,18 +578,32 @@ if __name__ == '__main__':
     #'Variables' and 'Expressions' views stopped working when debugging interactive console
     import pydevconsole
     sys.stdin = pydevconsole.BaseStdIn(sys.stdin)
-    port, client_port = sys.argv[1:3]
-    from _pydev_bundle import pydev_localhost
 
-    if int(port) == 0 and int(client_port) == 0:
-        (h, p) = pydev_localhost.get_socket_name()
+    # parse command-line arguments
+    optlist, _ = gnu_getopt(sys.argv, 'm:h:p', ['mode=', 'host=', 'port='])
+    mode = None
+    host = None
+    port = None
+    for opt, arg in optlist:
+        if opt in ('-m', '--mode'):
+            mode = arg
+        elif opt in ('-h', '--host'):
+            host = arg
+        elif opt in ('-p', '--port'):
+            port = int(arg)
 
-        client_port = p
+    if mode not in ('client', 'server'):
+        sys.exit(-1)
 
-    if len(sys.argv) > 4:
-        host = sys.argv[3]
-        client_host = sys.argv[4]
-    else:
-        host = client_host = pydev_localhost.get_localhost()
+    if mode == 'client':
+        if not port:
+            # port must be set for client
+            sys.exit(-1)
 
-    pydevconsole.start_server(host, int(port), int(client_port), client_host)
+        if not host:
+            from _pydev_bundle import pydev_localhost
+            host = client_host = pydev_localhost.get_localhost()
+
+        pydevconsole.start_client(host, port)
+    elif mode == 'server':
+        pydevconsole.start_server()