MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

Numpy array loading.

Browse Source
This commit is contained in:
Dmitry Trofimov
2014-11-05 22:20:07 +01:00
parent 1e2b1a981c
commit 24da5e3dec
13 changed files with 274 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/helpers/pydev/pydevd.py
+13
View File
@@ -19,6 +19,7 @@ from pydevd_comm import CMD_CHANGE_VARIABLE, \
CMD_GET_COMPLETIONS, \
CMD_GET_FRAME, \
CMD_GET_VARIABLE, \
CMD_GET_ARRAY, \
CMD_LIST_THREADS, \
CMD_REMOVE_BREAK, \
CMD_RUN, \
@@ -47,6 +48,7 @@ from pydevd_comm import CMD_CHANGE_VARIABLE, \
InternalConsoleExec, \
InternalGetFrame, \
InternalGetVariable, \
InternalGetArray, \
InternalTerminateThread, \
InternalRunThread, \
InternalStepThread, \
@@ -791,6 +793,17 @@ class PyDB:
except:
traceback.print_exc()
elif cmd_id == CMD_GET_ARRAY:
# we received some command to get an array variable
# the text is: thread_id\tframe_id\tFRAME|GLOBAL\tname\ttemp\troffs\tcoffs\trows\tcols\tformat
try:
thread_id, frame_id, scope, name, temp, roffset, coffset, rows, cols, format = text.split('\t')
int_cmd = InternalGetArray(seq, thread_id, frame_id, scope, name, temp, roffset, coffset, rows, cols, format)
self.postInternalCommand(int_cmd, thread_id)
except:
traceback.print_exc()
elif cmd_id == CMD_GET_COMPLETIONS:
# we received some command to get a variable
# the text is: thread_id\tframe_id\tactivation token
python/helpers/pydev/pydevd_comm.py
+87 -1
View File
@@ -136,7 +136,7 @@ CMD_IGNORE_THROWN_EXCEPTION_AT = 140
CMD_ENABLE_DONT_TRACE = 141
CMD_SHOW_CONSOLE = 142
CMD_GET_ARRAY = 143
CMD_VERSION = 501
CMD_RETURN = 502
@@ -189,6 +189,8 @@ ID_TO_MEANING = {
'501':'CMD_VERSION',
'502':'CMD_RETURN',
'901':'CMD_ERROR',
'143':'CMD_GET_ARRAY',
}
MAX_IO_MSG_SIZE = 1000 #if the io is too big, we'll not send all (could make the debugger too non-responsive)
@@ -692,6 +694,13 @@ class NetCommandFactory:
except Exception:
return self.makeErrorMessage(seq, GetExceptionTracebackStr())
def makeGetArrayMessage(self, seq, payload):
try:
return NetCommand(CMD_GET_ARRAY, seq, payload)
except Exception:
return self.makeErrorMessage(seq, GetExceptionTracebackStr())
def makeGetFrameMessage(self, seq, payload):
try:
return NetCommand(CMD_GET_FRAME, seq, payload)
@@ -954,6 +963,83 @@ class InternalGetVariable(InternalThreadCommand):
dbg.writer.addCommand(cmd)
#=======================================================================================================================
# InternalGetArray
#=======================================================================================================================
from pydevd_vars import getVariable
MAXIMUM_ARRAY_SIZE = 300
class InternalGetArray(InternalThreadCommand):
def __init__(self, seq, thread_id, frame_id, scope, name, temp, roffset, coffset, rows, cols, format):
self.sequence = seq
self.thread_id = thread_id
self.frame_id = frame_id
self.scope = scope
self.name = name
self.temp = temp;
self.roffset = int(roffset)
self.coffset = int(coffset)
self.rows = int(rows)
self.cols = int(cols)
self.format = '\'' + format + '\''
def doIt(self, dbg):
try:
var = getVariable(self.thread_id, self.frame_id, self.scope, self.temp)
rows = min(self.rows, MAXIMUM_ARRAY_SIZE)
cols = min(self.cols, MAXIMUM_ARRAY_SIZE)
if self.rows == 1 and self.cols == 1:
rows = 1
cols = 1
elif self.rows == 1 or self.cols == 1:
is_row = True if (self.rows == 1) else False
pure_1d = False if (len(var) == 1) else True
if not pure_1d:
var = var[0]
if is_row:
var = var[self.coffset:]
cols = min(cols, len(var))
else:
var = var[self.roffset:]
rows = min(rows, len(var))
else:
var = var[self.roffset:, self.coffset:]
rows = min(rows, len(var))
cols = min(cols, len(var[0]))
xml = "<xml>"
xml += "<array name=\"%s\" rows=\"%s\" cols=\"%s\"/>" % (self.name, rows, cols)
for row in range(rows):
xml += "<row index=\"%s\"/>" % to_string(row)
for col in range(cols):
value = var
name = '%s[%s][%s]' % (self.name, row, col)
if self.rows == 1 or self.cols == 1:
if self.rows == 1 and self.cols == 1:
value = var
name = '%s' % self.name
else:
dim = col if (self.rows == 1) else row
value = var[dim]
name = '%s[%s]' % (self.name, dim)
else:
value = var[row][col]
value = self.format % value
xml += pydevd_vars.varToXML(value, name)
xml += "</xml>"
cmd = dbg.cmdFactory.makeGetArrayMessage(self.sequence, xml)
dbg.writer.addCommand(cmd)
except:
cmd = dbg.cmdFactory.makeErrorMessage(self.sequence, "Error resolving array " + GetExceptionTracebackStr())
dbg.writer.addCommand(cmd)
#=======================================================================================================================
# InternalChangeVariable
#=======================================================================================================================
python/pydevSrc/com/jetbrains/python/debugger/PyFrameAccessor.java
+1 -1
View File
@@ -21,5 +21,5 @@ public interface PyFrameAccessor {
@Nullable
PyReferrersLoader getReferrersLoader();
Object[][] getArrayItems(PyDebugValue var, int colOffset, int rowOffset, int cols, int rows, String format);
Object[][] getArrayItems(PyDebugValue var, int colOffset, int rowOffset, int rows, int cols, String format) throws PyDebuggerException;
}
python/pydevSrc/com/jetbrains/python/debugger/pydev/AbstractCommand.java
+1
View File
@@ -52,6 +52,7 @@ public abstract class AbstractCommand<T> {
public static final String NEW_LINE_CHAR = "@_@NEW_LINE_CHAR@_@";
public static final String TAB_CHAR = "@_@TAB_CHAR@_@";
public static final int GET_ARRAY = 143;
@NotNull private final RemoteDebugger myDebugger;
private final int myCommandCode;
python/pydevSrc/com/jetbrains/python/debugger/pydev/GetArrayCommand.java
+54
View File
@@ -0,0 +1,54 @@
package com.jetbrains.python.debugger.pydev;
import com.jetbrains.python.debugger.PyDebugValue;
import com.jetbrains.python.debugger.PyDebuggerException;
/**
* @author amarch
*/
public class GetArrayCommand extends GetFrameCommand {
private final String myVariableName;
private final int myRowOffset;
private final int myColOffset;
private final int myRows;
private final int myColumns;
private final String myFormat;
private final String myTempName;
private Object[][] myArrayItems;
public GetArrayCommand(final RemoteDebugger debugger, final String threadId, final String frameId, PyDebugValue var, int rowOffset, int colOffset, int rows, int cols, String format) {
super(debugger, GET_ARRAY, threadId, frameId);
myVariableName = var.getName();
myTempName = var.getTempName();
myRowOffset = rowOffset;
myColOffset = colOffset;
myRows = rows;
myColumns = cols;
myFormat = format;
}
@Override
protected void buildPayload(Payload payload) {
super.buildPayload(payload);
payload.add(myVariableName);
payload.add(myTempName);
payload.add(myRowOffset);
payload.add(myColOffset);
payload.add(myRows);
payload.add(myColumns);
payload.add(myFormat);
}
@Override
protected void processResponse(final ProtocolFrame response) throws PyDebuggerException {
if (response.getCommand() >= 900 && response.getCommand() < 1000) {
throw new PyDebuggerException(response.getPayload());
}
myArrayItems = ProtocolParser.parseArrayValues(response.getPayload(), myDebugProcess);
}
public Object[][] getArray(){
return myArrayItems;
}
}
python/pydevSrc/com/jetbrains/python/debugger/pydev/MultiProcessDebugger.java
+11
View File
@@ -179,6 +179,17 @@ public class MultiProcessDebugger implements ProcessDebugger {
return debugger(threadId).loadVariable(threadId, frameId, var);
}
public Object[][] loadArrayItems(String threadId,
String frameId,
PyDebugValue var,
int rowOffset,
int colOffset,
int rows,
int cols,
String format) throws PyDebuggerException {
return debugger(threadId).loadArrayItems(threadId, frameId, var, rowOffset, colOffset, rows, cols, format);
}
@Override
public void loadReferrers(String threadId, String frameId, PyReferringObjectsValue var, PyDebugCallback<XValueChildrenList> callback) {
debugger(threadId).loadReferrers(threadId, frameId, var, callback);
python/pydevSrc/com/jetbrains/python/debugger/pydev/ProcessDebugger.java
+2
View File
@@ -36,6 +36,8 @@ public interface ProcessDebugger {
// todo: don't generate temp variables for qualified expressions - just split 'em
XValueChildrenList loadVariable(String threadId, String frameId, PyDebugValue var) throws PyDebuggerException;
Object[][] loadArrayItems(String threadId, String frameId, PyDebugValue var, int rowOffset, int colOffset, int rows, int cols, String format) throws PyDebuggerException;
void loadReferrers(String threadId, String frameId, PyReferringObjectsValue var, PyDebugCallback<XValueChildrenList> callback);
PyDebugValue changeVariable(String threadId, String frameId, PyDebugValue var, String value)
python/pydevSrc/com/jetbrains/python/debugger/pydev/ProtocolParser.java
+52
View File
@@ -179,6 +179,58 @@ public class ProtocolParser {
return new PyDebugValue(name, type, value, "True".equals(isContainer), "True".equals(isErrorOnEval), frameAccessor);
}
public static Object[][] parseArrayValues(final String text, final PyFrameAccessor frameAccessor) throws PyDebuggerException {
final XppReader reader = openReader(text, false);
int cols = 0;
int rows = 0;
if (reader.hasMoreChildren()) {
reader.moveDown();
if (!"array".equals(reader.getNodeName())) {
throw new PyDebuggerException("Expected <array> at first node, found " + reader.getNodeName());
}
rows = readInt(reader, "rows", null);
cols = readInt(reader, "cols", null);
reader.moveUp();
}
return parseArrayValues(reader, frameAccessor, cols, rows);
}
public static Object[][] parseArrayValues(final XppReader reader, final PyFrameAccessor frameAccessor, final int cols, final int rows) throws PyDebuggerException {
if (rows <= 0 || cols <= 0) {
throw new PyDebuggerException("Array xml: bad rows or columns number: (" + rows + ", " + cols + ")");
}
Object[][] values = new Object[rows][cols];
int currRow = 0;
int currCol = 0;
while (reader.hasMoreChildren()) {
reader.moveDown();
if (!"var".equals(reader.getNodeName()) && !"row".equals(reader.getNodeName())) {
throw new PyDebuggerException("Expected <var> or <row>, found " + reader.getNodeName());
}
if ("row".equals(reader.getNodeName())) {
int index = readInt(reader, "index", null);
if (currRow != index) {
throw new PyDebuggerException("Array xml: expected " + currRow + " row, found " + index);
}
if (currRow > 0 && currCol != cols) {
throw new PyDebuggerException("Array xml: expected " + cols + " filled columns, got " + currCol + " instead.");
}
currRow += 1;
currCol = 0;
} else {
PyDebugValue value = parseValue(reader, frameAccessor);
values[currRow-1][currCol] = value.getValue();
currCol += 1;
}
reader.moveUp();
}
return values;
}
private static XppReader openReader(final String text, final boolean checkForContent) throws PyDebuggerException {
final XppReader reader = new XppReader(new StringReader(text), new MXParser(), new NoNameCoder());
if (checkForContent && !reader.hasMoreChildren()) {
python/pydevSrc/com/jetbrains/python/debugger/pydev/RemoteDebugger.java
+8
View File
@@ -154,6 +154,14 @@ public class RemoteDebugger implements ProcessDebugger {
return command.getVariables();
}
@Override
public Object[][] loadArrayItems(String threadId, String frameId, PyDebugValue var, int rowOffset, int colOffset, int rows, int cols, String format) throws PyDebuggerException {
setTempVariable(threadId, frameId, var);
final GetArrayCommand command = new GetArrayCommand(this, threadId, frameId, var, rowOffset, colOffset, rows, cols, format);
command.execute();
return command.getArray();
}
@Override
public void loadReferrers(final String threadId,
python/src/com/jetbrains/python/console/PydevConsoleCommunication.java
+1 -1
View File
@@ -532,7 +532,7 @@ public class PydevConsoleCommunication extends AbstractConsoleCommunication impl
}
@Override
public Object[][] getArrayItems(PyDebugValue var, int colOffset, int rowOffset, int cols, int rows, String format) {
public Object[][] getArrayItems(PyDebugValue var, int colOffset, int rowOffset, int rows, int cols, String format) {
return new Object[][]{new Object[]{1, 2}, new Object[]{3, 4}};
}
python/src/com/jetbrains/python/debugger/PyDebugProcess.java
+4 -2
View File
@@ -572,8 +572,10 @@ public class PyDebugProcess extends XDebugProcess implements IPyDebugProcess, Pr
}
@Override
public Object[][] getArrayItems(PyDebugValue var, int colOffset, int rowOffset, int cols, int rows, String format) {
return new Object[][]{new Object[]{1, 2}, new Object[]{3, 4}};
public Object[][] getArrayItems(PyDebugValue var, int rowOffset, int colOffset, int rows, int cols, String format)
throws PyDebuggerException {
final PyStackFrame frame = currentFrame();
return myDebugger.loadArrayItems(frame.getThreadId(), frame.getFrameId(), var, rowOffset, colOffset, rows, cols, format);
}
@Nullable
python/src/com/jetbrains/python/debugger/array/AsyncArrayTableModel.java
+12 -13
View File
@@ -20,26 +20,22 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListenableFutureTask;
import com.intellij.openapi.application.ApplicationManager;
import com.intellij.openapi.util.Pair;
import com.intellij.ui.components.JBScrollPane;
import com.intellij.ui.components.JBViewport;
import com.intellij.util.containers.HashMap;
import com.intellij.util.containers.Queue;
import com.intellij.util.ui.UIUtil;
import com.jetbrains.python.debugger.PyDebugValue;
import javax.swing.table.AbstractTableModel;
import java.awt.*;
import java.util.TreeSet;
import java.util.concurrent.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
/**
* @author traff
*/
public class AsyncArrayTableModel extends AbstractTableModel {
private static final int CHUNK_COL_SIZE = 2; //TODO set to 100
private static final int CHUNK_ROW_SIZE = 2;
private static final int CHUNK_COL_SIZE = 30;
private static final int CHUNK_ROW_SIZE = 30;
public static final String EMPTY_CELL_VALUE = "";
private final int myRows;
@@ -63,7 +59,9 @@ public class AsyncArrayTableModel extends AbstractTableModel {
@Override
public Object[][] call() throws Exception {
return value.getFrameAccessor()
.getArrayItems(slicedValue, key.first, key.second, CHUNK_COL_SIZE, CHUNK_ROW_SIZE, myProvider.getFormat());
.getArrayItems(slicedValue, key.first, key.second, Math.min(CHUNK_COL_SIZE, getRowCount() - key.first),
Math.min(CHUNK_ROW_SIZE, getColumnCount() - key.second),
myProvider.getFormat());
}
});
@@ -81,7 +79,7 @@ public class AsyncArrayTableModel extends AbstractTableModel {
@Override
public boolean isCellEditable(int row, int column) {
return !getValueAt(row, column).equals(EMPTY_CELL_VALUE);
return false;
}
public Object getValueAt(final int row, final int col) {
@@ -109,13 +107,14 @@ public class AsyncArrayTableModel extends AbstractTableModel {
if (r < chunk.get().length) {
if (c < chunk.get()[r].length) {
return chunk.get()[r][c];
return myProvider.correctStringValue((String)chunk.get()[r][c]);
}
}
}
return EMPTY_CELL_VALUE;
}
catch (Exception e) {
myProvider.showError(e.getMessage());
return EMPTY_CELL_VALUE; //TODO: handle it
}
}
python/src/com/jetbrains/python/debugger/array/NumpyArrayTable.java
+28 -3
View File
@@ -126,6 +126,7 @@ public class NumpyArrayTable {
// add slice actions
initSliceFieldActions();
//make value name read-only
myComponent.getSliceTextField().addFocusListener(new FocusListener() {
@Override
@@ -144,6 +145,20 @@ public class NumpyArrayTable {
//add format actions
initFormatFieldActions();
//clear error on scroll
myComponent.getScrollPane().getHorizontalScrollBar().addAdjustmentListener(new AdjustmentListener() {
@Override
public void adjustmentValueChanged(AdjustmentEvent e) {
clearErrorMessage();
}
});
myComponent.getScrollPane().getVerticalScrollBar().addAdjustmentListener(new AdjustmentListener() {
@Override
public void adjustmentValueChanged(AdjustmentEvent e) {
clearErrorMessage();
}
});
}
public void disableColor() {
@@ -233,9 +248,6 @@ public class NumpyArrayTable {
myComponent.getSliceTextField().setText(getDefaultPresentation());
myComponent.getFormatTextField().setText(getDefaultFormat());
myDialog.setTitle(getTitlePresentation(getDefaultPresentation()));
if (myTable.getColumnCount() > 0) {
myTable.setDefaultEditor(myTable.getColumnClass(0), getArrayTableCellEditor());
}
}
});
initTableModel(false);
@@ -441,6 +453,9 @@ public class NumpyArrayTable {
}
((AsyncArrayTableModel)myTable.getModel()).fireTableDataChanged();
((AsyncArrayTableModel)myTable.getModel()).fireTableCellUpdated(0, 0);
if (myTable.getColumnCount() > 0) {
myTable.setDefaultRenderer(myTable.getColumnClass(0), myTableCellRenderer);
}
}
});
}
@@ -541,6 +556,16 @@ public class NumpyArrayTable {
};
}
public String correctStringValue(@NotNull String value) {
String corrected = value;
if (isNumeric()) {
if (value.startsWith("\'") || value.startsWith("\"")) {
corrected = value.substring(1, value.length() - 1);
}
}
return corrected;
}
public void setDtypeKind(String dtype) {
this.myDtypeKind = dtype;
}