From a32b8f4aad5c079c54df03d0ce4728f0ce5b0e16 Mon Sep 17 00:00:00 2001 From: Aleksey Pivovarov Date: Thu, 2 Apr 2015 18:16:43 +0300 Subject: [PATCH 01/21] IDEA-135671 diff: better painting in gutter Paint half of folding outline in gutter like it is a part of editor --- .../com/intellij/diff/util/DiffDrawUtil.java | 13 +++++-- .../diff/util/DiffLineMarkerRenderer.java | 34 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/platform/diff-impl/src/com/intellij/diff/util/DiffDrawUtil.java b/platform/diff-impl/src/com/intellij/diff/util/DiffDrawUtil.java index 12ea0b3b038a..ce9d6ae8f93c 100644 --- a/platform/diff-impl/src/com/intellij/diff/util/DiffDrawUtil.java +++ b/platform/diff-impl/src/com/intellij/diff/util/DiffDrawUtil.java @@ -215,7 +215,7 @@ public class DiffDrawUtil { // TODO: diff looks cool with wide markers. Maybe we can keep them ? highlighter.setThinErrorStripeMark(true); - installGutterRenderer(highlighter, type); + installGutterRenderer(highlighter, type, ignored); return highlighter; } @@ -232,8 +232,15 @@ public class DiffDrawUtil { return highlighter; } - public static void installGutterRenderer(@NotNull RangeHighlighter highlighter, @NotNull TextDiffType type) { - highlighter.setLineMarkerRenderer(new DiffLineMarkerRenderer(type)); + public static void installGutterRenderer(@NotNull RangeHighlighter highlighter, + @NotNull TextDiffType type) { + installGutterRenderer(highlighter, type, false); + } + + public static void installGutterRenderer(@NotNull RangeHighlighter highlighter, + @NotNull TextDiffType type, + boolean ignoredFoldingOutline) { + highlighter.setLineMarkerRenderer(new DiffLineMarkerRenderer(type, ignoredFoldingOutline)); } public static void installEmptyRangeRenderer(@NotNull RangeHighlighter highlighter, @NotNull TextDiffType type) { diff --git a/platform/diff-impl/src/com/intellij/diff/util/DiffLineMarkerRenderer.java b/platform/diff-impl/src/com/intellij/diff/util/DiffLineMarkerRenderer.java index 09f8da940e7b..3589116171a7 100644 --- a/platform/diff-impl/src/com/intellij/diff/util/DiffLineMarkerRenderer.java +++ b/platform/diff-impl/src/com/intellij/diff/util/DiffLineMarkerRenderer.java @@ -25,32 +25,50 @@ import java.awt.*; public class DiffLineMarkerRenderer implements LineMarkerRenderer { @NotNull private final TextDiffType myDiffType; + private final boolean myIgnoredFoldingOutline; public DiffLineMarkerRenderer(@NotNull TextDiffType diffType) { - myDiffType = diffType; + this(diffType, false); } + public DiffLineMarkerRenderer(@NotNull TextDiffType diffType, boolean ignoredFoldingOutline) { + myDiffType = diffType; + myIgnoredFoldingOutline = ignoredFoldingOutline; + } + + @Override public void paint(Editor editor, Graphics g, Rectangle range) { Color color = myDiffType.getColor(editor); EditorGutterComponentEx gutter = ((EditorEx)editor).getGutterComponentEx(); Graphics2D g2 = (Graphics2D)g; - int x = 0; + int x1 = 0; + int x2 = x1 + gutter.getWidth(); int y = range.y; - int width = gutter.getWidth(); int height = range.height; if (height > 2) { - g.setColor(color); - g.fillRect(x, y, width, height); - DiffDrawUtil.drawChunkBorderLine(g2, x, x + width, y - 1, color); - DiffDrawUtil.drawChunkBorderLine(g2, x, x + width, y + height - 1, color); + if (myIgnoredFoldingOutline) { + int xOutline = gutter.getWhitespaceSeparatorOffset(); + + g.setColor(myDiffType.getIgnoredColor(editor)); + g.fillRect(xOutline, y, x2 - xOutline, height); + + g.setColor(color); + g.fillRect(x1, y, xOutline - x1, height); + } + else { + g.setColor(color); + g.fillRect(x1, y, x2 - x1, height); + } + DiffDrawUtil.drawChunkBorderLine(g2, x1, x2, y - 1, color); + DiffDrawUtil.drawChunkBorderLine(g2, x1, x2, y + height - 1, color); } else { // range is empty - insertion or deletion // Draw 2 pixel line in that case - DiffDrawUtil.drawDoubleChunkBorderLine(g2, x, x + width, y - 1, color); + DiffDrawUtil.drawDoubleChunkBorderLine(g2, x1, x2, y - 1, color); } } } From c08649b355268c2195389df7db24b249bb54c625 Mon Sep 17 00:00:00 2001 From: Anna Kozlova Date: Thu, 2 Apr 2015 17:15:55 +0200 Subject: [PATCH 02/21] new overload resolution: avoid second conflict resolution for the same conflicts, e.g. because resulted array would be available inside guard of ResolveCache (IDEA-138596) --- .../JavaMethodsConflictResolver.java | 9 ++++ ...ondConflictResolutionOnSameMethodCall.java | 16 +++++++ .../lambda/Java8ExpressionsCheckTest.java | 47 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/expressions/SecondConflictResolutionOnSameMethodCall.java create mode 100644 java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8ExpressionsCheckTest.java diff --git a/java/java-psi-impl/src/com/intellij/psi/scope/conflictResolvers/JavaMethodsConflictResolver.java b/java/java-psi-impl/src/com/intellij/psi/scope/conflictResolvers/JavaMethodsConflictResolver.java index 5755a53c69ef..a9e818c6905e 100644 --- a/java/java-psi-impl/src/com/intellij/psi/scope/conflictResolvers/JavaMethodsConflictResolver.java +++ b/java/java-psi-impl/src/com/intellij/psi/scope/conflictResolvers/JavaMethodsConflictResolver.java @@ -71,6 +71,15 @@ public class JavaMethodsConflictResolver implements PsiConflictResolver{ @Override public final CandidateInfo resolveConflict(@NotNull final List conflicts){ + final MethodCandidateInfo.CurrentCandidateProperties properties = MethodCandidateInfo.getCurrentMethod(myArgumentsList); + if (properties != null) { + final PsiMethod method = properties.getMethod(); + for (CandidateInfo conflict : conflicts) { + if (conflict.getElement() == method) { + return conflict; + } + } + } return MethodCandidateInfo.ourOverloadGuard.doPreventingRecursion(myArgumentsList, true, new Computable() { @Override public CandidateInfo compute() { diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/expressions/SecondConflictResolutionOnSameMethodCall.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/expressions/SecondConflictResolutionOnSameMethodCall.java new file mode 100644 index 000000000000..b541d523a898 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/expressions/SecondConflictResolutionOnSameMethodCall.java @@ -0,0 +1,16 @@ +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.Stream; + +class CommandTest { + + public static class Command { + public String[] getKeywords() { return new String[] {"GET", "PUT", "POST"}; } + public String getDescription() { return "Some HTTP command"; } + } + + + public static void main(Stream stream) { + stream.map(cmd -> Arrays.stream(cmd.getKeywords()).map(key -> String.format("%s -> %s", key, cmd.getDescription()))).flatMap(Function.identity()); + } +} \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8ExpressionsCheckTest.java b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8ExpressionsCheckTest.java new file mode 100644 index 000000000000..644663c3e471 --- /dev/null +++ b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8ExpressionsCheckTest.java @@ -0,0 +1,47 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.codeInsight.daemon.lambda; + +import com.intellij.codeInsight.daemon.LightDaemonAnalyzerTestCase; +import com.intellij.openapi.projectRoots.Sdk; +import com.intellij.psi.PsiMethodCallExpression; +import com.intellij.psi.util.PsiTreeUtil; +import com.intellij.testFramework.IdeaTestUtil; +import org.jetbrains.annotations.NonNls; + +import java.util.Collection; + +public class Java8ExpressionsCheckTest extends LightDaemonAnalyzerTestCase { + @NonNls static final String BASE_PATH = "/codeInsight/daemonCodeAnalyzer/lambda/expressions"; + + public void testSecondConflictResolutionOnSameMethodCall() throws Exception { + doTestAllMethodCallExpressions(); + } + + private void doTestAllMethodCallExpressions() { + configureByFile(BASE_PATH + "/" + getTestName(false) + ".java"); + final Collection methodCallExpressions = PsiTreeUtil.findChildrenOfType(getFile(), PsiMethodCallExpression.class); + for (PsiMethodCallExpression expression : methodCallExpressions) { + getPsiManager().dropResolveCaches(); + assertNotNull("Failed inference for: " + expression.getText(), expression.getType()); + } + } + + @Override + protected Sdk getProjectJDK() { + return IdeaTestUtil.getMockJdk18(); + } +} From 3b40026df9ebf264906fbc9cf9a49e2db1e9417a Mon Sep 17 00:00:00 2001 From: Anna Kozlova Date: Thu, 2 Apr 2015 17:26:37 +0200 Subject: [PATCH 03/21] testdata for IDEA-138696 --- .../lambda/regression/IDEA138696.java | 26 +++++++++++++++++++ .../daemon/lambda/Java8RegressionTest.java | 4 +++ 2 files changed, 30 insertions(+) create mode 100644 java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/regression/IDEA138696.java diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/regression/IDEA138696.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/regression/IDEA138696.java new file mode 100644 index 000000000000..c5e8f3cb20a8 --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/regression/IDEA138696.java @@ -0,0 +1,26 @@ +import java.io.IOException; + +class CommandTest { + + public URL someFunction(URI uri) { + return unchecked(uri::toURL); + } + + public interface UncheckedRun { + public T run() throws Throwable; + } + + public static T unchecked(UncheckedRun run) { + try { + return run.run(); + } catch (Throwable throwable) { + throw new AssertionError(); + } + } +} + +abstract class URI { + abstract URL toURL() throws IOException; +} + +class URL {} \ No newline at end of file diff --git a/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8RegressionTest.java b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8RegressionTest.java index c0d0fe1deaaf..26878d133536 100644 --- a/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8RegressionTest.java +++ b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/Java8RegressionTest.java @@ -63,6 +63,10 @@ public class Java8RegressionTest extends LightDaemonAnalyzerTestCase { doTest(); } + public void testIDEA138696() throws Exception { + doTest(); + } + private void doTest() { doTest(false); } From 80e3760c69d6b368221916f71196fa27b0ceb0c1 Mon Sep 17 00:00:00 2001 From: Anna Kozlova Date: Thu, 2 Apr 2015 17:49:40 +0200 Subject: [PATCH 04/21] redundant cast: ensure lambda return statements are not mixed with those from containing method (IDEA-138724) --- .../intellij/psi/util/RedundantCastUtil.java | 2 +- .../LambdaReturnExpressions1.java | 19 +++++++++++++++++++ .../lambda/LambdaRedundantCastTest.java | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/redundantCast/LambdaReturnExpressions1.java diff --git a/java/java-psi-api/src/com/intellij/psi/util/RedundantCastUtil.java b/java/java-psi-api/src/com/intellij/psi/util/RedundantCastUtil.java index d11b4a9cf984..2c438205c692 100644 --- a/java/java-psi-api/src/com/intellij/psi/util/RedundantCastUtil.java +++ b/java/java-psi-api/src/com/intellij/psi/util/RedundantCastUtil.java @@ -159,7 +159,7 @@ public class RedundantCastUtil { } @Override public void visitReturnStatement(PsiReturnStatement statement) { - final PsiMethod method = PsiTreeUtil.getParentOfType(statement, PsiMethod.class); + final PsiMethod method = PsiTreeUtil.getParentOfType(statement, PsiMethod.class, true, PsiLambdaExpression.class); if (method != null) { final PsiType returnType = method.getReturnType(); final PsiExpression returnValue = statement.getReturnValue(); diff --git a/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/redundantCast/LambdaReturnExpressions1.java b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/redundantCast/LambdaReturnExpressions1.java new file mode 100644 index 000000000000..412998e6316f --- /dev/null +++ b/java/java-tests/testData/codeInsight/daemonCodeAnalyzer/lambda/redundantCast/LambdaReturnExpressions1.java @@ -0,0 +1,19 @@ +import java.util.Optional; + +class CommandTest { + public Object getObject() { + Optional o = Optional.of("x"); + return o.map(bx -> { + return (Object)""; + }) + .orElse(new Integer(1)); + } + + public Object getObject1() { + Optional o = Optional.of("x"); + return o.map(bx -> { + return (Object)new Object(); + }) + .orElse(new Integer(1)); + } +} diff --git a/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/LambdaRedundantCastTest.java b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/LambdaRedundantCastTest.java index f66b9b95ad3b..78287076576e 100644 --- a/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/LambdaRedundantCastTest.java +++ b/java/java-tests/testSrc/com/intellij/codeInsight/daemon/lambda/LambdaRedundantCastTest.java @@ -37,6 +37,7 @@ public class LambdaRedundantCastTest extends LightDaemonAnalyzerTestCase { public void testIntersection() { doTest(); } public void testSer() { doTest(); } public void testLambdaReturnExpressions() { doTest(); } + public void testLambdaReturnExpressions1() { doTest(); } private void doTest() { doTest(BASE_PATH + "/" + getTestName(false) + ".java", true, false); } From 2bafe3458ec87f01fea8decb71ced598599c4736 Mon Sep 17 00:00:00 2001 From: "Maxim.Mossienko" Date: Thu, 2 Apr 2015 18:11:21 +0200 Subject: [PATCH 05/21] make visitedRoot to be concurrent set --- .../src/com/intellij/util/indexing/FileBasedIndexImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java index 7524630e9992..e6245b6108a2 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/FileBasedIndexImpl.java @@ -2608,7 +2608,7 @@ public class FileBasedIndexImpl extends FileBasedIndex { }); }*/ - final Set visitedRoots = new THashSet(); + final Set visitedRoots = ContainerUtil.newConcurrentSet(); for (IndexedRootsProvider provider : Extensions.getExtensions(IndexedRootsProvider.EP_NAME)) { //important not to depend on project here, to support per-project background reindex // each client gives a project to FileBasedIndex From 588427be73ccd786a47866b68e1a88061aa71585 Mon Sep 17 00:00:00 2001 From: Sergey Malenkov Date: Thu, 2 Apr 2015 18:55:37 +0300 Subject: [PATCH 06/21] Configurable refactoring: build tree of configurable (tested) --- .../ex/ConfigurableExtensionPointUtil.java | 142 +++++++++++++++++- 1 file changed, 135 insertions(+), 7 deletions(-) diff --git a/platform/platform-impl/src/com/intellij/openapi/options/ex/ConfigurableExtensionPointUtil.java b/platform/platform-impl/src/com/intellij/openapi/options/ex/ConfigurableExtensionPointUtil.java index 06637c48383d..54e153aaad8b 100644 --- a/platform/platform-impl/src/com/intellij/openapi/options/ex/ConfigurableExtensionPointUtil.java +++ b/platform/platform-impl/src/com/intellij/openapi/options/ex/ConfigurableExtensionPointUtil.java @@ -132,35 +132,130 @@ public class ConfigurableExtensionPointUtil { } /** - * @param project the project used to load application settings + * @param configurables a list of settings to process + * @return the map of different groups of settings + */ + public static Map> groupConfigurables(@NotNull List configurables) { + Map> tree = ContainerUtil.newHashMap(); + for (Configurable configurable : configurables) { + if (configurable instanceof ConfigurableWrapper) { + ConfigurableWrapper wrapper = (ConfigurableWrapper)configurable; + String id = wrapper.getId(); + Node node = Node.get(tree, id); + if (node.myValue != null) { + LOG.warn("ignore configurable with duplicated id: " + id); + } + else { + String parentId = wrapper.getParentId(); + String groupId = wrapper.getExtensionPoint().groupId; + if (groupId != null) { + if (parentId != null) { + LOG.warn("ignore deprecated groupId: " + groupId + " for id: " + id); + } + else { + LOG.warn("use deprecated groupId instead of parentId: " + groupId + " for id: " + id); + parentId = groupId; + } + } + if (Node.cyclic(tree, parentId, node)) { + LOG.warn("ignore cyclic dependency: " + parentId + " cannot contain " + id); + parentId = null; + } + node.myParent = Node.add(tree, parentId, node); + node.myValue = wrapper; + } + } + else { + Node.add(tree, null, configurable); + } + } + Map> map = ContainerUtil.newHashMap(); + for (String id : tree.keySet().toArray(new String[tree.size()])) { + Node node = tree.get(id); + if (node != null) { + List list = getConfigurables(tree, node); + if (list != null) { + map.put(id, list); + tree.remove(id); + } + } + } + return map; + } + + /** + * @param tree a map that represents a tree of nodes + * @param node a current node to process children recursively + * @return the list of settings for a group or {@code null} for internal node + */ + private static List getConfigurables(Map> tree, Node node) { + List list = ContainerUtil.newArrayListWithCapacity(node.myChildren.size()); + for (Iterator iterator = node.myChildren.iterator(); iterator.hasNext(); iterator.remove()) { + Object child = iterator.next(); + if (child instanceof Configurable) { + list.add((Configurable)child); + } + else { + @SuppressWarnings("unchecked") // expected type + Node value = (Node)child; + if (getConfigurables(tree, value) != null) { + throw new IllegalStateException("unexpected algorithm state"); + } + list.add(value.myValue); + tree.remove(value.myValue.getId()); + } + } + if (node.myValue == null) { + return list; // for group only + } + for (Configurable configurable : list) { + node.myValue = node.myValue.addChild(configurable); + } + return null; + } + + /** + * @param project a project used to load project settings or {@code null} * @param withIdeSettings specifies whether to load application settings or not * @param loadComponents specifies whether to load Configurable components or not - * @return the list of all available settings according to parameters + * @return the list of all valid settings according to parameters */ - private static List getAllConfigurables(@Nullable Project project, boolean withIdeSettings, boolean loadComponents) { + private static List getConfigurables(@Nullable Project project, boolean withIdeSettings, boolean loadComponents) { List list = ContainerUtil.newArrayList(); if (withIdeSettings) { Application application = ApplicationManager.getApplication(); if (application != null) { if (loadComponents) { - ContainerUtil.addAll(list, application.getComponents(Configurable.class)); + addValid(list, application.getComponents(Configurable.class), null); } for (ConfigurableEP extension : application.getExtensions(Configurable.APPLICATION_CONFIGURABLE)) { - ContainerUtil.addIfNotNull(list, ConfigurableWrapper.wrapConfigurable(extension)); + addValid(list, ConfigurableWrapper.wrapConfigurable(extension), null); } } } if (project != null) { if (loadComponents) { - ContainerUtil.addAll(list, project.getComponents(Configurable.class)); + addValid(list, project.getComponents(Configurable.class), project); } for (ConfigurableEP extension : project.getExtensions(Configurable.PROJECT_CONFIGURABLE)) { - ContainerUtil.addIfNotNull(list, ConfigurableWrapper.wrapConfigurable(extension)); + addValid(list, ConfigurableWrapper.wrapConfigurable(extension), project); } } return list; } + private static void addValid(List list, Configurable configurable, Project project) { + if (isValid(configurable, project)) { + list.add(configurable); + } + } + + private static void addValid(List list, Configurable[] configurables, Project project) { + for (Configurable configurable : configurables) { + addValid(list, configurable, project); + } + } + /** * @param configurable settings component to validate * @param project current project, default template project or {@code null} for IDE settings @@ -285,4 +380,37 @@ public class ConfigurableExtensionPointUtil { } return null; } + + /** + * Utility class that helps to build a tree. + */ + private static final class Node { + List myChildren = ContainerUtil.newArrayList(); + Node myParent; + V myValue; + + private static Node get(Map> tree, I id) { + Node node = tree.get(id); + if (node == null) { + node = new Node(); + tree.put(id, node); + } + return node; + } + + private static Node add(Map> tree, I id, Object child) { + Node node = get(tree, id); + node.myChildren.add(child); + return node; + } + + private static boolean cyclic(Map> tree, I id, Node parent) { + for (Node node = tree.get(id); node != null; node = node.myParent) { + if (node == parent) { + return true; + } + } + return false; + } + } } From bdd5cf9cc485346e231f94f439dbc53b1e3344c1 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 26 Mar 2015 16:26:32 +0300 Subject: [PATCH 07/21] properly refresh file system on getVirtualFileByName --- .../testSrc/com/jetbrains/python/fixtures/PyTestCase.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/testSrc/com/jetbrains/python/fixtures/PyTestCase.java b/python/testSrc/com/jetbrains/python/fixtures/PyTestCase.java index 3b73d5f59e27..a44ed7a65d8a 100644 --- a/python/testSrc/com/jetbrains/python/fixtures/PyTestCase.java +++ b/python/testSrc/com/jetbrains/python/fixtures/PyTestCase.java @@ -86,7 +86,12 @@ public abstract class PyTestCase extends UsefulTestCase { @Nullable protected static VirtualFile getVirtualFileByName(String fileName) { - return LocalFileSystem.getInstance().refreshAndFindFileByPath(fileName.replace(File.separatorChar, '/')); + final VirtualFile path = LocalFileSystem.getInstance().findFileByPath(fileName.replace(File.separatorChar, '/')); + if (path != null) { + refreshRecursively(path); + return path; + } + return null; } @Override From e1d916dd04286ad05182515c4a4004b6b4f246d0 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Wed, 1 Apr 2015 19:36:07 +0300 Subject: [PATCH 08/21] PY-15302 Numpy: TypeChecker: false positive for random.rayleigh: expand scalar type with ndarray added ufunc rules --- .../NumpyDocStringTypeProvider.java | 16 ++- .../numpy/codeInsight/NumpyUfuncs.java | 106 ++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 python/src/com/jetbrains/numpy/codeInsight/NumpyUfuncs.java diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index 0d7c73481048..8ad65b1633ae 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -46,7 +46,7 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray"); NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.ndarray", "numpy.core.multiarray.ndarray"); // 184 occurrences - NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "collections.Iterable or int or long or float or complex"); + NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "collections.Iterable"); // Parameters marked as 'data-type' actually get any Python type identifier such as 'bool' or // an instance of 'numpy.core.multiarray.dtype', however the type checker isn't able to check it. // 30 occurrences @@ -88,6 +88,8 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { // Function returns single value final String typeName = returns.get(0).getType(); if (typeName != null) { + final PyType genericType = getPsiFacade(function).parseTypeAnnotation("T", function); + if (isUfuncType(function, typeName)) return genericType; return parseNumpyDocType(function, typeName); } return null; @@ -188,6 +190,16 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { return getPsiFacade(anchor).createUnionType(types); } + private static boolean isUfuncType(@NotNull PsiElement anchor, @NotNull final String typeString) { + for (String typeName : NumPyDocString.getNumpyUnionType(typeString)) { + if (anchor instanceof PyFunction && NumpyUfuncs.isUFunc(((PyFunction)anchor).getName()) && + (typeName.equals("array_like") || typeName.equals("ndarray"))) { + return true; + } + } + return false; + } + @Nullable private static PyType getParameterType(@NotNull PyFunction function, @NotNull String parameterName) { final NumPyDocString docString = NumPyDocString.forFunction(function, function); @@ -200,6 +212,8 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { parameter = docString.getNamedParameter(parameterName.substring(2)); } if (parameter != null) { + final PyType numberType = getPsiFacade(function).parseTypeAnnotation("T <= numbers.Number|numpy.core.multiarray.ndarray", function); + if (isUfuncType(function, parameter.getType())) return numberType; return parseNumpyDocType(function, parameter.getType()); } } diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyUfuncs.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyUfuncs.java new file mode 100644 index 000000000000..5f5abc8ce584 --- /dev/null +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyUfuncs.java @@ -0,0 +1,106 @@ +package com.jetbrains.numpy.codeInsight; + +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; + +public class NumpyUfuncs { + private static final List UFUNC_LIST = new ArrayList(); + + public static boolean isUFunc(@Nullable final String name) { + return UFUNC_LIST.contains(name); + } + + static { + //Math operations + UFUNC_LIST.add("add"); + UFUNC_LIST.add("subtract"); + UFUNC_LIST.add("multiply"); + UFUNC_LIST.add("divide"); + UFUNC_LIST.add("logaddexp"); + UFUNC_LIST.add("logaddexp2"); + UFUNC_LIST.add("true_divide"); + UFUNC_LIST.add("floor_divide"); + UFUNC_LIST.add("negative"); + UFUNC_LIST.add("power"); + UFUNC_LIST.add("remainder"); + UFUNC_LIST.add("mod"); + UFUNC_LIST.add("fmod"); + UFUNC_LIST.add("absolute"); + UFUNC_LIST.add("rint"); + UFUNC_LIST.add("sign"); + UFUNC_LIST.add("conj"); + UFUNC_LIST.add("exp"); + UFUNC_LIST.add("exp2"); + UFUNC_LIST.add("log"); + UFUNC_LIST.add("log2"); + UFUNC_LIST.add("log10"); + UFUNC_LIST.add("expm1"); + UFUNC_LIST.add("log1p"); + UFUNC_LIST.add("sqrt"); + UFUNC_LIST.add("square"); + UFUNC_LIST.add("reciprocal"); + UFUNC_LIST.add("ones_like"); + + //Trigonometric + UFUNC_LIST.add("sin"); + UFUNC_LIST.add("cos"); + UFUNC_LIST.add("tan"); + UFUNC_LIST.add("arcsin"); + UFUNC_LIST.add("arccos"); + UFUNC_LIST.add("arctan"); + UFUNC_LIST.add("arctan2"); + UFUNC_LIST.add("hypot"); + UFUNC_LIST.add("sinh"); + UFUNC_LIST.add("cosh"); + UFUNC_LIST.add("tanh"); + UFUNC_LIST.add("arcsinh"); + UFUNC_LIST.add("arccosh"); + UFUNC_LIST.add("arctanh"); + UFUNC_LIST.add("deg2rad"); + UFUNC_LIST.add("rad2deg"); + + //Bit-twiddling functions + UFUNC_LIST.add("bitwise_and"); + UFUNC_LIST.add("bitwise_or"); + UFUNC_LIST.add("bitwise_xor"); + UFUNC_LIST.add("invert"); + UFUNC_LIST.add("left_shift"); + UFUNC_LIST.add("right_shift"); + + //Comparison functions + UFUNC_LIST.add("greater"); + UFUNC_LIST.add("greater_equal"); + UFUNC_LIST.add("less"); + UFUNC_LIST.add("less_equal"); + UFUNC_LIST.add("not_equal"); + UFUNC_LIST.add("equal"); + UFUNC_LIST.add("logical_and"); + UFUNC_LIST.add("logical_or"); + UFUNC_LIST.add("logical_xor"); + UFUNC_LIST.add("logical_not"); + UFUNC_LIST.add("maximum"); + UFUNC_LIST.add("minimum"); + UFUNC_LIST.add("fmax"); + UFUNC_LIST.add("fmin"); + + ///Floating functions + UFUNC_LIST.add("isreal"); + UFUNC_LIST.add("iscomplex"); + UFUNC_LIST.add("isfinite"); + UFUNC_LIST.add("isinf"); + UFUNC_LIST.add("isnan"); + UFUNC_LIST.add("signbit"); + UFUNC_LIST.add("copysign"); + UFUNC_LIST.add("nextafter"); + UFUNC_LIST.add("modf"); + UFUNC_LIST.add("ldexp"); + UFUNC_LIST.add("frexp"); + UFUNC_LIST.add("fmod"); + UFUNC_LIST.add("floor"); + UFUNC_LIST.add("ceil"); + UFUNC_LIST.add("trunc"); + + } +} From 493e762adc5c3f824dff612b700d5ae1f53b2f61 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Wed, 1 Apr 2015 20:40:46 +0300 Subject: [PATCH 09/21] PY-15296 Numpy: allow passing all sequence types for array and array-like parameters one can pass iterable instad of array and vice versa --- .../numpy/codeInsight/NumpyDocStringTypeProvider.java | 8 ++++---- .../com/jetbrains/numpy/documentation/NumPyDocString.java | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index 8ad65b1633ae..75f5a2f42b52 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -43,10 +43,10 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { private static final Map NUMPY_ALIAS_TO_REAL_TYPE = new HashMap(); static { - NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray"); + NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray or collections.Iterable"); NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.ndarray", "numpy.core.multiarray.ndarray"); // 184 occurrences - NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "numpy.core.multiarray.ndarray or collections.Iterable"); // Parameters marked as 'data-type' actually get any Python type identifier such as 'bool' or // an instance of 'numpy.core.multiarray.dtype', however the type checker isn't able to check it. // 30 occurrences @@ -55,8 +55,8 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { // 16 occurrences NUMPY_ALIAS_TO_REAL_TYPE.put("scalar", "int or long or float or complex"); // 10 occurrences - NUMPY_ALIAS_TO_REAL_TYPE.put("array", "numpy.core.multiarray.ndarray"); - NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.array", "numpy.core.multiarray.ndarray"); + NUMPY_ALIAS_TO_REAL_TYPE.put("array", "numpy.core.multiarray.ndarray or collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.array", "numpy.core.multiarray.ndarray or collections.Iterable"); // 9 occurrences NUMPY_ALIAS_TO_REAL_TYPE.put("any", "object"); // 5 occurrences diff --git a/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java index 72ab80c195a0..42af2edf5bf4 100644 --- a/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java +++ b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java @@ -283,10 +283,6 @@ public class NumPyDocString { if (index >= 0) { return typeString.substring(0, index); } - index = typeString.indexOf(" of "); // e.g. ndarray of dtype float - if (index >= 0) { - return typeString.substring(0, index); - } return typeString; } From 6c468c099be2c26d1c56b3a9f29c6633777fe85d Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Wed, 1 Apr 2015 20:53:32 +0300 Subject: [PATCH 10/21] PY-15296 Numpy: allow passing all sequence types for array and array-like parameters replace every occurence with real type name --- .../NumpyDocStringTypeProvider.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index 75f5a2f42b52..ee196f7b03bf 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -17,6 +17,7 @@ package com.jetbrains.numpy.codeInsight; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.Ref; +import com.intellij.openapi.util.text.StringUtil; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.psi.PsiElement; import com.intellij.psi.PsiFile; @@ -143,13 +144,8 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { @Nullable private static PyType parseSingleNumpyDocType(@NotNull PsiElement anchor, @NotNull String typeString) { final PyPsiFacade facade = getPsiFacade(anchor); - final String realTypeName = NUMPY_ALIAS_TO_REAL_TYPE.get(typeString); - if (realTypeName != null) { - final PyType type = facade.parseTypeAnnotation(realTypeName, anchor); - if (type != null) { - return type; - } - } + typeString = getNumpyRealTypeName(typeString); + final PyType type = facade.parseTypeAnnotation(typeString, anchor); if (type != null) { return type; @@ -157,6 +153,22 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { return getNominalType(anchor, typeString); } + @NotNull + private static String getNumpyRealTypeName(@NotNull String typeString) { + final String realTypeName = NUMPY_ALIAS_TO_REAL_TYPE.get(typeString); + if (realTypeName != null) { + return realTypeName; + } + final List typeSubStrings = StringUtil.split(typeString, " "); + List typeParts = new ArrayList(); + for (String string : typeSubStrings) { + final String type = NUMPY_ALIAS_TO_REAL_TYPE.get(string); + typeParts.add(type != null ? type : string); + } + typeString = StringUtil.join(typeParts, " "); + return typeString; + } + /** * Converts literal into type, e.g. -1 -> int, 'fro' -> str */ From 60d853e324de7d52d1a898eee8a24dffb2f10245 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Wed, 1 Apr 2015 21:05:27 +0300 Subject: [PATCH 11/21] PY-15296 Numpy: allow passing all sequence types for array and array-like parameters special-cased all "size" params, because numpy arrays coud have ndim=(1,) so it should be possible to pass int as size --- .../codeInsight/NumpyDocStringTypeProvider.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index ee196f7b03bf..624a8120d028 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -15,6 +15,7 @@ */ package com.jetbrains.numpy.codeInsight; +import com.google.common.collect.Lists; import com.intellij.openapi.application.ApplicationManager; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.text.StringUtil; @@ -48,6 +49,7 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.ndarray", "numpy.core.multiarray.ndarray"); // 184 occurrences NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "numpy.core.multiarray.ndarray or collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("array-like", "numpy.core.multiarray.ndarray or collections.Iterable"); // Parameters marked as 'data-type' actually get any Python type identifier such as 'bool' or // an instance of 'numpy.core.multiarray.dtype', however the type checker isn't able to check it. // 30 occurrences @@ -70,6 +72,8 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { NUMPY_ALIAS_TO_REAL_TYPE.put("number", "int or long or float or complex"); NUMPY_ALIAS_TO_REAL_TYPE.put("sequence", "collections.Iterable"); + + NUMPY_ALIAS_TO_REAL_TYPE.put("ints", "int"); } @Nullable @@ -205,7 +209,7 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { private static boolean isUfuncType(@NotNull PsiElement anchor, @NotNull final String typeString) { for (String typeName : NumPyDocString.getNumpyUnionType(typeString)) { if (anchor instanceof PyFunction && NumpyUfuncs.isUFunc(((PyFunction)anchor).getName()) && - (typeName.equals("array_like") || typeName.equals("ndarray"))) { + ("array_like".equals(typeName) || "ndarray".equals(typeName))) { return true; } } @@ -224,9 +228,14 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { parameter = docString.getNamedParameter(parameterName.substring(2)); } if (parameter != null) { - final PyType numberType = getPsiFacade(function).parseTypeAnnotation("T <= numbers.Number|numpy.core.multiarray.ndarray", function); - if (isUfuncType(function, parameter.getType())) return numberType; - return parseNumpyDocType(function, parameter.getType()); + if (isUfuncType(function, parameter.getType())) { + return getPsiFacade(function).parseTypeAnnotation("T <= numbers.Number|numpy.core.multiarray.ndarray", function); + } + final PyType numpyDocType = parseNumpyDocType(function, parameter.getType()); + if ("size".equals(parameterName)) { + return getPsiFacade(function).createUnionType(Lists.newArrayList(numpyDocType, PyBuiltinCache.getInstance(function).getIntType())); + } + return numpyDocType; } } return null; From ee45301b111d740f5165d2a54f4d6da191c4980b Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 12:54:36 +0300 Subject: [PATCH 12/21] PY-15295 Numpy: allow passing list arguments for tuple expected types treat all collections as iterable --- .../numpy/codeInsight/NumpyDocStringTypeProvider.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index 624a8120d028..2478a2d5a1cb 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -71,7 +71,11 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { // 3 occurrences NUMPY_ALIAS_TO_REAL_TYPE.put("number", "int or long or float or complex"); + //treat all collections as iterable NUMPY_ALIAS_TO_REAL_TYPE.put("sequence", "collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("set", "collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("list", "collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("tuple", "collections.Iterable"); NUMPY_ALIAS_TO_REAL_TYPE.put("ints", "int"); } From c5255cc25313f98f96b47fb32682e472a7d78991 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 16:10:26 +0300 Subject: [PATCH 13/21] added tests for PY-15295 --- .../NumpyDocStringTypeProvider.java | 2 +- .../inspections/PyNumpyType/ArgSort.py | 89 +++++++++++++ .../testData/inspections/PyNumpyType/Empty.py | 46 +++++++ .../inspections/PyNumpyType/Transpose.py | 47 +++++++ .../inspections/PyNumpyType/Vectorize.py | 122 ++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 16 +++ 6 files changed, 321 insertions(+), 1 deletion(-) create mode 100644 python/testData/inspections/PyNumpyType/ArgSort.py create mode 100644 python/testData/inspections/PyNumpyType/Empty.py create mode 100644 python/testData/inspections/PyNumpyType/Transpose.py create mode 100644 python/testData/inspections/PyNumpyType/Vectorize.py diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index 2478a2d5a1cb..f5ee66376ddd 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -45,7 +45,7 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { private static final Map NUMPY_ALIAS_TO_REAL_TYPE = new HashMap(); static { - NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray or collections.Iterable"); + NUMPY_ALIAS_TO_REAL_TYPE.put("ndarray", "numpy.core.multiarray.ndarray"); NUMPY_ALIAS_TO_REAL_TYPE.put("numpy.ndarray", "numpy.core.multiarray.ndarray"); // 184 occurrences NUMPY_ALIAS_TO_REAL_TYPE.put("array_like", "numpy.core.multiarray.ndarray or collections.Iterable"); diff --git a/python/testData/inspections/PyNumpyType/ArgSort.py b/python/testData/inspections/PyNumpyType/ArgSort.py new file mode 100644 index 000000000000..59e682cb7a1d --- /dev/null +++ b/python/testData/inspections/PyNumpyType/ArgSort.py @@ -0,0 +1,89 @@ + + +def argsort(a, axis=-1, kind='quicksort', order=None): + """ + Returns the indices that would sort an array. + + Perform an indirect sort along the given axis using the algorithm specified + by the `kind` keyword. It returns an array of indices of the same shape as + `a` that index data along the given axis in sorted order. + + Parameters + ---------- + a : array_like + Array to sort. + axis : int or None, optional + Axis along which to sort. The default is -1 (the last axis). If None, + the flattened array is used. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. + order : list, optional + When `a` is an array with fields defined, this argument specifies + which fields to compare first, second, etc. Not all fields need be + specified. + + Returns + ------- + index_array : ndarray, int + Array of indices that sort `a` along the specified axis. + In other words, ``a[index_array]`` yields a sorted `a`. + + See Also + -------- + sort : Describes sorting algorithms used. + lexsort : Indirect stable sort with multiple keys. + ndarray.sort : Inplace sort. + argpartition : Indirect partial sort. + + Notes + ----- + See `sort` for notes on the different sorting algorithms. + + As of NumPy 1.4.0 `argsort` works with real/complex arrays containing + nan values. The enhanced sort order is documented in `sort`. + + Examples + -------- + One dimensional array: + + >>> x = np.array([3, 1, 2]) + >>> np.argsort(x) + array([1, 2, 0]) + + Two-dimensional array: + + >>> x = np.array([[0, 3], [2, 2]]) + >>> x + array([[0, 3], + [2, 2]]) + + >>> np.argsort(x, axis=0) + array([[0, 1], + [1, 0]]) + + >>> np.argsort(x, axis=1) + array([[0, 1], + [0, 1]]) + + Sorting with keys: + + >>> x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> x + array([(1, 0), (0, 1)], + dtype=[('x', '>> np.argsort(x, order=('x','y')) + array([1, 0]) + + >>> np.argsort(x, order=('y','x')) + array([0, 1]) + + """ + try: + argsort = a.argsort + except AttributeError: + return _wrapit(a, 'argsort', axis, kind, order) + return argsort(axis, kind, order) + +x = np.array([(1, 0), (0, 1)], dtype=[('x', '>> np.empty([2, 2]) + array([[ -9.74499359e+001, 6.69583040e-309], + [ 2.13182611e-314, 3.06959433e-309]]) #random + + >>> np.empty([2, 2], dtype=int) + array([[-1073741821, -1067949133], + [ 496041986, 19249760]]) #random + """ + pass + +empty([2, 2]) \ No newline at end of file diff --git a/python/testData/inspections/PyNumpyType/Transpose.py b/python/testData/inspections/PyNumpyType/Transpose.py new file mode 100644 index 000000000000..fe16529bdc27 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/Transpose.py @@ -0,0 +1,47 @@ + +def transpose(a, axes=None): + """ + Permute the dimensions of an array. + + Parameters + ---------- + a : array_like + Input array. + axes : list of ints, optional + By default, reverse the dimensions, otherwise permute the axes + according to the values given. + + Returns + ------- + p : ndarray + `a` with its axes permuted. A view is returned whenever + possible. + + See Also + -------- + rollaxis + + Examples + -------- + >>> x = np.arange(4).reshape((2,2)) + >>> x + array([[0, 1], + [2, 3]]) + + >>> np.transpose(x) + array([[0, 2], + [1, 3]]) + + >>> x = np.ones((1, 2, 3)) + >>> np.transpose(x, (1, 0, 2)).shape + (2, 1, 3) + + """ + try: + transpose = a.transpose + except AttributeError: + return _wrapit(a, 'transpose', axes) + return transpose(axes) + +x = np.ones((1, 2, 3)) +a = transpose(x, (1, 0, 2)).shape \ No newline at end of file diff --git a/python/testData/inspections/PyNumpyType/Vectorize.py b/python/testData/inspections/PyNumpyType/Vectorize.py new file mode 100644 index 000000000000..520aca8f78b4 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/Vectorize.py @@ -0,0 +1,122 @@ + +class vectorize(object): + """ + vectorize(pyfunc, otypes='', doc=None, excluded=None, cache=False) + + Generalized function class. + + Define a vectorized function which takes a nested sequence + of objects or numpy arrays as inputs and returns a + numpy array as output. The vectorized function evaluates `pyfunc` over + successive tuples of the input arrays like the python map function, + except it uses the broadcasting rules of numpy. + + The data type of the output of `vectorized` is determined by calling + the function with the first element of the input. This can be avoided + by specifying the `otypes` argument. + + Parameters + ---------- + pyfunc : callable + A python function or method. + otypes : str or list of dtypes, optional + The output data type. It must be specified as either a string of + typecode characters or a list of data type specifiers. There should + be one data type specifier for each output. + doc : str, optional + The docstring for the function. If `None`, the docstring will be the + ``pyfunc.__doc__``. + excluded : set, optional + Set of strings or integers representing the positional or keyword + arguments for which the function will not be vectorized. These will be + passed directly to `pyfunc` unmodified. + + .. versionadded:: 1.7.0 + + cache : bool, optional + If `True`, then cache the first function call that determines the number + of outputs if `otypes` is not provided. + + .. versionadded:: 1.7.0 + + Returns + ------- + vectorized : callable + Vectorized function. + + Examples + -------- + >>> def myfunc(a, b): + ... "Return a-b if a>b, otherwise return a+b" + ... if a > b: + ... return a - b + ... else: + ... return a + b + + >>> vfunc = np.vectorize(myfunc) + >>> vfunc([1, 2, 3, 4], 2) + array([3, 4, 1, 2]) + + The docstring is taken from the input function to `vectorize` unless it + is specified + + >>> vfunc.__doc__ + 'Return a-b if a>b, otherwise return a+b' + >>> vfunc = np.vectorize(myfunc, doc='Vectorized `myfunc`') + >>> vfunc.__doc__ + 'Vectorized `myfunc`' + + The output type is determined by evaluating the first element of the input, + unless it is specified + + >>> out = vfunc([1, 2, 3, 4], 2) + >>> type(out[0]) + + >>> vfunc = np.vectorize(myfunc, otypes=[np.float]) + >>> out = vfunc([1, 2, 3, 4], 2) + >>> type(out[0]) + + + The `excluded` argument can be used to prevent vectorizing over certain + arguments. This can be useful for array-like arguments of a fixed length + such as the coefficients for a polynomial as in `polyval`: + + >>> def mypolyval(p, x): + ... _p = list(p) + ... res = _p.pop(0) + ... while _p: + ... res = res*x + _p.pop(0) + ... return res + >>> vpolyval = np.vectorize(mypolyval, excluded=['p']) + >>> vpolyval(p=[1, 2, 3], x=[0, 1]) + array([3, 6]) + + Positional arguments may also be excluded by specifying their position: + + >>> vpolyval.excluded.add(0) + >>> vpolyval([1, 2, 3], x=[0, 1]) + array([3, 6]) + + Notes + ----- + The `vectorize` function is provided primarily for convenience, not for + performance. The implementation is essentially a for loop. + + If `otypes` is not specified, then a call to the function with the + first argument will be used to determine the number of outputs. The + results of this call will be cached if `cache` is `True` to prevent + calling the function twice. However, to implement the cache, the + original function must be wrapped which will slow down subsequent + calls, so only do this if your function is expensive. + + The new keyword argument interface and `excluded` argument support + further degrades performance. + + """ + + def __init__(self, pyfunc, otypes='', doc=None, excluded=None, + cache=False): + pass + +def mypolyval(): pass +vpolyval = vectorize(mypolyval, excluded=['p']) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index 26ad34baee3e..326f616e60ff 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -33,4 +33,20 @@ public class PyNumpyTypeTest extends PyTestCase { public void testDtype() { doTest(); } + + public void testEmpty() { + doTest(); + } + + public void testTranspose() { + doTest(); + } + + public void testArgSort() { + doTest(); + } + + public void testVectorize() { + doTest(); + } } From 6a253e178b17d052688e5a4bc382ee0d2b6f107b Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 16:31:47 +0300 Subject: [PATCH 14/21] PY-15298 Numpy: Type Checker: do not higlight argument equal to default as unexpected type do not check type for argument equals default --- .../inspections/PyTypeCheckerInspection.java | 11 +++++-- .../inspections/PyNumpyType/DefaultValue.py | 31 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 4 +++ 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 python/testData/inspections/PyNumpyType/DefaultValue.py diff --git a/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java b/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java index 4de73fbc47e0..75bb46cd5f7c 100644 --- a/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java @@ -93,20 +93,27 @@ public class PyTypeCheckerInspection extends PyInspection { boolean genericsCollected = false; for (Map.Entry entry : results.getArguments().entrySet()) { final PyNamedParameter p = entry.getValue(); + final PyExpression key = entry.getKey(); if (p.isPositionalContainer() || p.isKeywordContainer()) { // TODO: Support *args, **kwargs continue; } + if (p.hasDefaultValue()) { + final PyExpression value = p.getDefaultValue(); + final String keyName = key.getName(); + if (value != null && keyName != null && keyName.equals(value.getName())) + continue; + } final PyType paramType = myTypeEvalContext.getType(p); if (paramType == null) { continue; } - final PyType argType = myTypeEvalContext.getType(entry.getKey()); + final PyType argType = myTypeEvalContext.getType(key); if (!genericsCollected) { substitutions.putAll(PyTypeChecker.unifyReceiver(results.getReceiver(), myTypeEvalContext)); genericsCollected = true; } - checkTypes(paramType, argType, entry.getKey(), myTypeEvalContext, substitutions); + checkTypes(paramType, argType, key, myTypeEvalContext, substitutions); } } } diff --git a/python/testData/inspections/PyNumpyType/DefaultValue.py b/python/testData/inspections/PyNumpyType/DefaultValue.py new file mode 100644 index 000000000000..f37ac60f457d --- /dev/null +++ b/python/testData/inspections/PyNumpyType/DefaultValue.py @@ -0,0 +1,31 @@ +def lstrip(a, chars=None): + """ + For each element in `a`, return a copy with the leading characters + removed. + + Calls `str.lstrip` element-wise. + + Parameters + ---------- + a : array-like, {str, unicode} + Input array. + + chars : {str, unicode}, optional + The `chars` argument is a string specifying the set of + characters to be removed. If omitted or None, the `chars` + argument defaults to removing whitespace. The `chars` argument + is not a prefix; rather, all combinations of its values are + stripped. + + Returns + ------- + out : ndarray, {str, unicode} + Output array of str or unicode, depending on input type + + + """ + a_arr = numpy.asarray(a) + return _vec_string(a_arr, a_arr.dtype, 'lstrip', (chars,)) + +c = np.array(['aAaAaA', ' aA ', 'abBABba']) +lstrip(c, None) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index 326f616e60ff..a46abd359b02 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -49,4 +49,8 @@ public class PyNumpyTypeTest extends PyTestCase { public void testVectorize() { doTest(); } + + public void testDefaultValue() { + doTest(); + } } From c2490f811429566fe3d4b2bd4702fda502311a15 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 16:37:40 +0300 Subject: [PATCH 15/21] test PY-15304 Numpy: TypeChecker: overstrict check for the order param in sort method --- .../testData/inspections/PyNumpyType/Sort.py | 23 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 4 ++++ 2 files changed, 27 insertions(+) create mode 100644 python/testData/inspections/PyNumpyType/Sort.py diff --git a/python/testData/inspections/PyNumpyType/Sort.py b/python/testData/inspections/PyNumpyType/Sort.py new file mode 100644 index 000000000000..824bca5fe6a3 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/Sort.py @@ -0,0 +1,23 @@ +def sort(self, axis=-1, kind='quicksort', order=None): # real signature unknown; restored from __doc__ + """ + a.sort(axis=-1, kind='quicksort', order=None) + + Sort an array, in-place. + + Parameters + ---------- + axis : int, optional + Axis along which to sort. Default is -1, which means sort along the + last axis. + kind : {'quicksort', 'mergesort', 'heapsort'}, optional + Sorting algorithm. Default is 'quicksort'. + order : list, optional + When `a` is an array with fields defined, this argument specifies + which fields to compare first, second, etc. Not all fields need be + specified. + + """ + pass + +a = np.array([('a', 2), ('c', 1)], dtype=[('x', 'S1'), ('y', int)]) +print(sort(a, order='y')) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index a46abd359b02..e27376d4e832 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -53,4 +53,8 @@ public class PyNumpyTypeTest extends PyTestCase { public void testDefaultValue() { doTest(); } + + public void testSort() { + doTest(); + } } From 53228479630470632eaec512d3751c1e799e4e59 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 16:40:01 +0300 Subject: [PATCH 16/21] test PY-15302 Numpy: TypeChecker: false positive for random.rayleigh: expand scalar type with ndarray --- .../testData/inspections/PyNumpyType/UFunc.py | 31 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 4 +++ 2 files changed, 35 insertions(+) create mode 100644 python/testData/inspections/PyNumpyType/UFunc.py diff --git a/python/testData/inspections/PyNumpyType/UFunc.py b/python/testData/inspections/PyNumpyType/UFunc.py new file mode 100644 index 000000000000..34ff9499c832 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/UFunc.py @@ -0,0 +1,31 @@ +def sqrt(x, out=None): # real signature unknown; restored from __doc__ + """ + sqrt(x[, out]) + + Return the positive square-root of an array, element-wise. + + Parameters + ---------- + x : array_like + The values whose square-roots are required. + out : ndarray, optional + Alternate array object in which to put the result; if provided, it + must have the same shape as `x` + + Returns + ------- + y : ndarray + An array of the same shape as `x`, containing the positive + square-root of each element in `x`. If any element in `x` is + complex, a complex array is returned (and the square-roots of + negative reals are calculated). If all of the elements in `x` + are real, so is `y`, with negative elements returning ``nan``. + If `out` was provided, `y` is a reference to it. + + """ + pass + + +meanvalue = 1 +modevalue = sqrt(2 / np.pi) * meanvalue +s = np.random.rayleigh(modevalue, 1000000) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index e27376d4e832..e53f1b977ba3 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -57,4 +57,8 @@ public class PyNumpyTypeTest extends PyTestCase { public void testSort() { doTest(); } + + public void testUFunc() { + doTest(); + } } From 8ab0661bc1161ea2634c0d4ad4ea8c9dbcf43f71 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 16:49:19 +0300 Subject: [PATCH 17/21] PY-15298 Numpy: Type Checker: do not higlight argument equal to default as unexpected type keyword argument --- .../inspections/PyTypeCheckerInspection.java | 8 ++- .../PyNumpyType/DefaultValueKeyword.py | 59 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 4 ++ 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 python/testData/inspections/PyNumpyType/DefaultValueKeyword.py diff --git a/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java b/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java index 75bb46cd5f7c..509803fb2b4c 100644 --- a/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java +++ b/python/src/com/jetbrains/python/inspections/PyTypeCheckerInspection.java @@ -99,8 +99,12 @@ public class PyTypeCheckerInspection extends PyInspection { continue; } if (p.hasDefaultValue()) { - final PyExpression value = p.getDefaultValue(); - final String keyName = key.getName(); + final PyExpression value = p.getDefaultValue(); + String keyName = key.getName(); + if (key instanceof PyKeywordArgument) { + final PyExpression valueExpression = ((PyKeywordArgument)key).getValueExpression(); + keyName = valueExpression != null ? valueExpression.getName() : ""; + } if (value != null && keyName != null && keyName.equals(value.getName())) continue; } diff --git a/python/testData/inspections/PyNumpyType/DefaultValueKeyword.py b/python/testData/inspections/PyNumpyType/DefaultValueKeyword.py new file mode 100644 index 000000000000..9ae9aca6137c --- /dev/null +++ b/python/testData/inspections/PyNumpyType/DefaultValueKeyword.py @@ -0,0 +1,59 @@ +def set_printoptions(precision=None, threshold=None, edgeitems=None, + linewidth=None, suppress=None, + nanstr=None, infstr=None, + formatter=None): + """ + Set printing options. + + These options determine the way floating point numbers, arrays and + other NumPy objects are displayed. + + Parameters + ---------- + precision : int, optional + Number of digits of precision for floating point output (default 8). + threshold : int, optional + Total number of array elements which trigger summarization + rather than full repr (default 1000). + edgeitems : int, optional + Number of array items in summary at beginning and end of + each dimension (default 3). + linewidth : int, optional + The number of characters per line for the purpose of inserting + line breaks (default 75). + suppress : bool, optional + Whether or not suppress printing of small floating point values + using scientific notation (default False). + nanstr : str, optional + String representation of floating point not-a-number (default nan). + infstr : str, optional + String representation of floating point infinity (default inf). + formatter : dict of callables, optional + If not None, the keys should indicate the type(s) that the respective + formatting function applies to. Callables should return a string. + Types that are not specified (by their corresponding keys) are handled + by the default formatters. Individual types for which a formatter + can be set are:: + + - 'bool' + - 'int' + - 'timedelta' : a `numpy.timedelta64` + - 'datetime' : a `numpy.datetime64` + - 'float' + - 'longfloat' : 128-bit floats + - 'complexfloat' + - 'longcomplexfloat' : composed of two 128-bit floats + - 'numpy_str' : types `numpy.string_` and `numpy.unicode_` + - 'str' : all other strings + + Other keys that can be used to set a group of types at once are:: + + - 'all' : sets all types + - 'int_kind' : sets 'int' + - 'float_kind' : sets 'float' and 'longfloat' + - 'complex_kind' : sets 'complexfloat' and 'longcomplexfloat' + - 'str_kind' : sets 'str' and 'numpystr' + """ + pass + +set_printoptions(formatter=None) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index e53f1b977ba3..84bc1a9f5372 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -54,6 +54,10 @@ public class PyNumpyTypeTest extends PyTestCase { doTest(); } + public void testDefaultValueKeyword() { + doTest(); + } + public void testSort() { doTest(); } From f4ff6c2a78a32de8146855f2f1f9ac7fdc612f05 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 17:35:54 +0300 Subject: [PATCH 18/21] PY-15286 Numpy: Tuple assignment balance is incorrect: false positive for unique: return type is not detected properly for optional return parameters --- .../NumpyDocStringTypeProvider.java | 28 ++++++++++-- .../inspections/PyNumpyType/ReturnOptional.py | 43 ++++++++++++++++++ .../inspections/PyNumpyType/ReturnTuple.py | 44 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 12 +++++ 4 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 python/testData/inspections/PyNumpyType/ReturnOptional.py create mode 100644 python/testData/inspections/PyNumpyType/ReturnTuple.py diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index f5ee66376ddd..b9d995963bae 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -104,12 +104,34 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { return null; default: // Function returns a tuple + final ArrayList unionMembers = new ArrayList(); + final List members = new ArrayList(); - for (NumPyDocStringParameter ret : returns) { + + for (int i = 0; i < returns.size(); i++) { + NumPyDocStringParameter ret = returns.get(i); final String memberTypeName = ret.getType(); - members.add(memberTypeName != null ? parseNumpyDocType(function, memberTypeName) : null); + final PyType returnType = memberTypeName != null ? parseNumpyDocType(function, memberTypeName) : null; + final boolean isOptional = memberTypeName != null && memberTypeName.contains("optional"); + + if (isOptional) { + if (i != 0) { + if(members.size() > 1) + unionMembers.add(facade.createTupleType(members, function)); + else + unionMembers.add(returnType); + } + } + members.add(returnType); + + if (i == returns.size() - 1 && isOptional) { + unionMembers.add(facade.createTupleType(members, function)); + } } - return facade.createTupleType(members, function); + if (unionMembers.isEmpty()) { + return facade.createTupleType(members, function); + } + return facade.createUnionType(unionMembers); } } } diff --git a/python/testData/inspections/PyNumpyType/ReturnOptional.py b/python/testData/inspections/PyNumpyType/ReturnOptional.py new file mode 100644 index 000000000000..391cf06e6579 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/ReturnOptional.py @@ -0,0 +1,43 @@ +def unique(ar, return_index=False, return_inverse=False, return_counts=False): + """ + Find the unique elements of an array. + + Returns the sorted unique elements of an array. There are two optional + outputs in addition to the unique elements: the indices of the input array + that give the unique values, and the indices of the unique array that + reconstruct the input array. + + Parameters + ---------- + ar : array_like + Input array. This will be flattened if it is not already 1-D. + return_index : bool, optional + If True, also return the indices of `ar` that result in the unique + array. + return_inverse : bool, optional + If True, also return the indices of the unique array that can be used + to reconstruct `ar`. + return_counts : bool, optional + .. versionadded:: 1.9.0 + If True, also return the number of times each unique value comes up + in `ar`. + + Returns + ------- + unique : ndarray + The sorted unique values. + unique_indices : ndarray, optional + The indices of the first occurrences of the unique values in the + (flattened) original array. Only provided if `return_index` is True. + unique_inverse : ndarray, optional + The indices to reconstruct the (flattened) original array from the + unique array. Only provided if `return_inverse` is True. + unique_counts : ndarray, optional + .. versionadded:: 1.9.0 + The number of times each of the unique values comes up in the + original array. Only provided if `return_counts` is True. + + """ + ar = np.asanyarray(ar).flatten() + +u, indices = unique(a, return_index=True) \ No newline at end of file diff --git a/python/testData/inspections/PyNumpyType/ReturnTuple.py b/python/testData/inspections/PyNumpyType/ReturnTuple.py new file mode 100644 index 000000000000..d96798f39bd3 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/ReturnTuple.py @@ -0,0 +1,44 @@ +def unique(ar, return_index=False, return_inverse=False, return_counts=False): + """ + Find the unique elements of an array. + + Returns the sorted unique elements of an array. There are two optional + outputs in addition to the unique elements: the indices of the input array + that give the unique values, and the indices of the unique array that + reconstruct the input array. + + Parameters + ---------- + ar : array_like + Input array. This will be flattened if it is not already 1-D. + return_index : bool, optional + If True, also return the indices of `ar` that result in the unique + array. + return_inverse : bool, optional + If True, also return the indices of the unique array that can be used + to reconstruct `ar`. + return_counts : bool, optional + .. versionadded:: 1.9.0 + If True, also return the number of times each unique value comes up + in `ar`. + + Returns + ------- + unique : ndarray + The sorted unique values. + unique_indices : ndarray + The indices of the first occurrences of the unique values in the + (flattened) original array. Only provided if `return_index` is True. + unique_inverse : ndarray + The indices to reconstruct the (flattened) original array from the + unique array. Only provided if `return_inverse` is True. + unique_counts : ndarray + .. versionadded:: 1.9.0 + The number of times each of the unique values comes up in the + original array. Only provided if `return_counts` is True. + + """ + ar = np.asanyarray(ar).flatten() + +a=1 +u, indices = unique(a, return_index=True) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index 84bc1a9f5372..e0e30889cd72 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -65,4 +65,16 @@ public class PyNumpyTypeTest extends PyTestCase { public void testUFunc() { doTest(); } + + public void testReturnOptional() { + myFixture.configureByFile(TEST_DIRECTORY + getTestName(false) + ".py"); + myFixture.enableInspections(PyTupleAssignmentBalanceInspection.class); + myFixture.checkHighlighting(true, false, true); + } + + public void testReturnTuple() { + myFixture.configureByFile(TEST_DIRECTORY + getTestName(false) + ".py"); + myFixture.enableInspections(PyTupleAssignmentBalanceInspection.class); + myFixture.checkHighlighting(true, false, true); + } } From e30a31744863b14d5aaadb7bd2a63fe637585768 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 18:13:29 +0300 Subject: [PATCH 19/21] PY-15287 Numpy: Tuple assignment balance is incorrect: false positive for slogdet: invalid type detected for the return value --- .../numpy/documentation/NumPyDocString.java | 16 ++++++---- .../inspections/PyNumpyType/Slogdet.py | 30 +++++++++++++++++++ .../python/inspections/PyNumpyTypeTest.java | 6 ++++ 3 files changed, 46 insertions(+), 6 deletions(-) create mode 100644 python/testData/inspections/PyNumpyType/Slogdet.py diff --git a/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java index 42af2edf5bf4..f8563020dfc7 100644 --- a/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java +++ b/python/src/com/jetbrains/numpy/documentation/NumPyDocString.java @@ -45,6 +45,7 @@ public class NumPyDocString { private static final Pattern PARAMETER_WITHOUT_TYPE = Pattern.compile("^([^ :,]+)$"); private static final Pattern REDIRECT = Pattern.compile("^Refer to `(.*)` for full documentation.$"); private static final Pattern NUMPY_UNION_PATTERN = Pattern.compile("^\\{(.*)\\}$"); + private static final Pattern NUMPY_ARRAY_PATTERN = Pattern.compile("(\\(\\.\\.\\..*\\))(.*)"); private static final Pattern QUOTED_STRING_PATTERN = Pattern.compile("^(?:\\\"(.*)\\\")|(?:\\'(.*)\\')$"); private final String mySignature; @@ -251,19 +252,21 @@ public class NumPyDocString { DocStringParameterBuilder builder = null; for (String line : lines) { if (!HAS_INDENT.matcher(line).find()) { - if (builder != null) { - parameters.add(builder.build()); - } builder = new DocStringParameterBuilder(); Matcher parameterWithTypeMatcher = PARAMETER_WITH_TYPE.matcher(line); if (parameterWithTypeMatcher.matches()) { builder.setName(parameterWithTypeMatcher.group(1)); builder.setType(parameterWithTypeMatcher.group(2)); + parameters.add(builder.build()); } else { Matcher parameterWithoutTypeMatcher = PARAMETER_WITHOUT_TYPE.matcher(line); if (parameterWithoutTypeMatcher.matches()) { builder.setName(parameterWithoutTypeMatcher.group(1)); builder.setType("object"); + parameters.add(builder.build()); + } + else { + builder.appendDescription(line.trim()); } } } else { @@ -272,9 +275,6 @@ public class NumPyDocString { } } } - if (builder != null) { - parameters.add(builder.build()); - } } @NotNull @@ -288,6 +288,10 @@ public class NumPyDocString { @NotNull public static List getNumpyUnionType(@NotNull String typeString) { + final Matcher arrayMatcher = NUMPY_ARRAY_PATTERN.matcher(typeString); + if (arrayMatcher.matches()) { + typeString = arrayMatcher.group(2); + } Matcher matcher = NUMPY_UNION_PATTERN.matcher(typeString); if (matcher.matches()) { typeString = matcher.group(1); diff --git a/python/testData/inspections/PyNumpyType/Slogdet.py b/python/testData/inspections/PyNumpyType/Slogdet.py new file mode 100644 index 000000000000..cbe4f4e5a983 --- /dev/null +++ b/python/testData/inspections/PyNumpyType/Slogdet.py @@ -0,0 +1,30 @@ + +def slogdet(a): + """ + Compute the sign and (natural) logarithm of the determinant of an array. + + If an array has a very small or very large determinant, than a call to + `det` may overflow or underflow. This routine is more robust against such + issues, because it computes the logarithm of the determinant rather than + the determinant itself. + + Parameters + ---------- + a : (..., M, M) array_like + Input array, has to be a square 2-D array. + + Returns + ------- + sign : (...) array_like + A number representing the sign of the determinant. For a real matrix, + this is 1, 0, or -1. For a complex matrix, this is a complex number + with absolute value 1 (i.e., it is on the unit circle), or else 0. + logdet : (...) array_like + The natural log of the absolute value of the determinant. + + If the determinant is zero, then `sign` will be 0 and `logdet` will be + -Inf. In all cases, the determinant is equal to ``sign * np.exp(logdet)``. + """ + pass + +(sign, logdet) = slogdet(a) \ No newline at end of file diff --git a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java index e0e30889cd72..2b6ca4acfd6a 100644 --- a/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java +++ b/python/testSrc/com/jetbrains/python/inspections/PyNumpyTypeTest.java @@ -77,4 +77,10 @@ public class PyNumpyTypeTest extends PyTestCase { myFixture.enableInspections(PyTupleAssignmentBalanceInspection.class); myFixture.checkHighlighting(true, false, true); } + + public void testSlogdet() { + myFixture.configureByFile(TEST_DIRECTORY + getTestName(false) + ".py"); + myFixture.enableInspections(PyTupleAssignmentBalanceInspection.class); + myFixture.checkHighlighting(true, false, true); + } } From e54b0ef049621589331e77444e55128c6a1fe8d9 Mon Sep 17 00:00:00 2001 From: Ekaterina Tuzova Date: Thu, 2 Apr 2015 18:25:11 +0300 Subject: [PATCH 20/21] PY-15282 Numpy: Function doesn't return anything: false positive for tensordot function could return value even if it's not declared --- .../numpy/codeInsight/NumpyDocStringTypeProvider.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java index b9d995963bae..1804e7ebbe11 100644 --- a/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java +++ b/python/src/com/jetbrains/numpy/codeInsight/NumpyDocStringTypeProvider.java @@ -91,8 +91,7 @@ public class NumpyDocStringTypeProvider extends PyTypeProviderBase { final PyPsiFacade facade = getPsiFacade(function); switch (returns.size()) { case 0: - // Function returns nothing - return facade.parseTypeAnnotation("None", function); + return null; case 1: // Function returns single value final String typeName = returns.get(0).getType(); From c3f294867f7699d5dd32ad1355976b5043d896fc Mon Sep 17 00:00:00 2001 From: Dmitry Batkovich Date: Thu, 2 Apr 2015 19:51:47 +0300 Subject: [PATCH 21/21] IDEA-86919 navigate to the next incomplete property in resource bundle --- ...esourceBundleFileStructureViewElement.java | 61 ++++++++++--------- plugins/properties/src/META-INF/plugin.xml | 4 ++ .../GotoNextIncompletePropertyAction.java | 40 ++++++++++++ .../editor/ResourceBundleEditor.java | 34 +++++++++++ 4 files changed, 111 insertions(+), 28 deletions(-) create mode 100644 plugins/properties/src/com/intellij/lang/properties/editor/GotoNextIncompletePropertyAction.java diff --git a/plugins/properties/properties-psi-impl/src/com/intellij/lang/properties/editor/ResourceBundleFileStructureViewElement.java b/plugins/properties/properties-psi-impl/src/com/intellij/lang/properties/editor/ResourceBundleFileStructureViewElement.java index a158a0b2a5db..dde89d00cdca 100644 --- a/plugins/properties/properties-psi-impl/src/com/intellij/lang/properties/editor/ResourceBundleFileStructureViewElement.java +++ b/plugins/properties/properties-psi-impl/src/com/intellij/lang/properties/editor/ResourceBundleFileStructureViewElement.java @@ -60,10 +60,40 @@ public class ResourceBundleFileStructureViewElement implements StructureViewTree return myResourceBundle; } - private Map getChildrenIdShowOnlyIncomplete() { + @NotNull + public StructureViewTreeElement[] getChildren() { + final Map propertyNames = getPropertiesMap(myResourceBundle, myShowOnlyIncomplete); + List result = new ArrayList(propertyNames.size()); + for (IProperty property : propertyNames.values()) { + result.add(new ResourceBundlePropertyStructureViewElement(myResourceBundle, property)); + } + return result.toArray(new StructureViewTreeElement[result.size()]); + } + + public static Map getPropertiesMap(ResourceBundle resourceBundle, boolean onlyIncomplete) { + List propertiesFiles = resourceBundle.getPropertiesFiles(); + final Map propertyNames; + if (onlyIncomplete) { + propertyNames = getChildrenIdShowOnlyIncomplete(resourceBundle); + } else { + propertyNames = new LinkedHashMap(); + for (PropertiesFile propertiesFile : propertiesFiles) { + List properties = propertiesFile.getProperties(); + for (IProperty property : properties) { + String name = property.getKey(); + if (!propertyNames.containsKey(name)) { + propertyNames.put(name, property); + } + } + } + } + return propertyNames; + } + + private static Map getChildrenIdShowOnlyIncomplete(ResourceBundle resourceBundle) { final Map propertyNames = new LinkedHashMap(); TObjectIntHashMap occurrences = new TObjectIntHashMap(); - for (PropertiesFile file : myResourceBundle.getPropertiesFiles()) { + for (PropertiesFile file : resourceBundle.getPropertiesFiles()) { Map currentFilePropertyNames = new LinkedHashMap(); for (IProperty property : file.getProperties()) { String name = property.getKey(); @@ -80,7 +110,7 @@ public class ResourceBundleFileStructureViewElement implements StructureViewTree } } } - final int targetOccurrences = myResourceBundle.getPropertiesFiles().size(); + final int targetOccurrences = resourceBundle.getPropertiesFiles().size(); occurrences.forEachEntry(new TObjectIntProcedure() { @Override public boolean execute(String propertyName, int occurrences) { @@ -93,31 +123,6 @@ public class ResourceBundleFileStructureViewElement implements StructureViewTree return propertyNames; } - @NotNull - public StructureViewTreeElement[] getChildren() { - List propertiesFiles = myResourceBundle.getPropertiesFiles(); - final Map propertyNames; - if (myShowOnlyIncomplete) { - propertyNames = getChildrenIdShowOnlyIncomplete(); - } else { - propertyNames = new LinkedHashMap(); - for (PropertiesFile propertiesFile : propertiesFiles) { - List properties = propertiesFile.getProperties(); - for (IProperty property : properties) { - String name = property.getKey(); - if (!propertyNames.containsKey(name)) { - propertyNames.put(name, property); - } - } - } - } - List result = new ArrayList(propertyNames.size()); - for (IProperty property : propertyNames.values()) { - result.add(new ResourceBundlePropertyStructureViewElement(myResourceBundle, property)); - } - return result.toArray(new StructureViewTreeElement[result.size()]); - } - @NotNull public ItemPresentation getPresentation() { return new ItemPresentation() { diff --git a/plugins/properties/src/META-INF/plugin.xml b/plugins/properties/src/META-INF/plugin.xml index 2a0cfe4623e1..782f31acd4c2 100644 --- a/plugins/properties/src/META-INF/plugin.xml +++ b/plugins/properties/src/META-INF/plugin.xml @@ -134,5 +134,9 @@ description="Add Property to Resource Bundle"> + diff --git a/plugins/properties/src/com/intellij/lang/properties/editor/GotoNextIncompletePropertyAction.java b/plugins/properties/src/com/intellij/lang/properties/editor/GotoNextIncompletePropertyAction.java new file mode 100644 index 000000000000..b7937f1b84ac --- /dev/null +++ b/plugins/properties/src/com/intellij/lang/properties/editor/GotoNextIncompletePropertyAction.java @@ -0,0 +1,40 @@ +/* + * Copyright 2000-2015 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.intellij.lang.properties.editor; + +import com.intellij.openapi.actionSystem.AnAction; +import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.actionSystem.PlatformDataKeys; +import com.intellij.openapi.diagnostic.Logger; + +/** + * @author Dmitry Batkovich + */ +public class GotoNextIncompletePropertyAction extends AnAction { + private final static Logger LOG = Logger.getInstance(GotoNextIncompletePropertyAction.class); + + @Override + public void actionPerformed(AnActionEvent e) { + final ResourceBundleEditor editor = (ResourceBundleEditor)PlatformDataKeys.FILE_EDITOR.getData(e.getDataContext()); + LOG.assertTrue(editor != null); + editor.selectNextIncompleteProperty(); + } + + @Override + public void update(AnActionEvent e) { + e.getPresentation().setEnabled(PlatformDataKeys.FILE_EDITOR.getData(e.getDataContext()) instanceof ResourceBundleEditor); + } +} diff --git a/plugins/properties/src/com/intellij/lang/properties/editor/ResourceBundleEditor.java b/plugins/properties/src/com/intellij/lang/properties/editor/ResourceBundleEditor.java index e21f79d5a3d9..60f4a9939742 100644 --- a/plugins/properties/src/com/intellij/lang/properties/editor/ResourceBundleEditor.java +++ b/plugins/properties/src/com/intellij/lang/properties/editor/ResourceBundleEditor.java @@ -27,9 +27,11 @@ import com.intellij.ide.structureView.newStructureView.StructureViewComponent; import com.intellij.ide.util.treeView.AbstractTreeNode; import com.intellij.ide.util.treeView.AbstractTreeUi; import com.intellij.ide.util.treeView.smartTree.CachingChildrenTreeNode; +import com.intellij.ide.util.treeView.smartTree.Sorter; import com.intellij.ide.util.treeView.smartTree.TreeElement; import com.intellij.lang.properties.IProperty; import com.intellij.lang.properties.PropertiesImplUtil; +import com.intellij.lang.properties.PropertiesUtil; import com.intellij.lang.properties.ResourceBundle; import com.intellij.lang.properties.psi.PropertiesFile; import com.intellij.lang.properties.psi.PropertiesResourceBundleUtil; @@ -682,6 +684,38 @@ public class ResourceBundleEditor extends UserDataHolderBase implements FileEdit return selectedElements.size() == 1 ? ContainerUtil.getFirstItem(selectedElements) : null; } + public void selectNextIncompleteProperty() { + if (getSelectedNodes().size() != 1) { + return; + } + final IProperty selectedProperty = getSelectedProperty(); + if (selectedProperty == null) { + return; + } + + final ResourceBundleFileStructureViewElement root = + (ResourceBundleFileStructureViewElement)myStructureViewComponent.getTreeModel().getRoot(); + final Map propertiesMap = + ResourceBundleFileStructureViewElement.getPropertiesMap(myResourceBundle, root.isShowOnlyIncomplete()); + final boolean isAlphaSorted = myStructureViewComponent.isActionActive(Sorter.ALPHA_SORTER_ID); + final List keysOrder = new ArrayList(propertiesMap.keySet()); + if (isAlphaSorted) { + Collections.sort(keysOrder); + } + + final String currentKey = selectedProperty.getKey(); + final int idx = keysOrder.indexOf(currentKey); + LOG.assertTrue(idx != -1); + for (int i = 1; i < keysOrder.size(); i++) { + int trimmedIndex = (i + idx) % keysOrder.size(); + final String key = keysOrder.get(trimmedIndex); + if (!PropertiesUtil.isPropertyComplete(myResourceBundle, key)) { + selectProperty(key); + return; + } + } + } + @Override @NotNull public JComponent getComponent() {