Merge branch 'python-fixes'

python/src/com/jetbrains/python/codeInsight/codeFragment/PyCodeFragmentBuilder.java

@@ -139,9 +139,6 @@ public class PyCodeFragmentBuilder extends PyRecursiveElementVisitor {
         }
         // If declaration is before we look for modifications inside
         if (pos == Position.BEFORE) {
-          if (!isTopLevel(element)) {
-            inElements.add(name);
-          }
           final List<PyElement> list = modifiedInsideMap.get(name);
           boolean modificationSeen = false;
           if (list != null) {
@@ -154,6 +151,9 @@ public class PyCodeFragmentBuilder extends PyRecursiveElementVisitor {
               }
             }
             if (modificationSeen) {
+              if (!isTopLevel(element)) {
+                inElements.add(name);
+              }
               break;
             }
           }

python/src/com/jetbrains/python/codeInsight/stdlib/StdlibTypes2.properties

@@ -87,7 +87,7 @@ __builtin__.locals = \
   :rtype: dict of (bytes, object)
 
 __builtin__.map = \
-  :type function: collections.Callable \n\
+  :type function: collections.Callable or None \n\
   :type sequence: collections.Iterable \n\
   :rtype: list \n\
 

python/src/com/jetbrains/python/packaging/PyRequirement.java

@@ -21,7 +21,7 @@ import java.util.regex.Pattern;
 public class PyRequirement {
   private static final Pattern NAME = Pattern.compile("\\s*(\\w(\\w|[-.])*)\\s*(.*)");
   private static final Pattern VERSION_SPEC = Pattern.compile("\\s*(<=?|>=?|==|!=)\\s*((\\w|[-.])+)");
-  private static final Pattern EDITABLE_EGG = Pattern.compile("\\s*-e\\s+([^#]*)#egg=(.*)");
+  private static final Pattern EDITABLE_EGG = Pattern.compile("\\s*(-e)?\\s*([^#]*)#egg=(.*)");
   private static final Pattern RECURSIVE_REQUIREMENT = Pattern.compile("\\s*-r\\s+(.*)");
   private static final Pattern NAME_VERSION = Pattern.compile("\\s*(\\w(\\w|[.])*)-((\\w|[-.])+)");
 
@@ -119,6 +119,7 @@ public class PyRequirement {
   @NotNull private final String myName;
   @NotNull private final List<VersionSpec> myVersionSpecs;
   @Nullable private final String myURL;
+  private final boolean myEditable;
 
   public static final Comparator<String> VERSION_COMPARATOR = new Comparator<String>() {
     @Override
@@ -171,6 +172,14 @@ public class PyRequirement {
         }
         results.add(component);
       }
+      for (int i = results.size() - 1; i > 0; i--) {
+        if ("00000000".equals(results.get(i))) {
+          results.remove(i);
+        }
+        else {
+          break;
+        }
+      }
       results.add("*final");
       return results;
     }
@@ -188,17 +197,19 @@ public class PyRequirement {
     myName = name;
     myVersionSpecs = versionSpecs;
     myURL = null;
+    myEditable = false;
   }
 
-  public PyRequirement(@NotNull String name, @Nullable String version, @NotNull String url) {
+  public PyRequirement(@NotNull String name, @Nullable String version, @NotNull String url, boolean editable) {
     myName = name;
     if (version != null) {
-      myVersionSpecs = Collections.singletonList(new VersionSpec(Relation.GTE, version));
+      myVersionSpecs = Collections.singletonList(new VersionSpec(Relation.EQ, version));
     }
     else {
       myVersionSpecs = Collections.emptyList();
     }
     myURL = url;
+    myEditable = editable;
   }
 
   @NotNull
@@ -216,11 +227,13 @@ public class PyRequirement {
 
   @NotNull
   public List<String> toOptions() {
+    final List<String> results = new ArrayList<String>();
+    if (myEditable) {
+      results.add("-e");
+    }
     if (myURL != null) {
       final int size = myVersionSpecs.size();
       assert size <= 1;
-      final List<String> results = new ArrayList<String>();
-      results.add("-e");
       final String urlAndName = myURL + "#egg=" + myName;
       if (size == 0) {
         results.add(urlAndName);
@@ -233,8 +246,9 @@ public class PyRequirement {
       return results;
     }
     else {
-      return Collections.singletonList(toString());
+      results.add(toString());
     }
+    return results;
   }
 
   @Override
@@ -372,8 +386,9 @@ public class PyRequirement {
     if (!editableEggMatcher.matches()) {
       return null;
     }
-    final String url = editableEggMatcher.group(1);
-    final String egg = editableEggMatcher.group(2);
+    final boolean editable = editableEggMatcher.group(1) != null;
+    final String url = editableEggMatcher.group(2);
+    final String egg = editableEggMatcher.group(3);
     if (url == null) {
       return null;
     }
@@ -381,7 +396,7 @@ public class PyRequirement {
     if (nameVersionMatcher.matches()) {
       final String name = normalizeName(nameVersionMatcher.group(1));
       final String version = normalizeVersion(nameVersionMatcher.group(3));
-      return new PyRequirement(name, version, url);
+      return new PyRequirement(name, version, url, editable);
     }
     else {
       final Matcher nameMatcher = NAME.matcher(egg);
@@ -389,7 +404,7 @@ public class PyRequirement {
         return null;
       }
       final String name = normalizeName(nameMatcher.group(1));
-      return new PyRequirement(name, null, url);
+      return new PyRequirement(name, null, url, editable);
     }
   }
 

python/src/com/jetbrains/python/psi/PySliceExpression.java

@@ -1,11 +1,13 @@
 package com.jetbrains.python.psi;
 
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
  * @author yole
  */
 public interface PySliceExpression extends PyExpression {
+  @NotNull
   PyExpression getOperand();
 
   @Nullable

python/src/com/jetbrains/python/psi/impl/PySliceExpressionImpl.java

@@ -2,11 +2,11 @@ package com.jetbrains.python.psi.impl;
 
 import com.intellij.lang.ASTNode;
 import com.intellij.psi.util.PsiTreeUtil;
-import com.jetbrains.python.PyElementTypes;
 import com.jetbrains.python.PythonDialectsTokenSetProvider;
 import com.jetbrains.python.psi.PyExpression;
 import com.jetbrains.python.psi.PySliceExpression;
 import com.jetbrains.python.psi.PySliceItem;
+import com.jetbrains.python.psi.types.PyTupleType;
 import com.jetbrains.python.psi.types.PyType;
 import com.jetbrains.python.psi.types.TypeEvalContext;
 import org.jetbrains.annotations.NotNull;
@@ -20,15 +20,25 @@ public class PySliceExpressionImpl extends PyElementImpl implements PySliceExpre
     super(astNode);
   }
 
+  @Nullable
+  @Override
   public PyType getType(@NotNull TypeEvalContext context) {
-    return context.getType(getOperand());
+    final PyType type = context.getType(getOperand());
+    // TODO: Currently we don't evaluate the static range of the slice, so we have to return a generic tuple type without elements
+    if (type instanceof PyTupleType) {
+      return PyBuiltinCache.getInstance(this).getTupleType();
+    }
+    return type;
   }
 
+  @NotNull
+  @Override
   public PyExpression getOperand() {
     return childToPsiNotNull(PythonDialectsTokenSetProvider.INSTANCE.getExpressionTokens(), 0);
   }
 
   @Nullable
+  @Override
   public PySliceItem getSliceItem() {
     return PsiTreeUtil.getChildOfType(this, PySliceItem.class);
   }

python/src/com/jetbrains/python/psi/impl/references/PyReferenceImpl.java

@@ -147,9 +147,12 @@ public class PyReferenceImpl implements PsiReferenceEx, PsiPolyVariantReference
       PsiElement definition = instruction.getElement();
       NameDefiner definer = null;
       // TODO: This check may slow down resolving, but it is the current solution to the comprehension scopes problem
-      final PyComprehensionElement defComprh = PsiTreeUtil.getParentOfType(definition, PyComprehensionElement.class);
-      if (defComprh != null && defComprh != PsiTreeUtil.getParentOfType(element, PyComprehensionElement.class)) {
-        continue;
+      final PyComprehensionElement definitionComprehension = PsiTreeUtil.getParentOfType(definition, PyComprehensionElement.class);
+      if (definitionComprehension != null) {
+        final PyComprehensionElement elementComprehension = PsiTreeUtil.getParentOfType(element, PyComprehensionElement.class);
+        if (elementComprehension == null || !PsiTreeUtil.isAncestor(definitionComprehension, elementComprehension, false)) {
+          continue;
+        }
       }
       if (definition instanceof NameDefiner && !(definition instanceof PsiNamedElement)) {
         definer = (NameDefiner)definition;

python/src/com/jetbrains/python/psi/types/PyTypeReference.java

@@ -1,6 +1,7 @@
 package com.jetbrains.python.psi.types;
 
 import com.intellij.psi.PsiElement;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -8,5 +9,5 @@ import org.jetbrains.annotations.Nullable;
  */
 public interface PyTypeReference extends PyType {
   @Nullable
-  PyType resolve(PsiElement context, TypeEvalContext typeEvalContext);
+  PyType resolve(@Nullable PsiElement context, @NotNull TypeEvalContext typeEvalContext);
 }

python/src/com/jetbrains/python/psi/types/PyTypeReferenceImpl.java

@@ -31,8 +31,9 @@ public abstract class PyTypeReferenceImpl implements PyTypeReference {
     return ArrayUtil.EMPTY_OBJECT_ARRAY;
   }
 
+  @Nullable
   @Override
-  public PyType resolve(@Nullable PsiElement context, TypeEvalContext typeEvalContext) {
+  public PyType resolve(@Nullable PsiElement context, @NotNull TypeEvalContext typeEvalContext) {
     Set<PyTypeReferenceImpl> seen = new HashSet<PyTypeReferenceImpl>();
     seen.add(this);
     PyType resolved;

python/testData/inspections/PyTupleAssignmentBalanceInspection3/test.py

@@ -12,4 +12,10 @@ a, b = <warning descr="Too many values to unpack">c</warning>
 a, b = <warning descr="Too many values to unpack">[1, 2, 3]</warning>
 a, b = <warning descr="Too many values to unpack">'str'</warning>
 a, b = <warning descr="Too many values to unpack">{1, 2, 3}</warning>
-a, b = <warning descr="Too many values to unpack">{1:2, 2: 3, 3:4}</warning>
+a, b = <warning descr="Too many values to unpack">{1:2, 2: 3, 3:4}</warning>
+
+# PY-6315
+def test_tuple_slice():
+    def f():
+        return 1, 2, 3
+    x, y = f()[:2]

python/testData/inspections/PyUnresolvedReferencesInspection/nestedComprehensions.py

@@ -0,0 +1,2 @@
+def f(xs, ys):
+    return [[(x, y) for y in ys] for x in xs]

python/testData/refactoring/extractmethod/DefinedBeforeAccessedAfter.after.py

@@ -0,0 +1,9 @@
+def bar():
+    if 11 > 4:
+        print('ok')
+
+
+def main(indices):
+    foo = True
+    bar()
+    return foo

python/testData/refactoring/extractmethod/DefinedBeforeAccessedAfter.before.py

@@ -0,0 +1,5 @@
+def main(indices):
+    foo = True
+    <selection>if 11 > 4:
+        print('ok')</selection>
+    return foo

python/testSrc/com/jetbrains/python/inspections/PyUnresolvedReferencesInspectionTest.java

@@ -97,6 +97,11 @@ public class PyUnresolvedReferencesInspectionTest extends PyTestCase {
     doTest();
   }
 
+  // PY-6316
+  public void testNestedComprehensions() {
+    doTest();
+  }
+
   private void doMultiFileTest() {
     myFixture.copyFileToProject("inspections/PyUnresolvedReferencesInspection/__init__.py", "PyUnresolvedReferencesInspection/__init__.py");
     myFixture.copyFileToProject("inspections/PyUnresolvedReferencesInspection/" + getTestName(true) + ".py",

python/testSrc/com/jetbrains/python/refactoring/PyExtractMethodTest.java

@@ -162,4 +162,9 @@ public class PyExtractMethodTest extends LightMarkedTestCase {
   public void testLocalVarDefinedBeforeModifiedInside() {
     doTest("bar");
   }
+
+  // PY-6391
+  public void testDefinedBeforeAccessedAfter() {
+    doTest("bar");
+  }
 }