fixed PY-8004 Doctest: support doctests in restructured text

python/python-rest/resources/META-INF/python-rest-plugin.xml

@@ -13,6 +13,8 @@
     <configurationProducer implementation="com.jetbrains.rest.run.docutils.DocutilsConfigurationProducer"/>
     <configurationProducer implementation="com.jetbrains.rest.run.sphinx.SphinxConfigurationProducer"/>
     <lang.substitutor language="TEXT" implementationClass="com.jetbrains.rest.RestLanguageSubstitutor"/>
+
+    <languageInjector implementation="com.jetbrains.rest.PyRestDocstringLanguageInjector"/>
   </extensions>
 
   <actions>

python/python-rest/src/com/jetbrains/rest/PyRestDocstringLanguageInjector.java

@@ -0,0 +1,73 @@
+package com.jetbrains.rest;
+
+import com.intellij.openapi.util.TextRange;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.psi.InjectedLanguagePlaces;
+import com.intellij.psi.LanguageInjector;
+import com.intellij.psi.PsiLanguageInjectionHost;
+import com.jetbrains.python.documentation.doctest.PyDocstringLanguageDialect;
+import com.jetbrains.rest.psi.RestLine;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.List;
+
+/**
+ * User: ktisha
+ */
+public class PyRestDocstringLanguageInjector implements LanguageInjector {
+  @Override
+  public void getLanguagesToInject(@NotNull final PsiLanguageInjectionHost host, @NotNull final InjectedLanguagePlaces injectionPlacesRegistrar) {
+    if (host instanceof RestLine) {
+      int start = 0;
+      int end = host.getTextLength() - 1;
+      final String text = host.getText();
+      final List<String> strings = StringUtil.split(text, "\n", false);
+
+      boolean gotExample = false;
+
+      int currentPosition = 0;
+      boolean endsWithSlash = false;
+      for (String string : strings) {
+        final String trimmedString = string.trim();
+        if (!trimmedString.startsWith(">>>") && !trimmedString.startsWith("...") && gotExample && start < end) {
+          gotExample = false;
+          if (!endsWithSlash)
+            injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(), TextRange.create(start, end),  null, null);
+        }
+        if (endsWithSlash) {
+          endsWithSlash = false;
+          injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(),
+                                            TextRange.create(start, getEndOffset(currentPosition, string)),  null, null);
+        }
+
+        if (trimmedString.startsWith(">>>")) {
+          if (trimmedString.endsWith("\\"))
+            endsWithSlash = true;
+
+          if (!gotExample)
+            start = currentPosition;
+
+          gotExample = true;
+          end = getEndOffset(currentPosition, string);
+        }
+        else if (trimmedString.startsWith("...") && gotExample) {
+          if (trimmedString.endsWith("\\"))
+            endsWithSlash = true;
+
+          end = getEndOffset(currentPosition, string);
+        }
+        currentPosition += string.length();
+      }
+      if (gotExample && start < end)
+        injectionPlacesRegistrar.addPlace(PyDocstringLanguageDialect.getInstance(), TextRange.create(start, end),  null, null);
+    }
+  }
+
+  private int getEndOffset(int start, String s) {
+    int length = s.length();
+    int end = start + length;
+    if (s.endsWith("\n"))
+      end -= 1;
+    return end;
+  }
+}

python/rest/rest.iml

@@ -5,7 +5,6 @@
     <content url="file://$MODULE_DIR$">
       <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/resources" isTestSource="false" />
-      <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

python/rest/src/com/jetbrains/rest/RestElementTypes.java

@@ -10,5 +10,7 @@ public interface RestElementTypes {
   RestElementType REFERENCE_TARGET = new RestElementType("REFERENCE");
   RestElementType DIRECTIVE_BLOCK = new RestElementType("DIRECTIVE_BLOCK");
   RestElementType INLINE_BLOCK = new RestElementType("INLINE_BLOCK");
+
+  RestElementType LINE_TEXT = new RestElementType("LINE_TEXT");
 }
 

python/rest/src/com/jetbrains/rest/parsing/RestParser.java

@@ -61,6 +61,15 @@ public class RestParser implements PsiParser {
         builder.advanceLexer();
         marker.done(RestElementTypes.REFERENCE_TARGET);
       }
+      else if (type == RestTokenTypes.LINE) {
+        PsiBuilder.Marker marker = builder.mark();
+        while (type == RestTokenTypes.LINE || type == RestTokenTypes.WHITESPACE) {
+          builder.advanceLexer();
+          type = builder.getTokenType();
+        }
+
+        marker.done(RestElementTypes.LINE_TEXT);
+      }
       else
         builder.advanceLexer();
     }

python/rest/src/com/jetbrains/rest/psi/RestASTFactory.java

@@ -33,6 +33,9 @@ public class RestASTFactory implements RestTokenTypes, RestElementTypes {
     if (type == INLINE_BLOCK) {
       return new RestInlineBlock(node);
     }
+    if (type == LINE_TEXT) {
+      return new RestLine(node);
+    }
     return new ASTWrapperPsiElement(node);
   }
 }

python/rest/src/com/jetbrains/rest/psi/RestLine.java

@@ -0,0 +1,61 @@
+package com.jetbrains.rest.psi;
+
+import com.intellij.lang.ASTNode;
+import com.intellij.openapi.util.TextRange;
+import com.intellij.psi.LiteralTextEscaper;
+import com.intellij.psi.PsiLanguageInjectionHost;
+import com.intellij.psi.impl.source.tree.LeafElement;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * User : ktisha
+ */
+public class RestLine extends RestElement implements PsiLanguageInjectionHost {
+  public RestLine(@NotNull final ASTNode node) {
+    super(node);
+  }
+
+  @Override
+  public String toString() {
+    return "RestLine:" + getNode().getElementType().toString();
+  }
+
+
+  @Override
+  public boolean isValidHost() {
+    return true;
+  }
+
+  @Override
+  public PsiLanguageInjectionHost updateText(@NotNull String text) {
+    ASTNode valueNode = getNode().getFirstChildNode();
+    assert valueNode instanceof LeafElement;
+    ((LeafElement)valueNode).replaceWithText(text);
+    return this;
+  }
+
+  @NotNull
+  @Override
+  public LiteralTextEscaper<? extends PsiLanguageInjectionHost> createLiteralTextEscaper() {
+    return new LiteralTextEscaper<PsiLanguageInjectionHost>(this) {
+      @Override
+      public boolean decode(@NotNull TextRange rangeInsideHost, @NotNull StringBuilder outChars) {
+        outChars.append(rangeInsideHost.substring(myHost.getText()));
+        return true;
+      }
+
+      @Override
+      public int getOffsetInHost(int offsetInDecoded, @NotNull TextRange rangeInsideHost) {
+        int offset = offsetInDecoded + rangeInsideHost.getStartOffset();
+        if (offset < rangeInsideHost.getStartOffset()) offset = rangeInsideHost.getStartOffset();
+        if (offset > rangeInsideHost.getEndOffset()) offset = rangeInsideHost.getEndOffset();
+        return offset;
+      }
+
+      @Override
+      public boolean isOneLine() {
+        return false;
+      }
+    };
+  }
+}

python/src/com/jetbrains/python/documentation/doctest/PyDocReference.java

@@ -11,6 +11,7 @@ import com.intellij.psi.PsiFile;
 import com.intellij.psi.PsiLanguageInjectionHost;
 import com.intellij.psi.ResolveResult;
 import com.jetbrains.python.codeInsight.controlflow.ScopeOwner;
+import com.jetbrains.python.codeInsight.dataflow.scope.ScopeUtil;
 import com.jetbrains.python.psi.PyQualifiedExpression;
 import com.jetbrains.python.psi.impl.references.PyReferenceImpl;
 import com.jetbrains.python.psi.resolve.*;
@@ -62,7 +63,8 @@ public class PyDocReference extends PyReferenceImpl {
 
       ResolveProcessor processor = new ResolveProcessor(referencedName);
 
-      PyResolveUtil.scopeCrawlUp(processor, (ScopeOwner)file, referencedName, file);
+      final ScopeOwner scopeOwner = ScopeUtil.getScopeOwner(myElement);
+      PyResolveUtil.scopeCrawlUp(processor, scopeOwner != null? scopeOwner : (ScopeOwner)file, referencedName, file);
       final List<RatedResolveResult> resultList = getResultsFromProcessor(referencedName, processor, file, file);
       if (resultList.size() > 0)
         return resultList.toArray(new RatedResolveResult[resultList.size()]);