MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-02-04 15:06:56 +07:00
Code Issues Packages Projects Releases Wiki Activity

PY-52574: Remove Epytext docstring format & Py2 docstring rendering

Browse Source 
Docstring rendering is no longer supported for Python 2, which became obsolete after reaching its end of life in 2020. Without updates or security patches, most tools, including documentation generators like Epydoc, have shifted focus exclusively to Python 3.


(cherry picked from commit ace78ac9ad943278449d5b20bb92db9f7571b5b5)

IJ-CR-148150

GitOrigin-RevId: 75cc87e05c61c3c17c26689552080e3c3082bfdc
This commit is contained in:
Irina Fediaeva
2024-10-24 20:04:14 +02:00
committed by intellij-monorepo-bot
parent e5b07397c4
commit 2ac1a788e3
17 changed files with 296 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
python/helpers/docstring_formatter.py
View File

@@ -5,7 +5,6 @@ import re
import sys
import textwrap
import six
from six import text_type, u
ENCODING = 'utf-8'
@@ -264,51 +263,11 @@ def format_numpy(docstring):
return format_rest(transformed)
def format_epytext(docstring):
if six.PY3:
return u('Epydoc is not compatible with Python 3 interpreter')
import epydoc.markup.epytext
from epydoc.markup import DocstringLinker
from epydoc.markup.epytext import parse_docstring, ParseError, _colorize
def _add_para(doc, para_token, stack, indent_stack, errors):
"""Colorize the given paragraph, and add it to the DOM tree."""
para = _colorize(doc, para_token, errors)
if para_token.inline:
para.attribs['inline'] = True
stack[-1].children.append(para)
epydoc.markup.epytext._add_para = _add_para
ParseError.is_fatal = lambda self: False
errors = []
class EmptyLinker(DocstringLinker):
def translate_indexterm(self, indexterm):
return ""
def translate_identifier_xref(self, identifier, label=None):
return identifier
docstring = parse_docstring(docstring, errors)
docstring, fields = docstring.split_fields()
html = docstring.to_html(EmptyLinker())
if errors and not html:
# It's not possible to recover original stacktraces of the errors
error_lines = '\n'.join(text_type(e) for e in errors)
raise Exception('Error parsing docstring. Probable causes:\n' + error_lines)
return html
def format_body(docstring_format, input_body):
formatter = {
'rest': format_rest,
'google': format_google,
'numpy': format_numpy,
'epytext': format_epytext
}.get(docstring_format, format_rest)
return formatter(input_body)

4
python/python-grazie/src/com/intellij/python/grazie/PythonTextExtractor.kt
View File

@@ -13,13 +13,11 @@ import com.intellij.psi.impl.source.tree.PsiCommentImpl
import com.intellij.psi.util.PsiUtilCore
import com.jetbrains.python.PyTokenTypes
import com.jetbrains.python.PyTokenTypes.FSTRING_TEXT
import com.jetbrains.python.documentation.docstrings.EpydocString
import com.jetbrains.python.documentation.docstrings.SphinxDocString
import com.jetbrains.python.psi.PyFormattedStringElement
import java.util.regex.Pattern
private val KNOWN_DOCSTRING_TAGS_PATTERN = (SphinxDocString.ALL_TAGS + EpydocString.ALL_TAGS)
.joinToString("|", transform = Pattern::quote, prefix = "(", postfix = ")")
private val KNOWN_DOCSTRING_TAGS_PATTERN = SphinxDocString.ALL_TAGS.joinToString("|", transform = Pattern::quote, prefix = "(", postfix = ")")
private val DOCSTRING_DIRECTIVE_PATTERN = "^$KNOWN_DOCSTRING_TAGS_PATTERN[^\n:]*: *".toPattern(Pattern.MULTILINE)
internal class PythonTextExtractor : TextExtractor() {

2
python/python-psi-impl/resources/messages/PyPsiBundle.properties
View File

@@ -164,8 +164,8 @@ ANN.await.outside.async.function='await' outside async function
### quick doc generator
QDOC.module.path.unknown=(Module path is unknown)
QDOC.epydoc.python2.sdk.not.found=You need a configured Python 2 SDK to render <a href='http://epydoc.sourceforge.net/'>Epydoc</a> docstrings
QDOC.local.sdk.not.found=You need a configured local Python SDK to render docstrings.
QDOC.docstring.rendering.is.not.supported.for.python.2=Docstring rendering is not supported for Python 2 SDK.
QDOC.assigned.to=Assigned to:
QDOC.copied.from=Copied from:
QDOC.accessor.kind=Accessor kind:

81
python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/DocStringTagCompletionContributor.java
View File

@@ -1,81 +0,0 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.documentation.docstrings;
import com.intellij.codeInsight.completion.*;
import com.intellij.codeInsight.lookup.LookupElementBuilder;
import com.intellij.openapi.project.DumbAware;
import com.intellij.patterns.PsiElementPattern;
import com.intellij.psi.PsiFile;
import com.intellij.util.ProcessingContext;
import com.jetbrains.python.psi.PyDocStringOwner;
import com.jetbrains.python.psi.PyExpressionStatement;
import com.jetbrains.python.psi.PyStringLiteralExpression;
import org.jetbrains.annotations.NotNull;
import static com.intellij.patterns.PlatformPatterns.psiElement;
public final class DocStringTagCompletionContributor extends CompletionContributor implements DumbAware {
public static final PsiElementPattern.Capture<PyStringLiteralExpression> DOCSTRING_PATTERN = psiElement(PyStringLiteralExpression.class)
.withParent(psiElement(PyExpressionStatement.class).inside(PyDocStringOwner.class));
public DocStringTagCompletionContributor() {
extend(CompletionType.BASIC, psiElement().withParent(DOCSTRING_PATTERN),
new CompletionProvider<>() {
@Override
protected void addCompletions(@NotNull CompletionParameters parameters,
@NotNull ProcessingContext context,
@NotNull CompletionResultSet result) {
final PsiFile file = parameters.getOriginalFile();
DocStringFormat format = DocStringParser.getConfiguredDocStringFormat(file);
if (format == DocStringFormat.EPYTEXT || format == DocStringFormat.REST) {
int offset = parameters.getOffset();
final String text = file.getText();
char prefix = format == DocStringFormat.EPYTEXT ? '@' : ':';
if (offset > 0) {
offset--;
}
StringBuilder prefixBuilder = new StringBuilder();
while (offset > 0 && (Character.isLetterOrDigit(text.charAt(offset)) || text.charAt(offset) == prefix)) {
prefixBuilder.insert(0, text.charAt(offset));
if (text.charAt(offset) == prefix) {
offset--;
break;
}
offset--;
}
while (offset > 0) {
offset--;
if (text.charAt(offset) == '\n' || text.charAt(offset) == '\"' || text.charAt(offset) == '\'') {
break;
}
if (!Character.isWhitespace(text.charAt(offset))) {
return;
}
}
String[] allTags = format == DocStringFormat.EPYTEXT ? EpydocString.ALL_TAGS : SphinxDocString.ALL_TAGS;
if (prefixBuilder.length() > 0) {
result = result.withPrefixMatcher(prefixBuilder.toString());
}
for (String tag : allTags) {
result.addElement(LookupElementBuilder.create(tag));
}
}
}
});
}
}

84
python/python-psi-impl/src/com/jetbrains/python/documentation/docstrings/DocStringTagCompletionContributor.kt Normal file
View File

@@ -0,0 +1,84 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.documentation.docstrings
import com.intellij.codeInsight.completion.*
import com.intellij.codeInsight.lookup.LookupElementBuilder
import com.intellij.openapi.project.DumbAware
import com.intellij.patterns.PlatformPatterns
import com.intellij.patterns.PsiElementPattern
import com.intellij.util.ProcessingContext
import com.jetbrains.python.psi.PyDocStringOwner
import com.jetbrains.python.psi.PyExpressionStatement
import com.jetbrains.python.psi.PyStringLiteralExpression
class DocStringTagCompletionContributor : CompletionContributor(), DumbAware {
init {
extend(CompletionType.BASIC, PlatformPatterns.psiElement().withParent(DOCSTRING_PATTERN),
object : CompletionProvider<CompletionParameters?>() {
override fun addCompletions(
parameters: CompletionParameters,
context: ProcessingContext,
result: CompletionResultSet,
) {
val file = parameters.originalFile
if (DocStringParser.getConfiguredDocStringFormat(file) == DocStringFormat.REST) {
var offset = parameters.offset
val text = file.getText()
val prefix = ':'
if (offset > 0) {
offset--
}
val prefixBuilder = StringBuilder()
while (offset > 0 && (Character.isLetterOrDigit(text[offset]) || text[offset] == prefix)) {
prefixBuilder.insert(0, text[offset])
if (text[offset] == prefix) {
offset--
break
}
offset--
}
while (offset > 0) {
offset--
if (text[offset] == '\n' || text[offset] == '\"' || text[offset] == '\'') {
break
}
if (!Character.isWhitespace(text[offset])) {
return
}
}
var resultSet = result
if (!prefixBuilder.isEmpty()) {
resultSet = resultSet.withPrefixMatcher(prefixBuilder.toString())
}
for (tag in SphinxDocString.ALL_TAGS) {
resultSet.addElement(LookupElementBuilder.create(tag))
}
}
}
})
}
companion object {
@JvmField
val DOCSTRING_PATTERN: PsiElementPattern.Capture<PyStringLiteralExpression?> =
PlatformPatterns.psiElement<PyStringLiteralExpression?>(PyStringLiteralExpression::class.java)
.withParent(
PlatformPatterns
.psiElement<PyExpressionStatement?>(PyExpressionStatement::class.java)
.inside(PyDocStringOwner::class.java))
}
}

88
python/python-psi-impl/src/com/jetbrains/python/validation/DocStringAnnotator.java
View File

@@ -1,88 +0,0 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.validation;
import com.intellij.lang.annotation.HighlightSeverity;
import com.intellij.openapi.util.TextRange;
import com.jetbrains.python.PyNames;
import com.jetbrains.python.documentation.docstrings.*;
import com.jetbrains.python.highlighting.PyHighlighter;
import com.jetbrains.python.psi.*;
import org.jetbrains.annotations.NotNull;
/**
* Highlights doc strings in classes, functions, and files.
*/
public final class DocStringAnnotator extends PyAnnotator {
@Override
public void visitPyFile(final @NotNull PyFile node) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node));
}
@Override
public void visitPyFunction(final @NotNull PyFunction node) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node.getStatementList()));
}
@Override
public void visitPyClass(final @NotNull PyClass node) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node.getStatementList()));
}
@Override
public void visitPyAssignmentStatement(@NotNull PyAssignmentStatement node) {
if (node.isAssignmentTo(PyNames.DOC)) {
PyExpression right = node.getAssignedValue();
if (right instanceof PyStringLiteralExpression) {
getHolder().newSilentAnnotation(HighlightSeverity.INFORMATION).range(right).textAttributes(PyHighlighter.PY_DOC_COMMENT).create();
annotateDocStringStmt((PyStringLiteralExpression)right);
}
}
}
@Override
public void visitPyExpressionStatement(@NotNull PyExpressionStatement node) {
if (node.getExpression() instanceof PyStringLiteralExpression &&
DocStringUtil.isVariableDocString((PyStringLiteralExpression)node.getExpression())) {
annotateDocStringStmt((PyStringLiteralExpression)node.getExpression());
}
}
private void annotateDocStringStmt(final PyStringLiteralExpression stmt) {
if (stmt != null) {
final DocStringFormat format = DocStringParser.getConfiguredDocStringFormat(stmt);
final String[] tags;
if (format == DocStringFormat.EPYTEXT) {
tags = EpydocString.ALL_TAGS;
}
else if (format == DocStringFormat.REST) {
tags = SphinxDocString.ALL_TAGS;
}
else {
return;
}
int pos = 0;
while (true) {
TextRange textRange = DocStringReferenceProvider.findNextTag(stmt.getText(), pos, tags);
if (textRange == null) break;
getHolder().newSilentAnnotation(
HighlightSeverity.INFORMATION).range(textRange.shiftRight(stmt.getTextRange().getStartOffset())).textAttributes(PyHighlighter.PY_DOC_COMMENT_TAG).create();
pos = textRange.getEndOffset();
}
}
}
}

73
python/python-psi-impl/src/com/jetbrains/python/validation/DocStringAnnotator.kt Normal file
View File

@@ -0,0 +1,73 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.validation
import com.intellij.lang.annotation.HighlightSeverity
import com.jetbrains.python.PyNames
import com.jetbrains.python.documentation.docstrings.*
import com.jetbrains.python.highlighting.PyHighlighter
import com.jetbrains.python.psi.*
/**
* Highlights doc strings in classes, functions, and files.
*/
class DocStringAnnotator : PyAnnotator() {
override fun visitPyFile(node: PyFile) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node))
}
override fun visitPyFunction(node: PyFunction) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node.statementList))
}
override fun visitPyClass(node: PyClass) {
annotateDocStringStmt(DocStringUtil.findDocStringExpression(node.statementList))
}
override fun visitPyAssignmentStatement(node: PyAssignmentStatement) {
if (node.isAssignmentTo(PyNames.DOC)) {
val right = node.assignedValue
if (right is PyStringLiteralExpression) {
holder.newSilentAnnotation(HighlightSeverity.INFORMATION).range(right).textAttributes(PyHighlighter.PY_DOC_COMMENT).create()
annotateDocStringStmt(right)
}
}
}
override fun visitPyExpressionStatement(node: PyExpressionStatement) {
if (node.expression is PyStringLiteralExpression &&
DocStringUtil.isVariableDocString(node.expression as PyStringLiteralExpression)
) {
annotateDocStringStmt(node.expression as PyStringLiteralExpression)
}
}
private fun annotateDocStringStmt(stmt: PyStringLiteralExpression?) {
if (stmt == null) return
if (DocStringParser.getConfiguredDocStringFormat(stmt) == DocStringFormat.REST) {
val tags = SphinxDocString.ALL_TAGS
var pos = 0
while (true) {
val textRange = DocStringReferenceProvider.findNextTag(stmt.getText(), pos, tags)
if (textRange == null) break
holder.newSilentAnnotation(
HighlightSeverity.INFORMATION).range(textRange.shiftRight(stmt.getTextRange().getStartOffset()))
.textAttributes(PyHighlighter.PY_DOC_COMMENT_TAG).create()
pos = textRange.endOffset
}
}
}
}

5
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/DocStringFormat.java
View File

@@ -15,7 +15,6 @@
*/
package com.jetbrains.python.documentation.docstrings;
import com.intellij.psi.PsiElement;
import com.intellij.util.ObjectUtils;
import com.intellij.util.containers.ContainerUtil;
import org.jetbrains.annotations.NotNull;
@@ -25,11 +24,7 @@ import java.util.List;
public enum DocStringFormat {
/**
* @see DocStringUtil#ensureNotPlainDocstringFormat(PsiElement)
*/
PLAIN("Plain", ""),
EPYTEXT("Epytext", "epytext"),
REST("reStructuredText", "rest"),
NUMPY("NumPy", "numpy"),
GOOGLE("Google", "google");

13
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/DocStringParser.java
View File

@@ -30,7 +30,6 @@ public final class DocStringParser {
public static StructuredDocString parseDocString(@NotNull DocStringFormat format, @NotNull Substring content) {
return switch (format) {
case REST -> new SphinxDocString(content);
case EPYTEXT -> new EpydocString(content);
case GOOGLE -> new GoogleCodeStyleDocString(content);
case NUMPY -> new NumpyDocString(content);
case PLAIN -> new PlainDocString(content);
@@ -50,9 +49,6 @@ public final class DocStringParser {
*/
@NotNull
public static DocStringFormat guessDocStringFormat(@NotNull String text) {
if (isLikeEpydocDocString(text)) {
return DocStringFormat.EPYTEXT;
}
if (isLikeSphinxDocString(text)) {
return DocStringFormat.REST;
}
@@ -107,15 +103,6 @@ public final class DocStringParser {
text.contains(":var") || text.contains(":ivar") || text.contains(":cvar");
}
public static boolean isLikeEpydocDocString(@NotNull String text) {
return text.contains("@param ") ||
text.contains("@kwarg ") || text.contains("@keyword ") || text.contains("@kwparam ") ||
text.contains("@raise ") || text.contains("@raises ") || text.contains("@except ") || text.contains("@exception ") ||
text.contains("@return:") ||
text.contains("@rtype") || text.contains("@type") ||
text.contains("@var") || text.contains("@ivar") || text.contains("@cvar");
}
public static boolean isLikeGoogleDocString(@NotNull String text) {
for (@NonNls String title : StringUtil.findMatches(text, GoogleCodeStyleDocString.SECTION_HEADER, 1)) {
if (SectionBasedDocString.isValidSectionTitle(title)) {

292
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/EpydocString.java
View File

@@ -1,292 +0,0 @@
// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
package com.jetbrains.python.documentation.docstrings;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.xml.util.XmlStringUtil;
import com.jetbrains.python.toolbox.Substring;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class EpydocString extends TagBasedDocString {
public static String[] RTYPE_TAGS = new String[] { "rtype", "returntype" };
public static String[] KEYWORD_ARGUMENT_TAGS = new String[] { "keyword", "kwarg", "kwparam" };
public static String[] ALL_TAGS = new String[] {
"@param", "@type", "@return", "@rtype", "@keyword", "@raise", "@ivar", "@cvar", "@var", "@group", "@sort", "@note", "@attention",
"@bug", "@warning", "@version", "@todo", "@deprecated", "@since", "@status", "@change", "@permission", "@requires",
"@precondition", "@postcondition", "@invariant", "@author", "@organization", "@copyright", "@license", "@contact", "@summary", "@see"
};
public static String[] ADDITIONAL = new String[] {
"group", "sort", "note", "attention",
"bug", "warning", "version", "todo", "deprecated", "since", "status", "change", "permission", "requires",
"precondition", "postcondition", "invariant", "author", "organization", "copyright", "license", "contact", "summary", "see"
};
public EpydocString(@NotNull Substring docstringText) {
super(docstringText, "@");
}
@NotNull
@Override
public String getDescription() {
final String html = inlineMarkupToHTML(myDescription);
assert html != null;
return html;
}
@NotNull
@Override
public List<String> getKeywordArguments() {
return toUniqueStrings(getKeywordArgumentSubstrings());
}
@Override
@Nullable
public String getReturnType() {
return removeInlineMarkup(getReturnTypeSubstring());
}
@Override
public String getReturnDescription() {
return inlineMarkupToHTML(getTagValue(RETURN_TAGS));
}
@Override
@Nullable
public String getParamType(@Nullable String paramName) {
return removeInlineMarkup(getParamTypeSubstring(paramName));
}
@Override
@Nullable
public String getParamDescription(@Nullable String paramName) {
if (paramName == null) {
return null;
}
Substring value = getTagValue(PARAM_TAGS, paramName);
if (value == null) {
value = getTagValue(PARAM_TAGS, "*" + paramName);
}
if (value == null) {
value = getTagValue(PARAM_TAGS, "**" + paramName);
}
return inlineMarkupToHTML(value);
}
@Nullable
@Override
public String getKeywordArgumentDescription(@Nullable String paramName) {
if (paramName == null) {
return null;
}
return inlineMarkupToHTML(getTagValue(KEYWORD_ARGUMENT_TAGS, paramName));
}
@NotNull
@Override
public List<String> getRaisedExceptions() {
return toUniqueStrings(getTagArguments(RAISES_TAGS));
}
@Override
public String getRaisedExceptionDescription(@Nullable String exceptionName) {
if (exceptionName == null) {
return null;
}
return removeInlineMarkup(getTagValue(RAISES_TAGS, exceptionName));
}
@Override
public String getAttributeDescription() {
final Substring value = getTagValue(VARIABLE_TAGS);
return convertInlineMarkup(value != null ? value.toString() : null, true);
}
@Nullable
public static String removeInlineMarkup(@Nullable String s) {
return convertInlineMarkup(s, false);
}
@Nullable
private static String removeInlineMarkup(@Nullable Substring s) {
return convertInlineMarkup(s != null ? s.concatTrimmedLines(" ") : null, false);
}
@Nullable
private static String convertInlineMarkup(@Nullable String s, boolean toHTML) {
if (s == null) return null;
MarkupConverter converter = toHTML ? new HTMLConverter() : new MarkupConverter();
converter.appendWithMarkup(s);
return converter.result();
}
private static class MarkupConverter {
protected final StringBuilder myResult = new StringBuilder();
public void appendWithMarkup(String s) {
int pos = 0;
while(true) {
int bracePos = s.indexOf('{', pos);
if (bracePos < 1) break;
char prevChar = s.charAt(bracePos-1);
if (prevChar >= 'A' && prevChar <= 'Z') {
appendText(s.substring(pos, bracePos - 1));
int rbracePos = findMatchingEndBrace(s, bracePos);
if (rbracePos < 0) {
pos = bracePos + 1;
break;
}
final String inlineMarkupContent = s.substring(bracePos + 1, rbracePos);
appendMarkup(prevChar, inlineMarkupContent);
pos = rbracePos + 1;
}
else {
appendText(s.substring(pos, bracePos + 1));
pos = bracePos+1;
}
}
appendText(s.substring(pos));
}
protected void appendText(String text) {
myResult.append(text);
}
protected void appendMarkup(char markupChar, @NotNull String markupContent) {
appendWithMarkup(markupContent);
}
public String result() {
return myResult.toString();
}
}
private static class HTMLConverter extends MarkupConverter {
@Override
protected void appendText(String text) {
myResult.append(joinLines(XmlStringUtil.escapeString(text, false), true));
}
@Override
protected void appendMarkup(char markupChar, @NotNull String markupContent) {
if (markupChar == 'U') {
appendLink(markupContent);
return;
}
switch (markupChar) {
case 'I' -> appendTagPair(markupContent, "i");
case 'B' -> appendTagPair(markupContent, "b");
case 'C' -> appendTagPair(markupContent, "code");
default -> myResult.append(StringUtil.escapeXmlEntities(markupContent));
}
}
private void appendTagPair(String markupContent, final String tagName) {
myResult.append("<").append(tagName).append(">");
appendWithMarkup(markupContent);
myResult.append("</").append(tagName).append(">");
}
private void appendLink(@NotNull String markupContent) {
String linkText = StringUtil.escapeXmlEntities(markupContent);
String linkUrl = linkText;
int pos = markupContent.indexOf('<');
if (pos >= 0 && markupContent.endsWith(">")) {
linkText = StringUtil.escapeXmlEntities(markupContent.substring(0, pos).trim());
linkUrl = joinLines(StringUtil.escapeXmlEntities(markupContent.substring(pos + 1, markupContent.length() - 1)), false);
}
myResult.append("<a href=\"");
if (!linkUrl.matches("[a-z]+:.+")) {
myResult.append("http://");
}
myResult.append(linkUrl).append("\">").append(linkText).append("</a>");
}
}
private static int findMatchingEndBrace(String s, int bracePos) {
int braceCount = 1;
for(int pos=bracePos+1; pos < s.length(); pos++) {
char c = s.charAt(pos);
if (c == '{') braceCount++;
else if (c == '}') {
braceCount--;
if (braceCount == 0) return pos;
}
}
return -1;
}
private static String joinLines(String s, boolean addSpace) {
while(true) {
int lineBreakStart = s.indexOf('\n');
if (lineBreakStart < 0) break;
int lineBreakEnd = lineBreakStart+1;
int blankLines = 0;
while(lineBreakEnd < s.length() && (s.charAt(lineBreakEnd) == ' ' || s.charAt(lineBreakEnd) == '\n')) {
if (s.charAt(lineBreakEnd) == '\n') blankLines++;
lineBreakEnd++;
}
if (addSpace) {
String separator = blankLines > 0 ? "<p>" : " ";
s = s.substring(0, lineBreakStart) + separator + s.substring(lineBreakEnd);
}
else {
s = s.substring(0, lineBreakStart) + s.substring(lineBreakEnd);
}
}
return s;
}
@Nullable
public static String inlineMarkupToHTML(@Nullable String s) {
return convertInlineMarkup(s, true);
}
@Nullable
private static String inlineMarkupToHTML(@Nullable Substring s) {
return s != null ? inlineMarkupToHTML(s.concatTrimmedLines(" ")) : null;
}
@Override
public List<String> getAdditionalTags() {
List<String> list = new ArrayList<>();
for (String tagName : ADDITIONAL) {
final Map<Substring, Substring> map = myArgTagValues.get(tagName);
if (map != null) {
list.add(tagName);
}
}
return list;
}
@NotNull
@Override
public List<Substring> getKeywordArgumentSubstrings() {
return getTagArguments(KEYWORD_ARGUMENT_TAGS);
}
@Override
public Substring getReturnTypeSubstring() {
return getTagValue(RTYPE_TAGS);
}
@Override
public Substring getParamTypeSubstring(@Nullable String paramName) {
return paramName == null ? getTagValue("type") : getTagValue("type", paramName);
}
@Nullable
@Override
public String getAttributeDescription(@Nullable String attrName) {
return attrName != null ? inlineMarkupToHTML(getTagValue(VARIABLE_TAGS, attrName)) : null;
}
}

24
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/PyDocstringGenerator.java
View File

@@ -32,7 +32,9 @@ import com.jetbrains.python.ast.impl.PyUtilCore;
import com.jetbrains.python.codeInsight.PyCodeInsightSettings;
import com.jetbrains.python.debugger.PySignature;
import com.jetbrains.python.debugger.PySignatureCacheManager;
import com.jetbrains.python.psi.*;
import com.jetbrains.python.psi.PyAstElementGenerator;
import com.jetbrains.python.psi.PyIndentUtil;
import com.jetbrains.python.psi.StructuredDocString;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -86,7 +88,9 @@ public final class PyDocstringGenerator {
* generate properly formatted docstring.
*/
@NotNull
public static PyDocstringGenerator create(@NotNull DocStringFormat format, @NotNull String indentation, @NotNull PsiElement settingsAnchor) {
public static PyDocstringGenerator create(@NotNull DocStringFormat format,
@NotNull String indentation,
@NotNull PsiElement settingsAnchor) {
return new PyDocstringGenerator(null, null, format, indentation, settingsAnchor);
}
@@ -334,8 +338,8 @@ public final class PyDocstringGenerator {
@NotNull
private String createDocString() {
DocStringBuilder builder = null;
if (myDocStringFormat == DocStringFormat.EPYTEXT || myDocStringFormat == DocStringFormat.REST) {
builder = new TagBasedDocStringBuilder(myDocStringFormat == DocStringFormat.EPYTEXT ? "@" : ":");
if (myDocStringFormat == DocStringFormat.REST) {
builder = new TagBasedDocStringBuilder(SphinxDocString.TAG_PREFIX);
TagBasedDocStringBuilder tagBuilder = (TagBasedDocStringBuilder)builder;
if (myAddFirstEmptyLine) {
tagBuilder.addEmptyLine();
@@ -400,10 +404,9 @@ public final class PyDocstringGenerator {
@NotNull
private String updateDocString() {
DocStringUpdater updater = null;
if (myDocStringFormat == DocStringFormat.EPYTEXT || myDocStringFormat == DocStringFormat.REST) {
final String prefix = myDocStringFormat == DocStringFormat.EPYTEXT ? "@" : ":";
if (myDocStringFormat == DocStringFormat.REST) {
// noinspection ConstantConditions
updater = new TagBasedDocStringUpdater((TagBasedDocString)getStructuredDocString(), prefix, myDocStringIndent);
updater = new TagBasedDocStringUpdater((TagBasedDocString)getStructuredDocString(), SphinxDocString.TAG_PREFIX, myDocStringIndent);
}
else if (myDocStringFormat == DocStringFormat.GOOGLE) {
//noinspection ConstantConditions
@@ -416,7 +419,7 @@ public final class PyDocstringGenerator {
updater = new NumpyDocStringUpdater((SectionBasedDocString)getStructuredDocString(), myDocStringIndent);
}
// plain docstring - do nothing
else if (myDocStringText != null){
else if (myDocStringText != null) {
return myDocStringText;
}
if (updater != null) {
@@ -503,6 +506,7 @@ public final class PyDocstringGenerator {
private final String myName;
private final String myType;
private final boolean myReturnValue;
private DocstringParam(@NotNull String name, @Nullable String type, boolean isReturn) {
myName = name;
myType = type;
@@ -551,13 +555,14 @@ public final class PyDocstringGenerator {
", myReturnValue=" + myReturnValue +
'}';
}
}
private static class RaiseVisitor extends PyAstRecursiveElementVisitor {
private boolean myHasRaise = false;
private boolean myHasReturn = false;
@Nullable private PyAstExpression myRaiseTarget = null;
@Override
public void visitPyRaiseStatement(@NotNull PyAstRaiseStatement node) {
myHasRaise = true;
@@ -586,7 +591,6 @@ public final class PyDocstringGenerator {
}
return "";
}
}
@Nullable

129
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/SphinxDocString.java
View File

@@ -1,129 +0,0 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.documentation.docstrings;
import com.jetbrains.python.toolbox.Substring;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Collections;
import java.util.List;
public class SphinxDocString extends TagBasedDocString {
public static String[] KEYWORD_ARGUMENT_TAGS = new String[] { "keyword", "key" };
public static String[] ALL_TAGS = new String[] { ":param", ":parameter", ":arg", ":argument", ":keyword", ":key",
":type", ":raise", ":raises", ":var", ":cvar", ":ivar",
":return", ":returns", ":rtype", ":except", ":exception" };
public SphinxDocString(@NotNull final Substring docstringText) {
super(docstringText, ":");
}
@Nullable
protected static String concatTrimmedLines(@Nullable Substring s) {
return s != null ? s.concatTrimmedLines(" ") : null;
}
@NotNull
@Override
public List<String> getKeywordArguments() {
return toUniqueStrings(getKeywordArgumentSubstrings());
}
@Nullable
@Override
public String getKeywordArgumentDescription(@Nullable String paramName) {
if (paramName == null) {
return null;
}
return concatTrimmedLines(getTagValue(KEYWORD_ARGUMENT_TAGS, paramName));
}
@Override
public String getReturnType() {
return concatTrimmedLines(getReturnTypeSubstring());
}
@Override
public String getParamType(@Nullable String paramName) {
return concatTrimmedLines(getParamTypeSubstring(paramName));
}
@Nullable
@Override
public String getParamDescription(@Nullable String paramName) {
return paramName != null ? concatTrimmedLines(getTagValue(PARAM_TAGS, paramName)) : null;
}
@Override
public String getReturnDescription() {
return concatTrimmedLines(getTagValue(RETURN_TAGS));
}
@NotNull
@Override
public List<String> getRaisedExceptions() {
return toUniqueStrings(getTagArguments(RAISES_TAGS));
}
@Nullable
@Override
public String getRaisedExceptionDescription(@Nullable String exceptionName) {
if (exceptionName == null) {
return null;
}
return concatTrimmedLines(getTagValue(RAISES_TAGS, exceptionName));
}
@Override
public String getAttributeDescription() {
return concatTrimmedLines(getTagValue(VARIABLE_TAGS));
}
@Override
public List<String> getAdditionalTags() {
return Collections.emptyList();
}
@NotNull
@Override
public List<Substring> getKeywordArgumentSubstrings() {
return getTagArguments(KEYWORD_ARGUMENT_TAGS);
}
@Override
public Substring getReturnTypeSubstring() {
return getTagValue("rtype");
}
@Override
public Substring getParamTypeSubstring(@Nullable String paramName) {
return paramName == null ? getTagValue("type") : getTagValue("type", paramName);
}
@NotNull
@Override
public String getDescription() {
return myDescription.replaceAll("\n", "<br/>");
}
@Nullable
@Override
public String getAttributeDescription(@Nullable String attrName) {
return attrName != null ? concatTrimmedLines(getTagValue(VARIABLE_TAGS, attrName)) : null;
}
}

95
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/SphinxDocString.kt Normal file
View File

@@ -0,0 +1,95 @@
/*
* Copyright 2000-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.jetbrains.python.documentation.docstrings
import com.jetbrains.python.toolbox.Substring
class SphinxDocString(docstringText: Substring) : TagBasedDocString(docstringText, TAG_PREFIX) {
override fun getKeywordArguments(): MutableList<String?> {
return toUniqueStrings(keywordArgumentSubstrings)
}
override fun getKeywordArgumentDescription(paramName: String?): String? {
if (paramName == null) {
return null
}
return concatTrimmedLines(getTagValue(KEYWORD_ARGUMENT_TAGS, paramName))
}
override fun getReturnType(): String? {
return concatTrimmedLines(returnTypeSubstring)
}
override fun getParamType(paramName: String?): String? {
return concatTrimmedLines(getParamTypeSubstring(paramName))
}
override fun getParamDescription(paramName: String?): String? {
return if (paramName != null) concatTrimmedLines(getTagValue(PARAM_TAGS, paramName)) else null
}
override fun getReturnDescription(): String? {
return concatTrimmedLines(getTagValue(*RETURN_TAGS))
}
override fun getRaisedExceptions(): MutableList<String?> {
return toUniqueStrings(getTagArguments(*RAISES_TAGS))
}
override fun getRaisedExceptionDescription(exceptionName: String?): String? {
if (exceptionName == null) {
return null
}
return concatTrimmedLines(getTagValue(RAISES_TAGS, exceptionName))
}
override fun getAttributeDescription(): String? {
return concatTrimmedLines(getTagValue(*VARIABLE_TAGS))
}
override fun getKeywordArgumentSubstrings(): MutableList<Substring?> {
return getTagArguments(*KEYWORD_ARGUMENT_TAGS)
}
override fun getReturnTypeSubstring(): Substring? {
return getTagValue("rtype")
}
override fun getParamTypeSubstring(paramName: String?): Substring? {
return if (paramName == null) getTagValue("type") else getTagValue("type", paramName)
}
override fun getDescription(): String {
return myDescription.replace("\n".toRegex(), "<br/>")
}
override fun getAttributeDescription(attrName: String?): String? {
return if (attrName != null) concatTrimmedLines(getTagValue(VARIABLE_TAGS, attrName)) else null
}
companion object {
val KEYWORD_ARGUMENT_TAGS: Array<String> = arrayOf<String>("keyword", "key")
@JvmField
val ALL_TAGS: Array<String> = arrayOf<String>(":param", ":parameter", ":arg", ":argument", ":keyword", ":key",
":type", ":raise", ":raises", ":var", ":cvar", ":ivar",
":return", ":returns", ":rtype", ":except", ":exception")
const val TAG_PREFIX: String = ":"
private fun concatTrimmedLines(s: Substring?): String? {
return s?.concatTrimmedLines(" ")
}
}
}

14
python/python-syntax-core/src/com/jetbrains/python/documentation/docstrings/TagBasedDocString.java
View File

@@ -40,16 +40,16 @@ public abstract class TagBasedDocString extends DocStringLineParser implements S
private static final Pattern RE_LOOSE_TAG_LINE = Pattern.compile("([a-z]+)\\s+([a-zA-Z_0-9]*)\\s*:?\\s*?([^:]*)");
private static final Pattern RE_ARG_TYPE = Pattern.compile("(.*?)\\s+([a-zA-Z_0-9]+)");
public static String[] PARAM_TAGS = new String[]{"param", "parameter", "arg", "argument"};
public static String[] PARAM_TYPE_TAGS = new String[]{"type"};
public static String[] VARIABLE_TAGS = new String[]{"ivar", "cvar", "var"};
public static final String[] PARAM_TAGS = new String[]{"param", "parameter", "arg", "argument"};
public static final String[] PARAM_TYPE_TAGS = new String[]{"type"};
public static final String[] VARIABLE_TAGS = new String[]{"ivar", "cvar", "var"};
public static String[] RAISES_TAGS = new String[]{"raises", "raise", "except", "exception"};
public static String[] RETURN_TAGS = new String[]{"return", "returns"};
public static final String[] RAISES_TAGS = new String[]{"raises", "raise", "except", "exception"};
public static final String[] RETURN_TAGS = new String[]{"return", "returns"};
@NotNull
private final String myTagPrefix;
public static String TYPE = "type";
static String TYPE = "type";
protected TagBasedDocString(@NotNull Substring docStringText, @NotNull String tagPrefix) {
super(docStringText);
@@ -69,8 +69,6 @@ public abstract class TagBasedDocString extends DocStringLineParser implements S
myDescription = builder.toString();
}
public abstract List<String> getAdditionalTags();
@NotNull
@Override
public String getDescription() {

2
python/src/com/jetbrains/python/PySearchableOptionContributor.java
View File

@@ -49,8 +49,6 @@ final class PySearchableOptionContributor extends SearchableOptionContributor {
configurableId, displayName, false);
processor.addOptions("reStructuredText", displayName, "Docstring format",
configurableId, displayName, false);
processor.addOptions("Epytext", "Docstring format", "Docstring format",
configurableId, displayName, false);
processor.addOptions("Plain", displayName, "Docstring format",
configurableId, displayName, false);
processor.addOptions("Unittests", displayName, "Default test runner",

26
python/src/com/jetbrains/python/documentation/PyRuntimeDocstringFormatter.kt
View File

@@ -1,6 +1,7 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.jetbrains.python.documentation
import com.google.gson.Gson
import com.intellij.openapi.diagnostic.Logger
import com.intellij.openapi.module.Module
import com.intellij.openapi.util.text.HtmlChunk
@@ -10,12 +11,17 @@ import com.jetbrains.python.PyPsiBundle
import com.jetbrains.python.PythonHelper
import com.jetbrains.python.documentation.docstrings.DocStringFormat
import com.jetbrains.python.sdk.PySdkUtil
import com.jetbrains.python.sdk.PySdkUtil.getLanguageLevelForSdk
import com.jetbrains.python.sdk.PythonSdkType
import org.jetbrains.annotations.Nls
import java.io.File
object PyRuntimeDocstringFormatter {
fun runExternalTool(module: Module, format: DocStringFormat, input: String, formatterFlags: List<String>): String? {
val sdk = PythonSdkType.findLocalCPython(module) ?: return logErrorAndReturnMessage(format)
val sdk = PythonSdkType.findLocalCPython(module) ?: return logSdkNotFound(format)
if (getLanguageLevelForSdk(sdk).isPython2) {
return logPy2NotSupported()
}
val sdkHome = sdk.homePath ?: return null
val encodedInput = DEFAULT_CHARSET.encode(input)
@@ -39,11 +45,21 @@ object PyRuntimeDocstringFormatter {
else logScriptError(input)
}
private fun logErrorAndReturnMessage(format: DocStringFormat): String {
private fun logErrorToJsonBody(@Nls message: String): String {
return Gson().toJson(
PyDocumentationBuilder.DocstringFormatterRequest(
HtmlChunk.p().attr("color", ColorUtil.toHtmlColor(JBColor.RED)).addRaw(message).toString()))
}
private fun logPy2NotSupported(): String {
val message = PyPsiBundle.message("QDOC.docstring.rendering.is.not.supported.for.python.2")
LOG.warn(message)
return logErrorToJsonBody(message)
}
private fun logSdkNotFound(format: DocStringFormat): String {
LOG.warn("Python SDK for input formatter $format is not found")
val missingInterpreterMessage = PyPsiBundle.message("QDOC.local.sdk.not.found")
return HtmlChunk.p().attr("color", ColorUtil.toHtmlColor(JBColor.RED))
.addRaw(missingInterpreterMessage).toString()
return logErrorToJsonBody(PyPsiBundle.message("QDOC.local.sdk.not.found"))
}
private fun logScriptError(input: String): String? {

18
python/src/com/jetbrains/python/sdk/PythonSdkType.java
View File

@@ -243,6 +243,7 @@ public final class PythonSdkType extends SdkType {
return name;
}
}
@RequiresBackgroundThread(generateAssertion = false) //because of process output
public static @Nullable String suggestBaseSdkName(@NotNull String sdkHome) {
final PythonSdkFlavor flavor = PythonSdkFlavor.getFlavor(sdkHome);
@@ -558,23 +559,6 @@ public final class PythonSdkType extends SdkType {
return PySdkUtil.getLanguageLevelForSdk(sdk);
}
public static @Nullable Sdk findPython2Sdk(@Nullable Module module) {
final Sdk moduleSDK = PythonSdkUtil.findPythonSdk(module);
if (moduleSDK != null && getLanguageLevelForSdk(moduleSDK).isPython2()) {
return moduleSDK;
}
return findPython2Sdk(PythonSdkUtil.getAllSdks());
}
public static @Nullable Sdk findPython2Sdk(@NotNull List<? extends Sdk> sdks) {
for (Sdk sdk : ContainerUtil.sorted(sdks, PreferredSdkComparator.INSTANCE)) {
if (getLanguageLevelForSdk(sdk).isPython2()) {
return sdk;
}
}
return null;
}
@Override
public boolean allowWslSdkForLocalProject() {
return true;