Docs and the base class for Python token set contributors

python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributor.java

@@ -2,19 +2,61 @@ package com.jetbrains.python;
 
 import com.intellij.openapi.extensions.ExtensionPointName;
 import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
 
 /**
+ * Contributes element types of various kinds specific for a particular Python dialect.
+ *
  * @author vlan
  */
 public interface PythonDialectsTokenSetContributor {
   ExtensionPointName<PythonDialectsTokenSetContributor> EP_NAME = ExtensionPointName.create("Pythonid.dialectsTokenSetContributor");
 
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyStatement}.
+   */
+  @NotNull
   TokenSet getStatementTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyExpression}.
+   */
+  @NotNull
   TokenSet getExpressionTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.NameDefiner}.
+   */
+  @NotNull
   TokenSet getNameDefinerTokens();
+
+  /**
+   * Returns element types that are language keywords.
+   */
+  @NotNull
   TokenSet getKeywordTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyParameter}.
+   */
+  @NotNull
   TokenSet getParameterTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyFunction}.
+   */
+  @NotNull
   TokenSet getFunctionDeclarationTokens();
+
+  /**
+   * Returns element types that can be used as unbalanced braces recovery tokens in the lexer.
+   */
+  @NotNull
   TokenSet getUnbalancedBracesRecoveryTokens();
+
+  /**
+   * Returns element types that are subclasses of {@link com.jetbrains.python.psi.PyReferenceExpression}.
+   */
+  @NotNull
   TokenSet getReferenceExpressionTokens();
 }

python/psi-api/src/com/jetbrains/python/PythonDialectsTokenSetContributorBase.java

@@ -0,0 +1,57 @@
+package com.jetbrains.python;
+
+import com.intellij.psi.tree.TokenSet;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * @author vlan
+ */
+public abstract class PythonDialectsTokenSetContributorBase implements PythonDialectsTokenSetContributor {
+  @NotNull
+  @Override
+  public TokenSet getStatementTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getExpressionTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getNameDefinerTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getKeywordTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getParameterTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getFunctionDeclarationTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getUnbalancedBracesRecoveryTokens() {
+    return TokenSet.EMPTY;
+  }
+
+  @NotNull
+  @Override
+  public TokenSet getReferenceExpressionTokens() {
+    return TokenSet.EMPTY;
+  }
+}