API for custom generation of synthetic whitespace between tokens; fix all Py tests

platform/lang-api/src/com/intellij/lang/TokenSeparatorGenerator.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.lang;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * @author yole
+ */
+public interface TokenSeparatorGenerator {
+  @Nullable
+  ASTNode generateWhitespaceBetweenTokens(ASTNode left, ASTNode right);
+}

platform/lang-api/src/com/intellij/psi/util/PsiUtilBase.java

@@ -599,4 +599,11 @@ public class PsiUtilBase {
     });
     return result[0];
   }
+
+  public static Language getNotAnyLanguage(ASTNode node) {
+    if (node == null) return Language.ANY;
+
+    final Language lang = node.getElementType().getLanguage();
+    return lang == Language.ANY ? getNotAnyLanguage(node.getTreeParent()) : lang;
+  }
 }

platform/lang-impl/src/com/intellij/lang/LanguageTokenSeparatorGenerators.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2000-2010 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.intellij.lang;
+
+import com.intellij.psi.PsiManager;
+import com.intellij.psi.TokenType;
+import com.intellij.psi.impl.source.tree.Factory;
+import com.intellij.psi.util.PsiUtilBase;
+
+/**
+ * @author yole
+ */
+public class LanguageTokenSeparatorGenerators extends LanguageExtension<TokenSeparatorGenerator> {
+  public static final LanguageTokenSeparatorGenerators INSTANCE = new LanguageTokenSeparatorGenerators();
+
+  private LanguageTokenSeparatorGenerators() {
+    super("com.intellij.lang.tokenSeparatorGenerator", new TokenSeparatorGenerator() {
+      public ASTNode generateWhitespaceBetweenTokens(ASTNode left, ASTNode right) {
+        Language l = PsiUtilBase.getNotAnyLanguage(left);
+        final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(l);
+        if (parserDefinition != null) {
+          PsiManager manager = right.getTreeParent().getPsi().getManager();
+          ASTNode generatedWhitespace;
+          //noinspection EnumSwitchStatementWhichMissesCases
+          switch(parserDefinition.spaceExistanceTypeBetweenTokens(left, right)){
+            case MUST:
+              generatedWhitespace = Factory.createSingleLeafElement(TokenType.WHITE_SPACE, " ", 0, 1, null, manager);
+              break;
+            case MUST_LINE_BREAK:
+              generatedWhitespace = Factory.createSingleLeafElement(TokenType.WHITE_SPACE, "\n", 0, 1, null, manager);
+              break;
+            default:
+              generatedWhitespace = null;
+          }
+          return generatedWhitespace;
+        }
+        return null;
+      }
+    });
+  }
+}

platform/lang-impl/src/com/intellij/psi/impl/source/codeStyle/CodeEditUtil.java

@@ -34,6 +34,7 @@ import com.intellij.psi.codeStyle.CodeStyleSettingsManager;
 import com.intellij.psi.impl.source.tree.*;
 import com.intellij.psi.templateLanguages.OuterLanguageElement;
 import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.util.PsiUtilBase;
 import com.intellij.util.text.CharArrayUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -281,31 +282,13 @@ public class CodeEditUtil {
     right.getTreeParent().replaceChild(right, merged);
   }
 
-  private static Language getNotAnyLanguage(ASTNode node) {
-    if (node == null) return Language.ANY;
-
-    final Language lang = node.getElementType().getLanguage();
-    return lang == Language.ANY ? getNotAnyLanguage(node.getTreeParent()) : lang;
-  }
-
   private static void markToReformatBeforeOrInsertWhitespace(final ASTNode left, @NotNull final ASTNode right, PsiManager manager) {
-    final Language leftLang = left != null ? getNotAnyLanguage(left) : null;
-    final Language rightLang = getNotAnyLanguage(right);
+    final Language leftLang = left != null ? PsiUtilBase.getNotAnyLanguage(left) : null;
+    final Language rightLang = PsiUtilBase.getNotAnyLanguage(right);
     
-    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(rightLang);
-    LeafElement generatedWhitespace = null;
-    if(leftLang == rightLang && parserDefinition != null){
-      //noinspection EnumSwitchStatementWhichMissesCases
-      switch(parserDefinition.spaceExistanceTypeBetweenTokens(left, right)){
-        case MUST:
-          generatedWhitespace = Factory.createSingleLeafElement(TokenType.WHITE_SPACE, " ", 0, 1, null, manager);
-          break;
-        case MUST_LINE_BREAK:
-          generatedWhitespace = Factory.createSingleLeafElement(TokenType.WHITE_SPACE, "\n", 0, 1, null, manager);
-          break;
-        default:
-          generatedWhitespace = null;
-      }
+    ASTNode generatedWhitespace = null;
+    if(leftLang == rightLang) {
+      generatedWhitespace = LanguageTokenSeparatorGenerators.INSTANCE.forLanguage(leftLang).generateWhitespaceBetweenTokens(left, right);
     }
     if(generatedWhitespace != null){
       final TreeUtil.CommonParentState parentState = new TreeUtil.CommonParentState();

platform/platform-resources/src/META-INF/LangExtensionPoints.xml

@@ -113,6 +113,7 @@
   <extensionPoint name="lang.unwrapDescriptor" beanClass="com.intellij.lang.LanguageExtensionPoint"/>
   <extensionPoint name="lang.parserDefinition" beanClass="com.intellij.lang.LanguageExtensionPoint"/>
   <extensionPoint name="lang.treePatcher" beanClass="com.intellij.lang.LanguageExtensionPoint"/>
+  <extensionPoint name="lang.tokenSeparatorGenerator" beanClass="com.intellij.lang.LanguageExtensionPoint"/>
 
   <extensionPoint name="lang.fileViewProviderFactory" beanClass="com.intellij.lang.LanguageExtensionPoint"/>
   <extensionPoint name="fileType.fileViewProviderFactory" beanClass="com.intellij.openapi.fileTypes.FileTypeExtensionPoint"/>