RUBY-19104: respect tags when processing scalar values

plugins/yaml/src/org/jetbrains/yaml/YAMLElementTypes.java

@@ -1,5 +1,6 @@
 package org.jetbrains.yaml;
 
+import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IFileElementType;
 import com.intellij.psi.tree.TokenSet;
 
@@ -32,4 +33,11 @@ public interface YAMLElementTypes {
     YAMLTokenTypes.TEXT,
     SCALAR_LIST_VALUE
   );
+
+  TokenSet BLANK_ELEMENTS = TokenSet.create(
+    YAMLTokenTypes.WHITESPACE,
+    TokenType.WHITE_SPACE,
+    YAMLTokenTypes.EOL,
+    YAMLTokenTypes.INDENT,
+    YAMLTokenTypes.COMMENT);
 }

plugins/yaml/src/org/jetbrains/yaml/psi/impl/YAMLBlockScalarImpl.java

@@ -3,6 +3,7 @@ package org.jetbrains.yaml.psi.impl;
 import com.intellij.lang.ASTNode;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.TextRange;
+import com.intellij.psi.impl.source.tree.TreeUtil;
 import com.intellij.psi.tree.IElementType;
 import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
@@ -16,7 +17,7 @@ import java.util.List;
 
 public abstract class YAMLBlockScalarImpl extends YAMLScalarImpl {
   protected static final int DEFAULT_CONTENT_INDENT = 2;
-  
+
   public YAMLBlockScalarImpl(@NotNull ASTNode node) {
     super(node);
   }
@@ -32,13 +33,17 @@ public abstract class YAMLBlockScalarImpl extends YAMLScalarImpl {
   @NotNull
   @Override
   public List<TextRange> getContentRanges() {
-    final int myStart = getTextOffset();
-    final ASTNode node = getNode();
+    final ASTNode firstContentChild = getFirstContentNode();
+    if (firstContentChild == null) {
+      return Collections.emptyList();
+    }
+
+    final int myStart = getTextRange().getStartOffset();
     final List<TextRange> result = new ArrayList<>();
 
     final int indent = locateIndent();
 
-    final ASTNode firstEol = node.findChildByType(YAMLTokenTypes.EOL);
+    final ASTNode firstEol = TreeUtil.findSibling(firstContentChild, YAMLTokenTypes.EOL);
     if (firstEol == null) {
       return Collections.emptyList();
     }
@@ -47,7 +52,7 @@ public abstract class YAMLBlockScalarImpl extends YAMLScalarImpl {
     for (ASTNode child = firstEol.getTreeNext(); child != null; child = child.getTreeNext()) {
       final IElementType childType = child.getElementType();
       final TextRange childRange = child.getTextRange();
-      
+
       if (childType == YAMLTokenTypes.INDENT && isEol(child.getTreePrev())) {
         thisLineStart = child.getStartOffset() + Math.min(indent, child.getTextLength());
       }
@@ -89,7 +94,7 @@ public abstract class YAMLBlockScalarImpl extends YAMLScalarImpl {
     }
     return 0;
   }
-  
+
   private static boolean isEol(@Nullable ASTNode node) {
     return node != null && node.getElementType() == YAMLTokenTypes.EOL;
   }

plugins/yaml/src/org/jetbrains/yaml/psi/impl/YAMLPlainTextImpl.java

@@ -21,12 +21,11 @@ public class YAMLPlainTextImpl extends YAMLScalarImpl implements YAMLScalar {
   @NotNull
   @Override
   public List<TextRange> getContentRanges() {
-    final int myStart = getTextOffset();
-    final ASTNode node = getNode();
+    final int myStart = getTextRange().getStartOffset();
     final List<TextRange> result = new ArrayList<>();
 
     boolean seenText = false;
-    for (ASTNode child = node.getFirstChildNode(); child != null; child = child.getTreeNext()) {
+    for (ASTNode child = getFirstContentNode(); child != null; child = child.getTreeNext()) {
       if (child.getElementType() == YAMLTokenTypes.TEXT) {
         seenText = true;
         result.add(child.getTextRange().shiftRight(-myStart));
@@ -52,7 +51,7 @@ public class YAMLPlainTextImpl extends YAMLScalarImpl implements YAMLScalar {
       return " ";
     }
   }
-  
+
   private static boolean isNewline(@NotNull CharSequence text, @NotNull TextRange range) {
     return range.getLength() == 1 && text.charAt(range.getStartOffset()) == '\n';
   }
@@ -79,10 +78,10 @@ public class YAMLPlainTextImpl extends YAMLScalarImpl implements YAMLScalar {
         currentLength = 0;
         continue;
       }
-      
+
       currentLength++;
     }
-    
+
     return result;
   }
 

plugins/yaml/src/org/jetbrains/yaml/psi/impl/YAMLQuotedTextImpl.java

@@ -14,6 +14,7 @@ import org.jetbrains.yaml.lexer.YAMLGrammarCharUtil;
 import org.jetbrains.yaml.psi.YAMLQuotedText;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
@@ -22,17 +23,25 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
 
   public YAMLQuotedTextImpl(@NotNull ASTNode node) {
     super(node);
-    myIsSingleQuoted = getNode().getFirstChildNode().getElementType() == YAMLTokenTypes.SCALAR_STRING;
+    final ASTNode firstContentNode = getFirstContentNode();
+    myIsSingleQuoted = firstContentNode != null && firstContentNode.getElementType() == YAMLTokenTypes.SCALAR_STRING;
   }
 
   @NotNull
   @Override
   public List<TextRange> getContentRanges() {
-    List<TextRange> result = new ArrayList<>();
+    final ASTNode firstContentNode = getFirstContentNode();
+    if (firstContentNode == null) {
+      return Collections.emptyList();
+    }
 
-    final List<String> lines = StringUtil.split(getText(), "\n", true, false);
+    List<TextRange> result = new ArrayList<>();
+    TextRange contentRange = TextRange.create(firstContentNode.getStartOffset(), getTextRange().getEndOffset())
+      .shiftRight(-getTextRange().getStartOffset());
+
+    final List<String> lines = StringUtil.split(contentRange.substring(getText()), "\n", true, false);
     // First line has opening quote
-    int cumulativeOffset = 0;
+    int cumulativeOffset = contentRange.getStartOffset();
     for (int i = 0; i < lines.size(); ++i) {
       final String line = lines.get(i);
 
@@ -82,9 +91,9 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
   @Override
   protected List<Pair<TextRange, String>> getDecodeReplacements(@NotNull CharSequence input) {
     List<Pair<TextRange, String>> result = new ArrayList<>();
-    
+
     for (int i = 0; i + 1 < input.length(); ++i) {
-      
+
       if (isSingleQuote() && input.charAt(i) == '\'' && input.charAt(i + 1) == '\'') {
         result.add(Pair.create(TextRange.from(i, 2), "'"));
         i++;
@@ -115,10 +124,10 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
         }
       }
     }
-    
+
     final int indent = YAMLUtil.getIndentToThisElement(this);
     final String indentString = StringUtil.repeatSymbol(' ', indent);
-    
+
     final List<Pair<TextRange, String>> result = new ArrayList<>();
     int currentLength = 0;
     for (int i = 0; i < input.length(); ++i) {
@@ -138,7 +147,7 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
       }
 
 
-      if (currentLength > MAX_SCALAR_LENGTH_PREDEFINED 
+      if (currentLength > MAX_SCALAR_LENGTH_PREDEFINED
           && (!isSingleQuote() || (c == ' ' && isSurroundedByNoSpace(input, i)))) {
         final String replacement;
         if (isSingleQuote()) {
@@ -155,12 +164,12 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
       }
 
       currentLength++;
-      
+
       if (isSingleQuote() && c == '\'') {
         result.add(Pair.create(TextRange.from(i, 1), "''"));
         continue;
       }
-      
+
       if (!isSingleQuote()) {
         if (c == '"') {
           result.add(Pair.create(TextRange.from(i, 1), "\\\""));
@@ -186,7 +195,7 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
   public String toString() {
     return "YAML quoted text";
   }
-  
+
   private static class Escaper {
     private static final int[][] ONE_LETTER_CONVERSIONS = new int[][] {
       {'0', 0},
@@ -208,7 +217,7 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
       {'L', 8232},
       {'P', 8233},
     };
-    
+
     private static final NotNullLazyValue<Map<Integer, Integer>> ESC_TO_CODE = new NotNullLazyValue<Map<Integer, Integer>>() {
       @NotNull
       @Override
@@ -232,12 +241,12 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
         return map;
       }
     };
-    
+
     static int findEscapementLength(@NotNull CharSequence text, int pos) {
       if (pos + 1 >= text.length() || text.charAt(pos) != '\\') {
         throw new IllegalArgumentException("This is not an escapement start");
       }
-      
+
       final char c = text.charAt(pos + 1);
       if (c == 'x') {
         return 3;
@@ -252,7 +261,7 @@ public class YAMLQuotedTextImpl extends YAMLScalarImpl implements YAMLQuotedText
         return 1;
       }
     }
-    
+
     static int toUnicodeChar(@NotNull CharSequence text, int pos, int length) {
       if (length > 1) {
         CharSequence s = text.subSequence(pos + 2, Math.min(text.length(), pos + length + 1));

plugins/yaml/src/org/jetbrains/yaml/psi/impl/YAMLScalarImpl.java

@@ -10,6 +10,9 @@ import com.intellij.psi.PsiReference;
 import com.intellij.psi.impl.source.resolve.reference.ReferenceProvidersRegistry;
 import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.jetbrains.yaml.YAMLElementTypes;
+import org.jetbrains.yaml.YAMLTokenTypes;
 import org.jetbrains.yaml.lexer.YAMLGrammarCharUtil;
 import org.jetbrains.yaml.psi.YAMLScalar;
 
@@ -18,7 +21,7 @@ import java.util.List;
 
 public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar {
   protected static final int MAX_SCALAR_LENGTH_PREDEFINED = 60;
-  
+
   public YAMLScalarImpl(@NotNull ASTNode node) {
     super(node);
   }
@@ -28,11 +31,11 @@ public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar
 
   @NotNull
   protected abstract String getRangesJoiner(@NotNull CharSequence text, @NotNull List<TextRange> contentRanges, int indexBefore);
-  
+
   protected List<Pair<TextRange, String>> getDecodeReplacements(@NotNull CharSequence input) {
     return Collections.emptyList();
   }
-  
+
   protected List<Pair<TextRange, String>> getEncodeReplacements(@NotNull CharSequence input) throws IllegalArgumentException {
     throw new IllegalArgumentException("Not implemented");
   }
@@ -47,7 +50,7 @@ public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar
 
     for (int i = 0; i < contentRanges.size(); i++) {
       final TextRange range = contentRanges.get(i);
-      
+
       final CharSequence curString = range.subSequence(text);
       builder.append(curString);
 
@@ -85,10 +88,10 @@ public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar
   public LiteralTextEscaper<? extends PsiLanguageInjectionHost> createLiteralTextEscaper() {
     return new MyLiteralTextEscaper(this);
   }
-  
-  @NotNull 
-  static String processReplacements(@NotNull CharSequence input, 
-                                            @NotNull List<Pair<TextRange, String>> replacements) throws IndexOutOfBoundsException {
+
+  @NotNull
+  static String processReplacements(@NotNull CharSequence input,
+                                    @NotNull List<Pair<TextRange, String>> replacements) throws IndexOutOfBoundsException {
     StringBuilder result = new StringBuilder();
     int currentOffset = 0;
     for (Pair<TextRange, String> replacement : replacements) {
@@ -105,6 +108,16 @@ public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar
            && (pos + 1 >= text.length() || !YAMLGrammarCharUtil.isSpaceLike(text.charAt(pos + 1)));
   }
 
+  @Nullable
+  protected final ASTNode getFirstContentNode() {
+    ASTNode node = getNode().getFirstChildNode();
+    while (node != null && (
+      node.getElementType() == YAMLTokenTypes.TAG || YAMLElementTypes.BLANK_ELEMENTS.contains(node.getElementType()))) {
+      node = node.getTreeNext();
+    }
+    return node;
+  }
+
   private static class MyLiteralTextEscaper extends LiteralTextEscaper<YAMLScalarImpl> {
     public MyLiteralTextEscaper(YAMLScalarImpl scalar) {
       super(scalar);
@@ -120,7 +133,7 @@ public abstract class YAMLScalarImpl extends YAMLValueImpl implements YAMLScalar
     public int getOffsetInHost(int offsetInDecoded, @NotNull TextRange rangeInsideHost) {
       final String text = myHost.getText();
       final List<TextRange> contentRanges = myHost.getContentRanges();
-      
+
       int currentOffsetInDecoded = 0;
 
       for (int i = 0; i < contentRanges.size(); i++) {

plugins/yaml/testSrc/org/jetbrains/yaml/psi/YAMLScalarContentTest.java

@@ -30,6 +30,10 @@ public class YAMLScalarContentTest extends LightPlatformCodeInsightFixtureTestCa
     doTest();
   }
 
+  public void testPlainScalar3Tag() {
+    doTest();
+  }
+
   public void testLiteralStyle1() {
     doTest();
   }
@@ -57,7 +61,11 @@ public class YAMLScalarContentTest extends LightPlatformCodeInsightFixtureTestCa
   public void testFoldedStyle4() {
     doTest();
   }
-  
+
+  public void testFoldedStyle4Tag() {
+    doTest();
+  }
+
   public void testFoldedStyle5() {
     doTest();
   }
@@ -70,6 +78,10 @@ public class YAMLScalarContentTest extends LightPlatformCodeInsightFixtureTestCa
     doTest();
   }
 
+  public void testSingleQuote1Tag() {
+    doTest();
+  }
+
   public void testSingleQuote2() {
     doTest();
   }
@@ -77,15 +89,19 @@ public class YAMLScalarContentTest extends LightPlatformCodeInsightFixtureTestCa
   public void testDoubleQuote1() {
     doTest();
   }
-  
+
   public void testDoubleQuote2() {
     doTest();
   }
-  
+
   public void testDoubleQuote4() {
     doTest();
   }
 
+  public void testDoubleQuoteTag() {
+    doTest();
+  }
+
   private void doTest() {
     myFixture.configureByFile(getTestName(true) + ".yml");
 

plugins/yaml/testSrc/org/jetbrains/yaml/psi/YAMLScalarLiteralEscaperTest.java

@@ -50,6 +50,10 @@ public class YAMLScalarLiteralEscaperTest extends LightPlatformCodeInsightFixtur
     doTest();
   }
 
+  public void testPlainScalar3Tag() {
+    doTest();
+  }
+
   public void testLiteralStyle1() {
     doTest();
   }
@@ -78,10 +82,18 @@ public class YAMLScalarLiteralEscaperTest extends LightPlatformCodeInsightFixtur
     doTest();
   }
 
+  public void testFoldedStyle4Tag() {
+    doTest();
+  }
+
   public void testSingleQuote1() {
     doTest();
   }
 
+  public void testSingleQuote1Tag() {
+    doTest();
+  }
+
   public void testSingleQuote2() {
     doTest();
   }
@@ -89,15 +101,19 @@ public class YAMLScalarLiteralEscaperTest extends LightPlatformCodeInsightFixtur
   public void testDoubleQuote1() {
     doTest();
   }
-  
+
   public void testDoubleQuote2() {
     doTest();
   }
-  
+
   public void testDoubleQuote3() {
     doTest();
   }
 
+  public void testDoubleQuoteTag() {
+    doTest();
+  }
+
   private void doTest() {
     myFixture.configureByFile(getTestName(true) + ".yml");
 
@@ -112,12 +128,12 @@ public class YAMLScalarLiteralEscaperTest extends LightPlatformCodeInsightFixtur
     final StringBuilder builder = new StringBuilder();
     assertTrue(elementLiteralEscaper.decode(scalarElement.getTextRange(), builder));
     assertEquals(scalarElement.getTextValue(), builder.toString());
-    
+
     int[] offsets = new int[builder.length() + 1];
     for (int i = 0; i < builder.length() + 1; ++i) {
       offsets[i] = elementLiteralEscaper.getOffsetInHost(i, TextRange.from(0, scalarElement.getTextLength()));
     }
-    
+
     final String elementText = scalarElement.getText();
     StringBuilder description = new StringBuilder();
     for (int i = 0; i < builder.length(); ++i) {
@@ -126,8 +142,8 @@ public class YAMLScalarLiteralEscaperTest extends LightPlatformCodeInsightFixtur
         .append(elementText.subSequence(offsets[i], offsets[i + 1]))
         .append('\n');
     }
-    assertSameLinesWithFile(getTestDataPath() + getTestName(true) + ".positions.txt", 
-                            Arrays.toString(offsets) + "\n" + description, 
+    assertSameLinesWithFile(getTestDataPath() + getTestName(true) + ".positions.txt",
+                            Arrays.toString(offsets) + "\n" + description,
                             false);
   }
 }

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/doubleQuoteTag.positions.txt

@@ -0,0 +1,14 @@
+[6, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+->\n
+
+->\n
+'->'
+'->'
+c->c
+o->o
+n->n
+t->t
+e->e
+n->n
+t->t

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/doubleQuoteTag.txt

@@ -0,0 +1,3 @@
+
+
+''content

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/doubleQuoteTag.yml

@@ -0,0 +1 @@
+!Tag "\n\n''content"

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/foldedStyle4Tag.positions.txt

@@ -0,0 +1,86 @@
+[7, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 22, 24, 25, 26, 27, 28, 30, 31, 32, 33, 34, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 49, 50, 51, 52, 53, 54, 55, 56, 57, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 70, 71, 72, 73, 74, 76, 77, 78, 79, 80, 80]
+
+->
+ 
+f->f
+o->o
+l->l
+d->d
+e->e
+d->d
+ ->
+ 
+l->l
+i->i
+n->n
+e->e
+
+
+->
+ 
+n->n
+e->e
+x->x
+t->t
+ ->
+ 
+l->l
+i->i
+n->n
+e->e
+
+->
+ 
+ -> 
+ -> 
+*->*
+ -> 
+b->b
+u->u
+l->l
+l->l
+e->e
+t->t
+
+->
+
+
+->
+ 
+ -> 
+ -> 
+*->*
+ -> 
+l->l
+i->i
+s->s
+t->t
+
+->
+ 
+ -> 
+ -> 
+*->*
+ -> 
+l->l
+i->i
+n->n
+e->e
+
+->
+
+
+->
+ 
+l->l
+a->a
+s->s
+t->t
+ ->
+ 
+l->l
+i->i
+n->n
+e->e
+
+->

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/foldedStyle4Tag.txt

@@ -0,0 +1,9 @@
+
+folded line
+next line
+  * bullet
+
+  * list
+  * line
+
+last line

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/foldedStyle4Tag.yml

@@ -0,0 +1,16 @@
+!Tag >
+
+ folded
+ line
+
+ next
+ line
+   * bullet
+
+   * list
+   * line
+
+ last
+ line
+
+# Comment

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/plainScalar3Tag.positions.txt

@@ -0,0 +1,30 @@
+[5, 6, 7, 8, 13, 14, 15, 16, 19, 20, 21, 23, 26, 27, 28, 30, 33, 34, 35, 36, 37, 38]
+f->f
+o->o
+o->o
+ ->
+    
+b->b
+a->a
+r->r
+ ->
+  
+f->f
+a->a
+r->r
+
+
+->
+  
+b->b
+a->a
+z->z
+
+
+->
+  
+b->b
+a->a
+a->a
+a->a
+z->z

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/plainScalar3Tag.txt

@@ -0,0 +1,3 @@
+foo bar far
+baz
+baaaz

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/plainScalar3Tag.yml

@@ -0,0 +1,7 @@
+- !Tag foo
+    bar<caret>
+  far
+
+  baz
+
+  baaaz

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/singleQuote1Tag.positions.txt

@@ -0,0 +1,19 @@
+[6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]
+h->h
+e->e
+r->r
+e->e
+'->''
+s->s
+ -> 
+t->t
+o->o
+ -> 
+"->"
+q->q
+u->u
+o->o
+t->t
+e->e
+s->s
+"->"

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/singleQuote1Tag.txt

@@ -0,0 +1 @@
+here's to "quotes"

plugins/yaml/testSrc/org/jetbrains/yaml/psi/data/singleQuote1Tag.yml

@@ -0,0 +1 @@
+!Tag 'here''s to "quotes"'