MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[grazie] IJPL-204111 Text-level analysis: make sentence texts consistent with offsets, don't skip inter-sentence whitespace

Browse Source 
Merge-request: IJ-MR-175339
Merged-by: Ilia Permiashkin <ilia.permiashkin@jetbrains.com>

GitOrigin-RevId: 1046d8c736b0cb8a8fbd3a2681d053f7872f3fb1
This commit is contained in:
Ilia Permiashkin
2025-09-11 17:29:52 +00:00
committed by intellij-monorepo-bot
parent 13f41af54e
commit b09d3e2a78
2 changed files with 16 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/grazie/src/main/kotlin/com/intellij/grazie/rule/ParsedSentence.kt
+8 -2
View File
@@ -34,7 +34,13 @@ class ParsedSentence private constructor(
@JvmField val extractedText: TextContent,
/** The dependency tree for the sentence */
@JvmField val tree: Tree
@JvmField val tree: Tree,
/**
* The range of the sentence in [extractedText] as reported by the sentence tokenizer,
* including leading or trailing space
*/
@JvmField val untrimmedRange: TextRange
) {
fun textOffsetToFile(textOffset: Int): Int {
@@ -110,7 +116,7 @@ class ParsedSentence private constructor(
tree = tree.withStartOffset(start)!!
val stubbed = trees[sentence.stubbedSwe()]
if (stubbed != null) tree = tree.withStubbed(StubbedSentence(sentence.swe(), stubbed.withStartOffset(start)))!!
out.add(ParsedSentence(tree.startOffset(), tree.text(), content, tree))
out.add(ParsedSentence(tree.startOffset(), tree.text(), content, tree, TextRange(sentence.start, sentence.end())))
}
}
}
plugins/grazie/src/main/kotlin/com/intellij/grazie/text/TreeRuleChecker.java
+8 -5
View File
@@ -410,17 +410,20 @@ public final class TreeRuleChecker {
for (int i = 0; i < sentences.size(); i++) {
ParsedSentence parsed = sentences.get(i);
String trimmed = parsed.text.trim();
int trimmedStart = parsed.text.indexOf(trimmed);
TextRange untrimmedRange = parsed.untrimmedRange;
String untrimmedText = untrimmedRange.substring(content.toString());
String trimmed = untrimmedText.trim();
int trimmedStart = untrimmedText.indexOf(trimmed);
var suppressions = ContainerUtil.map2Set(suppressedRanges.getOrDefault(trimmed, Set.of()), r -> r.shiftRight(trimmedStart));
DocumentSentence.Analyzed ds = new DocumentSentence(parsed.text, parsed.tree.treeSupport().getGrazieLanguage())
DocumentSentence.Analyzed ds = new DocumentSentence(untrimmedText, parsed.tree.treeSupport().getGrazieLanguage())
.withIntro(i == 0 ? getIntro(content) : List.of())
.withExclusions(SentenceTokenizer.rangeExclusions(content, TextRange.from(parsed.textStartOffset, parsed.text.length())))
.withExclusions(SentenceTokenizer.rangeExclusions(content, untrimmedRange))
.withSuppressions(suppressions)
.withTree(parsed.tree.withStartOffset(offset))
.withMetadata(matches.get(i).metadata);
doc.add(new SentenceWithContent(ds, content, offset, offset + parsed.textStartOffset));
doc.add(new SentenceWithContent(ds, content, offset, offset + untrimmedRange.getStartOffset()));
}
offset += content.length();
}