diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/cloud/APIQueries.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/cloud/APIQueries.kt index 355a7564fe84..692db88e2181 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/cloud/APIQueries.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/cloud/APIQueries.kt @@ -32,10 +32,7 @@ import com.intellij.openapi.util.Disposer import com.intellij.openapi.util.IntellijInternalApi import com.intellij.util.io.computeDetached import io.ktor.client.plugins.HttpRequestTimeoutException -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.DelicateCoroutinesApi -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext +import kotlinx.coroutines.* import org.jetbrains.annotations.TestOnly import java.io.IOException import com.intellij.openapi.util.TextRange as IJTextRange @@ -60,28 +57,38 @@ object APIQueries { @Volatile @JvmStatic var rephraser: Rephraser = object : Rephraser { - override fun rephrase(text: String, range: IJTextRange, language: Language, project: Project): List? = + override fun rephrase(text: String, ranges: List, language: Language, project: Project): List>>? = request(project, null) { - val taskClient = GrazieCloudConnector.api()?.tasksWithStreamData() ?: return@request null - val taskCall = if (text.length < 100) { - val contentPrefix = text.take(range.startOffset) - val contentSuffix = text.drop(range.endOffset) - RewriteSelectionV2TaskDescriptor.createCallData( - RewriteSelectionV2TaskParams( - contentPrefix, contentSuffix, language.englishName, range.substring(text)) - ) + coroutineScope { + ranges.map { + async { rephrase(text, it, language) } + }.awaitAll().filterNotNull() } - else { - RewriteFullTaskDescriptor.createCallData( - RewriteFullTaskParams(text, language.englishName) - ) - } - taskClient.executeV2(taskCall) - .text { it.content } - .split("") - .filter { it.isNotBlank() } - .map { it.trim() } } + + private suspend fun rephrase(text: String, range: IJTextRange, language: Language): Pair>? { + val taskClient = GrazieCloudConnector.api()?.tasksWithStreamData() ?: return null + val taskCall = if (range.startOffset == 0 && range.length == text.length) { + RewriteFullTaskDescriptor.createCallData( + RewriteFullTaskParams(text, language.englishName) + ) + } + else { + val contentPrefix = text.take(range.startOffset) + val contentSuffix = text.drop(range.endOffset) + RewriteSelectionV2TaskDescriptor.createCallData( + RewriteSelectionV2TaskParams( + contentPrefix, contentSuffix, language.englishName, range.substring(text)) + ) + } + return range to taskClient.executeV2(taskCall) + .text { it.content } + .split("") + .filter { it.isNotBlank() } + .map { it.trim() } + .distinct() + .filter { it != text } + } } @JvmStatic @@ -192,7 +199,7 @@ interface Translator { } interface Rephraser { - fun rephrase(text: String, range: IJTextRange, language: Language, project: Project): List? + fun rephrase(text: String, ranges: List, language: Language, project: Project): List>>? } open class TaskServerException: RuntimeException() diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/inspection/ai/RephraseAction.java b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/inspection/ai/RephraseAction.java index ff0be33d30c8..7be1e85cf0ce 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/inspection/ai/RephraseAction.java +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/ide/inspection/ai/RephraseAction.java @@ -2,6 +2,7 @@ package com.intellij.grazie.ide.inspection.ai; import ai.grazie.nlp.langs.Language; import ai.grazie.nlp.tokenizer.word.StandardWordTokenizer; +import ai.grazie.rules.toolkit.LanguageToolkit; import com.intellij.codeInsight.hint.HintManager; import com.intellij.codeInspection.IntentionAndQuickFixAction; import com.intellij.codeInspection.util.IntentionFamilyName; @@ -13,6 +14,8 @@ import com.intellij.grazie.cloud.TaskServerException; import com.intellij.grazie.detection.LangDetector; import com.intellij.grazie.ide.fus.GrazieFUSCounter; import com.intellij.grazie.ide.ui.PaddedListCellRenderer; +import com.intellij.grazie.rule.ParsedSentence; +import com.intellij.grazie.rule.SentenceTokenizer; import com.intellij.grazie.text.TextContent; import com.intellij.grazie.text.TextExtractor; import com.intellij.grazie.utils.HighlightingUtil; @@ -32,17 +35,21 @@ import com.intellij.openapi.ui.popup.JBPopupListener; import com.intellij.openapi.ui.popup.LightweightWindowEvent; import com.intellij.openapi.util.Ref; import com.intellij.openapi.util.TextRange; -import com.intellij.openapi.util.text.StringUtil; +import com.intellij.openapi.util.text.Strings; import com.intellij.openapi.vcs.ui.CommitMessage; import com.intellij.psi.PsiFile; import com.intellij.util.containers.ContainerUtil; +import kotlin.Pair; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import javax.swing.*; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static com.intellij.grazie.utils.UtilsKt.ijRange; + @SuppressWarnings("IntentionDescriptionNotFoundInspection") public class RephraseAction extends IntentionAndQuickFixAction { @@ -64,9 +71,15 @@ public class RephraseAction extends IntentionAndQuickFixAction { } TextContent content = TextExtractor.findTextAt(psiFile, editor.getCaretModel().getOffset(), TextContent.TextDomain.ALL); - TextRange range = HighlightingUtil.selectionRange(editor); - if (content == null || (range.isEmpty() && !NaturalTextDetector.seemsNatural(content))) return false; - return content.fileRangeToText(range) != null; + if (content == null || !NaturalTextDetector.seemsNatural(content)) return false; + TextRange range = content.fileRangeToText(HighlightingUtil.selectionRange(editor)); + if (range == null) return false; + if (LangDetector.INSTANCE.getLanguage(content.toString()) == null) return false; + + return ContainerUtil.exists( + SentenceTokenizer.tokenize(content), + sentence -> sentence.start() <= range.getStartOffset() && range.getEndOffset() <= sentence.end() + ); } public record SuggestionsWithLanguage( @@ -82,36 +95,38 @@ public class RephraseAction extends IntentionAndQuickFixAction { public void applyFix(@NotNull Project project, PsiFile psiFile, @Nullable Editor editor) { if (editor == null) return; + int selStart = editor.getSelectionModel().getSelectionStart(); + int selEnd = editor.getSelectionModel().getSelectionEnd(); + SuggestionsWithLanguage rephraseData = ProgressManager.getInstance() .runProcessWithProgressSynchronously(() -> ReadAction.compute(() -> { - TextContent content = TextExtractor.findTextAt(psiFile, editor.getSelectionModel().getSelectionStart(), TextContent.TextDomain.ALL); - if (content == null) { - return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), null, null, null); + ParsedSentence sentence = ParsedSentence.findSentenceInFile(psiFile, selStart); + if (sentence == null) { + return rephraseTextContent(project, psiFile, selStart, selEnd); } - TextRange textRange = content.fileRangeToText(HighlightingUtil.selectionRange(editor)); + int sentenceLength = sentence.text.length(); + TextRange textRange = fileRangeToText(selStart, selEnd, sentence); if (textRange == null) { - return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), content.length(), null, null); + return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), sentenceLength, null, null); } - Language iso = LangDetector.INSTANCE.getLanguage(content.toString()); + Language iso = LangDetector.INSTANCE.getLanguage(sentence.text); if (iso == null) { - return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), content.length(), null, null); + return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), sentenceLength, null, null); } - int wordRangeCount = StandardWordTokenizer.INSTANCE.words(content.toString()).size(); - int rangeLength = textRange.getLength(); - GrazieFUSCounter.INSTANCE.reportRephraseRequested(iso, content.length(), rangeLength, wordRangeCount); - TextRange wordBoundRange = Text.alignToWordBounds(textRange, content.toString()); - List rephrasedSentences = rephrase(content, wordBoundRange, iso, project); + String rangeText = textRange.subSequence(sentence.text).toString(); + int wordsRangeCount = StandardWordTokenizer.INSTANCE.words(rangeText).size(); + GrazieFUSCounter.INSTANCE.reportRephraseRequested(iso, sentence.text.length(), textRange.getLength(), wordsRangeCount); + List ranges = getRangesToRephrase(sentence, textRange); + List>> rephrasedSentences = rephrase(sentence.text, ranges, iso, project); if (rephrasedSentences.isEmpty()) { - return new SuggestionsWithLanguage(iso, Collections.emptyList(), content.length(), rangeLength, wordRangeCount); + return new SuggestionsWithLanguage(iso, Collections.emptyList(), sentenceLength, textRange.getLength(), wordsRangeCount); } - List suggestions = ContainerUtil.mapNotNull( - rephrasedSentences, rephrasedSentence -> toListItem(wordBoundRange, content, rephrasedSentence) - ); - return new SuggestionsWithLanguage(iso, suggestions, content.length(), rangeLength, wordRangeCount); + List suggestions = ContainerUtil.flatMap(rephrasedSentences, s -> toListItem(sentence, s)); + return new SuggestionsWithLanguage(iso, suggestions, sentenceLength, textRange.getLength(), wordsRangeCount); }), GrazieBundle.message("intention.rephrase.progress.title"), true, project); if (rephraseData.suggestions().isEmpty()) { @@ -125,28 +140,42 @@ public class RephraseAction extends IntentionAndQuickFixAction { showPopup(project, editor, psiFile, rephraseData); } - private static ListItem toListItem(TextRange minRange, TextContent content, String suggestion) { - int commonPrefix = StringUtil.commonPrefixLength(content.toString(), suggestion); - int commonSuffix = StringUtil.commonSuffixLength(content.toString(), suggestion.substring(commonPrefix)); - - if (commonPrefix == 0 && commonSuffix == 0) { - return new ListItem(content.textRangeToFile(minRange), suggestion); + private static @NotNull SuggestionsWithLanguage rephraseTextContent(@NotNull Project project, PsiFile psiFile, int selStart, int selEnd) { + TextContent text = TextExtractor.findTextAt(psiFile, selStart, TextContent.TextDomain.ALL); + if (text == null || text.toString().isBlank()) { + return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), null, null, null); } - if (commonPrefix > minRange.getEndOffset() + 1 || content.length() - commonSuffix < minRange.getStartOffset() - 1) { - return null; + Language iso = LangDetector.INSTANCE.getLanguage(text.toString()); + if (iso == null) { + return new SuggestionsWithLanguage(Language.UNKNOWN, Collections.emptyList(), text.length(), null, null); } - TextRange range = Text.alignToWordBounds( - new TextRange( - Math.min(minRange.getStartOffset(), commonPrefix), - Math.max(content.length() - commonSuffix, minRange.getEndOffset()) - ), - content - ); - String replacement = suggestion.substring(range.getStartOffset(), suggestion.length() - (content.length() - range.getEndOffset())); + var textRange = selStart == selEnd ? TextRange.allOf(text.toString()) : text.fileRangeToText(TextRange.create(selStart, selEnd)); + if (textRange == null) { + return new SuggestionsWithLanguage(iso, Collections.emptyList(), text.length(), null, null); + } + + var rephrasedSentences = rephrase(text.toString(), List.of(textRange), iso, project); + int wordsRangeCount = StandardWordTokenizer.INSTANCE.words(text.toString()).size(); + if (rephrasedSentences.isEmpty()) { + return new SuggestionsWithLanguage(iso, Collections.emptyList(), text.length(), textRange.getLength(), wordsRangeCount); + } + + List suggestions = ContainerUtil.flatMap(rephrasedSentences, s -> toListItem(text, s)); + return new SuggestionsWithLanguage(iso, suggestions, text.length(), textRange.getLength(), wordsRangeCount); + } + + private static @NotNull List toListItem(TextContent content, Pair> rephrasedSentences) { + TextRange range = rephrasedSentences.getFirst(); TextRange fileRange = new TextRange(content.textOffsetToFile(range.getStartOffset()), content.textOffsetToFile(range.getEndOffset())); - return new ListItem(fileRange, replacement); + return ContainerUtil.map(rephrasedSentences.getSecond(), item -> new ListItem(fileRange, item)); + } + + private static @NotNull List toListItem(ParsedSentence sentence, Pair> rephrasedSentences) { + TextRange range = rephrasedSentences.getFirst(); + TextRange fileRange = new TextRange(sentence.textOffsetToFile(range.getStartOffset()), sentence.textOffsetToFile(range.getEndOffset())); + return ContainerUtil.map(rephrasedSentences.getSecond(), item -> new ListItem(fileRange, item)); } private void showPopup(Project project, Editor editor, PsiFile file, SuggestionsWithLanguage descriptor) { @@ -171,9 +200,11 @@ public class RephraseAction extends IntentionAndQuickFixAction { .setItemSelectedCallback(item -> { dropHighlighter(highlighter); if (item != null) { - highlighter.set(editor.getMarkupModel() - .addRangeHighlighter(EditorColors.SEARCH_RESULT_ATTRIBUTES, item.fileRange.getStartOffset(), item.fileRange.getEndOffset(), - HighlighterLayer.SELECTION + 1, HighlighterTargetArea.EXACT_RANGE)); + highlighter.set(editor.getMarkupModel().addRangeHighlighter( + EditorColors.SEARCH_RESULT_ATTRIBUTES, + item.fileRange.getStartOffset(), item.fileRange.getEndOffset(), + HighlighterLayer.SELECTION + 1, HighlighterTargetArea.EXACT_RANGE + )); } }) .addListener(new JBPopupListener() { @@ -198,9 +229,35 @@ public class RephraseAction extends IntentionAndQuickFixAction { highlighter.set(null); } - private static @NotNull List rephrase(TextContent content, TextRange wordBoundRange, Language iso, Project project) { + private static @Nullable TextRange fileRangeToText(int selStart, int selEnd, ParsedSentence sentence) { + Integer textStart = sentence.fileOffsetToText(selStart); + Integer textEnd = sentence.fileOffsetToText(selEnd); + return textStart == null || textEnd == null ? null : new TextRange(textStart, textEnd); + } + + private static List getRangesToRephrase(ParsedSentence sentence, TextRange textRange) { + if (textRange.getLength() > 0) { + return List.of(Text.alignToWordBounds(textRange, sentence.text)); + } + + var toolkit = LanguageToolkit.forLanguage(sentence.tree.treeSupport().getGrazieLanguage()); + var extendRanges = toolkit.selectioner().calcExtendSelectionRanges(sentence.tree, textRange.getStartOffset(), textRange.getEndOffset()); + assert !extendRanges.isEmpty(); + List result = new ArrayList<>(); + result.add(ijRange(extendRanges.getFirst())); + for (int i = 1; i < extendRanges.size(); i++) { + var range = extendRanges.get(i); + if (Strings.countChars(sentence.text.substring(range.start(), range.end()), ' ') > 10) { + break; + } + result.add(ijRange(range)); + } + return result; + } + + private static @NotNull List>> rephrase(String content, List ranges, Language iso, Project project) { try { - List rephrased = APIQueries.getRephraser().rephrase(content.toString(), wordBoundRange, iso, project); + List>> rephrased = APIQueries.getRephraser().rephrase(content, ranges, iso, project); return rephrased == null ? Collections.emptyList() : rephrased; } catch (TaskServerException e) { return Collections.emptyList(); @@ -208,11 +265,11 @@ public class RephraseAction extends IntentionAndQuickFixAction { } private record ListItem(TextRange fileRange, String replacement) { - @Override - public String toString() { - return replacement; - } + @Override + public String toString() { + return replacement; } + } @Override public boolean startInWriteAction() { diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/text/TreeRuleChecker.java b/plugins/grazie/src/main/kotlin/com/intellij/grazie/text/TreeRuleChecker.java index c0b218d57d5f..5e1cf2084086 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/text/TreeRuleChecker.java +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/text/TreeRuleChecker.java @@ -528,7 +528,7 @@ public final class TreeRuleChecker { } private static boolean touchesUnknownFragments(TextContent text, ai.grazie.rules.tree.TextRange range, ai.grazie.rules.Rule rule) { - var ruleRangeInText = toIdeaRange(range); + var ruleRangeInText = ijRange(range); if (ruleRangeInText.getEndOffset() > text.length()) { LOG.error( "Invalid match range " + ruleRangeInText + " for rule " + rule + " in a text of length " + text.length(), @@ -541,10 +541,6 @@ public final class TreeRuleChecker { return false; } - public static TextRange toIdeaRange(ai.grazie.rules.tree.TextRange reported) { - return new TextRange(reported.start(), reported.end()); - } - public static class TreeProblem extends GrazieProblem { public final RuleMatch match; private final List customFixes; diff --git a/plugins/grazie/src/main/kotlin/com/intellij/grazie/utils/Utils.kt b/plugins/grazie/src/main/kotlin/com/intellij/grazie/utils/Utils.kt index 3a7eb3fe930c..4e6ad2249a3d 100644 --- a/plugins/grazie/src/main/kotlin/com/intellij/grazie/utils/Utils.kt +++ b/plugins/grazie/src/main/kotlin/com/intellij/grazie/utils/Utils.kt @@ -18,6 +18,7 @@ import java.util.* fun ProblemFix.Part.Change.ijRange(): TextRange = TextRange(range.start, range.endExclusive) fun ai.grazie.text.TextRange.ijRange(): TextRange = TextRange(start, endExclusive) +fun ai.grazie.rules.tree.TextRange.ijRange(): TextRange = TextRange(start, end) fun TextRange.aiRange(): ai.grazie.text.TextRange = ai.grazie.text.TextRange(startOffset, endOffset) fun String.trimToNull(): String? = trim().takeIf(String::isNotBlank) diff --git a/plugins/grazie/src/tests.pro/testSrc/com/intellij/grazie/pro/RephraseTest.java b/plugins/grazie/src/tests.pro/testSrc/com/intellij/grazie/pro/RephraseTest.java index 6fdc6d49c7a7..3492e8bad45d 100644 --- a/plugins/grazie/src/tests.pro/testSrc/com/intellij/grazie/pro/RephraseTest.java +++ b/plugins/grazie/src/tests.pro/testSrc/com/intellij/grazie/pro/RephraseTest.java @@ -8,6 +8,8 @@ import com.intellij.openapi.progress.ProgressManager; import com.intellij.openapi.util.TextRange; import com.intellij.ui.ChooserInterceptor; import com.intellij.ui.UiInterceptors; +import com.intellij.util.containers.ContainerUtil; +import kotlin.Pair; import org.junit.jupiter.api.Test; import java.util.List; @@ -23,37 +25,30 @@ public class RephraseTest extends BaseTestCase { } private static final Map> REPHRASING = Map.of( + "horse", List.of("equine", "cavalry", "knight", "horsy"), "This horse raced past the barn", List.of( - "This equine raced past the barn", - "This cavalry raced past the barn", - "This knight raced past the barn", - "This horsy raced past the barn", "That horse ran past the house", "A horse ran past the house", "The horse ran past the gate" ), - "The horse raced past the barn", - List.of( - "The horse raced past the house", - "The horse raced past the gate" - ), + "the barn", List.of("the house", "the gate"), - "It's a type of popular music", - List.of( - "It's a type of popular song", - "It's a type of music that is popular", - "It's a type of music", - "It's a type of music that's popular" - ) + "of popular music", + List.of("of popular song", "of music that is popular", "of music", "of music that's popular"), + + "hester", + List.of("hest", "ponni", "søt hest") ); private void mockRephraser() { - APIQueries.overrideRephraser((text, range, language, project) -> { - List rephrased = REPHRASING.get(text); - if (rephrased == null) throw new IllegalArgumentException("Unable to rephrase: " + text); - return rephrased; + APIQueries.overrideRephraser((text, ranges, language, project) -> { + return ranges.stream() + .map(range -> new Pair(range, range.substring(text))) + .map(p -> new Pair<>(p.getFirst(), REPHRASING.get(p.getSecond()))) + .filter(p -> p.getSecond() != null) + .toList(); }, getTestRootDisposable()); } @@ -61,9 +56,7 @@ public class RephraseTest extends BaseTestCase { public void testRephraseSentence() { String text = "The library remained quiet throughout the afternoon, as students focused on their studies while the rain gently tapped on the windows."; - List rephrasedSentences = ProgressManager.getInstance().runProcess(() -> { - return APIQueries.getRephraser().rephrase(text, TextRange.allOf(text), Language.ENGLISH, getProject()); - }, new EmptyProgressIndicator()); + List rephrasedSentences = rephrase(text); assertFalse(rephrasedSentences.isEmpty(), "No rephrased sentences found"); rephrasedSentences.forEach(sentence -> assertNotEquals(text, sentence)); } @@ -71,13 +64,28 @@ public class RephraseTest extends BaseTestCase { @Test public void testRephraseShortSentenceInRussian() { String text = "Почему лошади умееют так высоко прыгать?"; - List rephrasedSentences = ProgressManager.getInstance().runProcess(() -> { - return APIQueries.getRephraser().rephrase(text, TextRange.allOf(text), Language.RUSSIAN, getProject()); - }, new EmptyProgressIndicator()); + List rephrasedSentences = rephrase(text); assertFalse(rephrasedSentences.isEmpty(), "No rephrased sentences found"); rephrasedSentences.forEach(sentence -> assertNotEquals(text, sentence)); } + @Test + public void testNorwegian() { + // Norwegian is not in the list of supported languages, but rephrasing is done via LLM which can easily understand it. + mockRephraser(); + UiInterceptors.register( + new ChooserInterceptor( + List.of("hest", "ponni", "søt hest"), + ".*ponni.*" + ) + ); + checkIntention( + "a.txt", getIntentionText(), + "Some other text.\n\nHvorfor kan hester hoppe så høyt? De er utrolig flinke til det!", + "Some other text.\n\nHvorfor kan ponni hoppe så høyt? De er utrolig flinke til det!" + ); + } + @Test public void testWord() { mockRephraser(); @@ -106,7 +114,6 @@ public class RephraseTest extends BaseTestCase { "the house" ) ); - mockRephraser(); checkIntention( "a.html", getIntentionText(), "The horse raced past the barn", @@ -138,4 +145,11 @@ public class RephraseTest extends BaseTestCase { message: Hello World """); } + + private List rephrase(String text) { + var rephrasedSentences = ProgressManager.getInstance().runProcess(() -> { + return APIQueries.getRephraser().rephrase(text, List.of(TextRange.allOf(text)), Language.RUSSIAN, getProject()); + }, new EmptyProgressIndicator()); + return ContainerUtil.flatMap(rephrasedSentences, s -> s.getSecond()); + } }