diff --git a/plugins/evaluation-plugin/core/resources/script.js b/plugins/evaluation-plugin/core/resources/script.js index 2a69d0e645df..479f578b46ad 100644 --- a/plugins/evaluation-plugin/core/resources/script.js +++ b/plugins/evaluation-plugin/core/resources/script.js @@ -292,7 +292,8 @@ function addSuggestions(sessionDiv, popup, lookup) { p.setAttribute("style", "font-weight: bold;") } const presentationText = suggestions[i].presentationText.replace(//g, '>') - p.innerHTML = removeCommonIndentFromCodeSnippet(presentationText) + p.innerHTML = presentationText + // p.innerHTML = removeCommonIndentFromCodeSnippet(presentationText) // FIXME doesn't work anyway? suggestionDiv.appendChild(p) popup.appendChild(suggestionDiv) } diff --git a/plugins/evaluation-plugin/core/resources/style.css b/plugins/evaluation-plugin/core/resources/style.css index 63e8e9e1351c..ca5bf15ad15b 100644 --- a/plugins/evaluation-plugin/core/resources/style.css +++ b/plugins/evaluation-plugin/core/resources/style.css @@ -69,6 +69,7 @@ button:focus { .session { position: relative; + display: block; font-weight: bold; } @@ -126,7 +127,7 @@ button:focus { border-bottom: none; border-top: none; z-index: 99; - top: 1.5em; + top: 100%; left: 0; } diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt index 8997df82d971..b426b015e693 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/Config.kt @@ -16,7 +16,8 @@ import java.nio.file.Paths * @property outputDir The output directory for the evaluation results. * @property strategy The evaluation strategy used. * @property actions The configuration for actions generation step. - * @property fileDataset Dataset configuration for standalone setups + * @property csvDataset Dataset configuration for CSV file. + * @property conflictDataset Dataset configuration for conflict resolution evaluations. * @property interpret The configuration for actions interpretation step. * @property reorder The configuration for element reordering step. * @property reports The configuration for report generation step. @@ -25,7 +26,8 @@ data class Config private constructor( val outputDir: String, val strategy: EvaluationStrategy, val actions: ActionsGeneration?, - val fileDataset: FileDataset?, + val csvDataset: FileDataset?, + val conflictDataset: ConflictDataset?, val interpret: ActionsInterpretation, val reorder: ReorderElements, val reports: ReportGeneration @@ -76,6 +78,10 @@ data class Config private constructor( val chunkSize: Int, ) + data class ConflictDataset internal constructor( + val url: String + ) + /** * Represents the configuration for the interpretation of actions. * @@ -135,6 +141,7 @@ data class Config private constructor( class Builder internal constructor() { var actions: ActionsGeneration? = null var fileDataset: FileDataset? = null + var conflictDataset: ConflictDataset? = null var outputDir: String? = null var strategy: EvaluationStrategy = EvaluationStrategy.defaultStrategy @@ -161,7 +168,8 @@ data class Config private constructor( constructor(config: Config) : this() { actions = config.actions - fileDataset = config.fileDataset + fileDataset = config.csvDataset + conflictDataset = config.conflictDataset outputDir = config.outputDir strategy = config.strategy saveLogs = config.interpret.saveLogs @@ -206,6 +214,7 @@ data class Config private constructor( strategy, actions, fileDataset, + conflictDataset, ActionsInterpretation( experimentGroup, sessionsLimit, diff --git a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt index 43997034d4fe..481e2b8be052 100644 --- a/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt +++ b/plugins/evaluation-plugin/core/src/com/intellij/cce/workspace/ConfigFactory.kt @@ -63,7 +63,8 @@ object ConfigFactory { map.getIfExists("language"), this ) - deserializeFileDataset(map.getIfExists("fileDataset"), this) + deserializeFileDataset(map.getIfExists("csvDataset"), this) + deserializeConflictDataset(map.getIfExists("conflictDataset"), this) deserializeStrategy(map.getIfExists("strategy"), strategySerializer, actions?.language, this) deserializeActionsInterpretation(map.getIfExists("interpret"), this) @@ -123,6 +124,19 @@ object ConfigFactory { ) } + private fun deserializeConflictDataset( + map: Map?, + builder: Config.Builder + ) { + if (map == null) { + return + } + + builder.conflictDataset = Config.ConflictDataset( + map.getAs("url") + ) + } + private fun deserializeActionsInterpretation(map: Map?, builder: Config.Builder) { if (map == null) return if (map.containsKey("experimentGroup")) { diff --git a/plugins/evaluation-plugin/evaluationconfig/conflictresolution/config.json b/plugins/evaluation-plugin/evaluationconfig/conflictresolution/config.json new file mode 100644 index 000000000000..a5911c46fa93 --- /dev/null +++ b/plugins/evaluation-plugin/evaluationconfig/conflictresolution/config.json @@ -0,0 +1,23 @@ +{ + "outputDir": "ml-eval-conflict-resolution-output", + "strategy": {}, + "conflictDataset": { + "url": "existing:conflicts" + }, + "interpret": { + "saveLogs": false, + "saveFeatures": false, + "sessionProbability": 1.0, + "sessionSeed": null, + "sessionsLimit": 30, + "filesLimit": 3, + "order": "LINEAR", + "trainTestSplit": 70 + }, + "reports": { + "evaluationTitle": "Default", + "defaultMetrics": null, + "sessionsFilters": [], + "comparisonFilters": [] + } +} diff --git a/plugins/evaluation-plugin/evaluationconfig/standaloneexample/config.json b/plugins/evaluation-plugin/evaluationconfig/standaloneexample/config.json index bfe29dd7c9c7..a2050befd92e 100644 --- a/plugins/evaluation-plugin/evaluationconfig/standaloneexample/config.json +++ b/plugins/evaluation-plugin/evaluationconfig/standaloneexample/config.json @@ -1,7 +1,7 @@ { "outputDir": "ml-eval-standalone-example-output", "strategy": {}, - "fileDataset": { + "csvDataset": { "url": "./latin_letters.csv", "chunkSize": 10 }, diff --git a/plugins/evaluation-plugin/resources/META-INF/plugin.xml b/plugins/evaluation-plugin/resources/META-INF/plugin.xml index e44ac615fe82..dcbb39d17625 100644 --- a/plugins/evaluation-plugin/resources/META-INF/plugin.xml +++ b/plugins/evaluation-plugin/resources/META-INF/plugin.xml @@ -55,6 +55,7 @@ + diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt index 99993a9be2a2..0f7e956f2aa4 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ContextCollectionEvaluationCommand.kt @@ -72,6 +72,7 @@ internal class ContextCollectionEvaluationCommand : CompletionEvaluationStarter. dataset.project, dataset.processor, feature.name, + dataset.featureInvoker ) { override fun prepare(datasetContext: DatasetContext, progress: Progress) { val files = runReadAction { diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt index 0425da96acda..67099c244055 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/CsvDataset.kt @@ -14,7 +14,8 @@ import kotlin.io.path.readLines class CsvDataset( private val datasetRef: DatasetRef, private val chunkSize: Int, - private val targetField: String + private val targetField: String, + private val featureInvoker: FeatureInvoker, ) : EvaluationDataset { override val setupSdk: EvaluationStep? = null override val checkSdk: EvaluationStep? = null @@ -70,7 +71,6 @@ class CsvDataset( override val presentationText: String = presentationText.toString() override fun evaluate( - featureInvoker: FeatureInvoker, handler: InterpretationHandler, filter: InterpretFilter, order: InterpretationOrder, diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt index 1e7442755b1b..8c7906fac164 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/EvaluationDataset.kt @@ -11,7 +11,7 @@ import com.intellij.cce.util.Progress /** * Represents data which will be used for evaluation. */ -interface EvaluationDataset { +interface EvaluationDataset { // FIXME with implicit invoker inside it may make sense to merge this interface into EvaluationEnvironment val setupSdk: EvaluationStep? val checkSdk: EvaluationStep? @@ -32,7 +32,6 @@ interface EvaluationDatasetChunk { val presentationText: String fun evaluate( - featureInvoker: FeatureInvoker, handler: InterpretationHandler, filter: InterpretFilter, order: InterpretationOrder, diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt index d53763ef2b49..a138ea02f16c 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/actions/ProjectActionsDataset.kt @@ -36,6 +36,7 @@ open class ProjectActionsDataset( val project: Project, val processor: GenerateActionsProcessor, private val featureName: String, + val featureInvoker: FeatureInvoker, ) : EvaluationDataset { private val datasetRef = config.sourceFile.run { val sf = this ?: "" @@ -237,7 +238,6 @@ open class ProjectActionsDataset( override val sessionCount: Int = fileActions.sessionsCount override fun evaluate( - featureInvoker: FeatureInvoker, handler: InterpretationHandler, filter: InterpretFilter, order: InterpretationOrder, diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt index 9b6c9e298cf6..38786415f269 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/EvaluableFeatureBase.kt @@ -54,9 +54,9 @@ abstract class EvaluableFeatureBase(override val name: S EvaluationRootInfo(true), project, getGenerateActionsProcessor(strategy, project), - name - ), - featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy) + name, + featureInvoker = getFeatureInvoker(project, Language.resolve(actions.language), strategy) + ) ) } } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt index 7fc8c7c6a698..68d3b3d4647d 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/StandaloneFeature.kt @@ -18,8 +18,6 @@ abstract class StandaloneFeature( abstract fun getDataset(config: Config): EvaluationDataset - abstract fun getFeatureInvoker(strategy: T): FeatureInvoker - override fun getPreliminaryEvaluationSteps(): List = emptyList() override fun getEvaluationSteps(config: Config): List = emptyList() @@ -34,8 +32,7 @@ abstract class StandaloneFeature( override fun prepareEnvironment(config: Config): EvaluationEnvironment { return StandaloneEnvironment( - getDataset(config), - getFeatureInvoker(config.strategy()) + getDataset(config) ) } } \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt new file mode 100644 index 000000000000..8b9c78483b61 --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictDataset.kt @@ -0,0 +1,124 @@ +package com.intellij.cce.evaluable.conflictResolution + +import com.intellij.cce.actions.* +import com.intellij.cce.core.* +import com.intellij.cce.evaluation.EvaluationStep +import com.intellij.cce.interpreter.InterpretFilter +import com.intellij.cce.interpreter.InterpretationHandler +import com.intellij.cce.interpreter.InterpretationOrder +import com.intellij.cce.util.Progress +import java.nio.file.Path +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.readText + +class ConflictDataset( + private val datasetRef: DatasetRef, + private val conflictResolver: ConflictResolver<*> +) : EvaluationDataset { + override val setupSdk: EvaluationStep? = null + override val checkSdk: EvaluationStep? = null + + override val preparationDescription: String = "Checking that conflict dataset exists" + + override fun prepare(datasetContext: DatasetContext, progress: Progress) { + datasetRef.prepare(datasetContext) + } + + override fun sessionCount(datasetContext: DatasetContext): Int = + datasetContext.path(datasetRef).listDirectoryEntries().size // FIXME should be pre-calculated + + override fun chunks(datasetContext: DatasetContext): Iterator { + return datasetContext.path(datasetRef).listDirectoryEntries().map { conflictPath -> + ConflictChunk(conflictPath) + }.iterator() + } + + private inner class ConflictChunk( + private val conflictPath: Path, + override val sessionCount: Int = 1 // FIXME we should pre-calculate it? + ) : EvaluationDatasetChunk { + override val datasetName: String = datasetRef.name + override val name: String = conflictPath.fileName.toString() + override val presentationText: String = readText("result") + + override fun evaluate( + handler: InterpretationHandler, // FIXME use this as well as filter and order + filter: InterpretFilter, + order: InterpretationOrder, + sessionHandler: (Session) -> Unit + ): List { + val props = ConflictResolver.Props( + base = readText("base"), + parent1 = readText("parent1"), + parent2 = readText("parent2"), + target = readText("result"), + ) + + val resolvedConflicts = conflictResolver.resolveConflicts(props) + + require(resolvedConflicts.keys.containsAll(TextLabel.entries)) + + return (0 until resolvedConflicts.size).map { conflictIndex -> + val session = conflictSession(resolvedConflicts, conflictIndex) + sessionHandler(session) + session + } + } + + private fun conflictSession( + conflicts: MatchSequence, + conflictIndex: Int, + ): Session { + val expected = conflicts.text(TextLabel.TARGET, conflictIndex) + val session = Session( + conflicts.charOffset(TextLabel.TARGET, conflictIndex), + expected, + expected.length, + TokenProperties.UNKNOWN + ) + + val result = conflicts.text(TextLabel.RESULT, conflictIndex) + + val suggestions = listOf( + Suggestion( + result, + result, + SuggestionSource.INTELLIJ, + emptyMap(), + isRelevant = expected == result + ) + ) + + val lookup = Lookup( + "", + 0, + suggestions, + 0, + null, + selectedPosition = suggestions.indexOfFirst { it.isRelevant }, + isNew = false, + mapOf( + "aia_context" to contextPresentation(conflicts, conflictIndex) + ) + ) + + session.addLookup(lookup) + + return session + } + + private fun readText(namePrefix: String): String = + conflictPath.listDirectoryEntries().first { it.fileName.toString().startsWith(namePrefix) }.readText() + + private fun contextPresentation(conflicts: MatchSequence<*>, conflictIndex: Int) = """ +<<<<<<< BASE +${conflicts.text(TextLabel.BASE, conflictIndex).trimEnd()} +======= PARENT1 +${conflicts.text(TextLabel.PARENT_1, conflictIndex).trimEnd()} +======= PARENT2 +${conflicts.text(TextLabel.PARENT_2, conflictIndex).trimEnd()} +>>>>>>> +""".trim() + } +} + diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt new file mode 100644 index 000000000000..e13ad2353af5 --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionFeature.kt @@ -0,0 +1,24 @@ +package com.intellij.cce.evaluable.conflictResolution + +import com.intellij.cce.actions.DatasetRef +import com.intellij.cce.actions.EvaluationDataset +import com.intellij.cce.evaluable.StandaloneFeature +import com.intellij.cce.evaluable.StrategySerializer +import com.intellij.cce.metric.Metric +import com.intellij.cce.metric.PrecisionMetric +import com.intellij.cce.metric.SessionsCountMetric +import com.intellij.cce.workspace.Config + +class ConflictResolutionFeature : StandaloneFeature("conflict-resolution") { + override fun getStrategySerializer(): StrategySerializer = ConflictResolutionStrategy.Serializer() + + override fun getDataset(config: Config): EvaluationDataset = ConflictDataset( + DatasetRef.parse(config.conflictDataset!!.url), + TheirConflictResolver() + ) + + override fun getMetrics(): List = listOf( + SessionsCountMetric(), + PrecisionMetric() + ) +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionStrategy.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionStrategy.kt new file mode 100644 index 000000000000..50e80b63a161 --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolutionStrategy.kt @@ -0,0 +1,17 @@ +package com.intellij.cce.evaluable.conflictResolution + +import com.google.gson.JsonObject +import com.google.gson.JsonSerializationContext +import com.intellij.cce.evaluable.EvaluationStrategy +import com.intellij.cce.evaluable.StrategySerializer +import com.intellij.cce.filter.EvaluationFilter +import java.lang.reflect.Type + +class ConflictResolutionStrategy : EvaluationStrategy { + override val filters: Map = mapOf() + + class Serializer : StrategySerializer { + override fun serialize(src: ConflictResolutionStrategy, typeOfSrc: Type, context: JsonSerializationContext): JsonObject = JsonObject() + override fun deserialize(map: Map, language: String): ConflictResolutionStrategy = ConflictResolutionStrategy() + } +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolver.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolver.kt new file mode 100644 index 000000000000..c3f070bcd12e --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/ConflictResolver.kt @@ -0,0 +1,40 @@ +package com.intellij.cce.evaluable.conflictResolution + +interface ConflictResolver { + fun resolveConflicts(props: Props): MatchSequence + + class Props( + val base: String, + val parent1: String, + val parent2: String, + val target: String + ) +} + +open class TheirConflictResolver : ConflictResolver { + override fun resolveConflicts(props: ConflictResolver.Props): MatchSequence { + val resultText = props.parent2 + val conflicts = calculateAlignment(TextLabel.BASE, props.base, TextLabel.RESULT, resultText) + .intersectAlignments(calculateAlignment(TextLabel.BASE, props.base, TextLabel.PARENT_1, props.parent1)) + .intersectAlignments(calculateAlignment(TextLabel.BASE, props.base, TextLabel.PARENT_2, props.parent2)) + .intersectAlignments(calculateAlignment(TextLabel.BASE, props.base, TextLabel.TARGET, props.target)) + .complement() + + val nonObviousConflicts = (0 until conflicts.size).filterNot { conflictIndex -> + conflicts.text(TextLabel.BASE, conflictIndex) == conflicts.text(TextLabel.PARENT_1, conflictIndex) || + conflicts.text(TextLabel.BASE, conflictIndex) == conflicts.text(TextLabel.PARENT_2, conflictIndex) || + conflicts.text(TextLabel.PARENT_1, conflictIndex) == conflicts.text(TextLabel.PARENT_2, conflictIndex) + } + + return conflicts.filter(nonObviousConflicts) + } + + @Suppress("SameParameterValue", "MemberVisibilityCanBePrivate") + protected fun calculateAlignment(label1: TextLabel, text1: String, label2: TextLabel, text2: String): MatchSequence { + val splitText1 = LineSplitText(text1) + val splitText2 = LineSplitText(text2) + val distances = calculateEditDistance(splitText1, splitText2) + val (ranges1, ranges2) = retrieveAlignment(distances) + return MatchSequence(mapOf(label1 to ranges1, label2 to ranges2), mapOf(label1 to splitText1, label2 to splitText2)) + } +} diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/MatchSequence.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/MatchSequence.kt new file mode 100644 index 000000000000..94b3b1c1315a --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/MatchSequence.kt @@ -0,0 +1,185 @@ +package com.intellij.cce.evaluable.conflictResolution + + +/** + * Represents a sequence of matched ranges within texts. + * Can be used to store texts alignment or conflict sequence. + * + * @constructor For each text range, there should be corresponding ranges in other texts (maybe empty ones). + * Ranges for each text should be sorted. + * Check the init section for range restrictions. + */ +class MatchSequence( + private val ranges: Map>, + private val originalTexts: Map>, +) { + init { + require(ranges.keys == originalTexts.keys) { + "Keys mismatch: ${ranges.keys} != ${originalTexts.keys}" + } + + require(ranges.map { it.value.size }.distinct().size <= 1) { + "For all texts there should be the same amount of ranges." + } + + for ((label, labelRanges) in ranges) { + require((labelRanges.firstOrNull()?.first ?: 0) >= 0) { + "First range should be non-negative in $label: $labelRanges" + } + + for (i in 0 until labelRanges.size - 1) { + require(labelRanges[i].last < labelRanges[i + 1].first) { + "Ranges should be sorted in $label: $labelRanges" + } + } + + require((labelRanges.lastOrNull()?.last ?: -1) < originalTexts.getValue(label).size) { + "Last range shouldn't be greater then text size in $label: $labelRanges" + } + } + } + + val keys get() = ranges.keys + + val size: Int = keys.firstOrNull()?.let { ranges[it]!!.size } ?: 0 + + fun range(label: TextLabel, index: Int) = ranges.getValue(label)[index] + + fun text(label: TextLabel) = originalTexts.getValue(label).text + fun text(label: TextLabel, index: Int): String = originalTexts.getValue(label).substring(range(label, index)) + + fun charOffset(label: TextLabel, index: Int): Int = originalTexts.getValue(label).charOffset(range(label, index).first) + + fun filter(indexes: Iterable): MatchSequence { + return MatchSequence( + ranges.mapValues { it.value.filterIndexed { index, _ -> indexes.contains(index) } }, + originalTexts + ) + } + + fun requireAlignment() { + for (index in 0 until size) { + val indexRanges = ranges.map { it.key to it.value[index] }.toMap() + require(indexRanges.map { it.value.last - it.value.first }.distinct().size == 1) { + "There are ranges of different sizes in mapping $index: $indexRanges\n$ranges" + } + } + } + + /** + * Generates a sequence of text ranges that are not covered by the original match ranges. + * + * This method computes the complement of the original match ranges, effectively identifying all + * text ranges that do not match. + */ + fun complement(): MatchSequence { + if (ranges.isEmpty() || size == 0) { + return MatchSequence(originalTexts.map { (key, text) -> key to listOf(0 until text.size) }.toMap(), originalTexts) + } + + val result = ranges.map { it.key to mutableListOf() }.toMap() + + if (ranges.any { it.value.first().first > 0 }) { + ranges.forEach { (key, keyRanges) -> + result.getValue(key) += (0 until keyRanges.first().first) + } + } + + for (i in 0 until size - 1) { + ranges.forEach { (key, keyRanges) -> + val prev = keyRanges[i] + val next = keyRanges[i + 1] + + result.getValue(key) += (prev.last + 1 until next.first) + } + } + + if (ranges.any { (key, ranges) -> ranges.last().last + 1 < originalTexts.getValue(key).size }) { + ranges.forEach { (key, keyRange) -> + result.getValue(key) += (keyRange.last().last + 1 until originalTexts.getValue(key).size) + } + } + + return MatchSequence(result, originalTexts) + } + + /** + * Intersects the current MatchSequence with another provided MatchSequence. + * + * This method requires that both sequences align properly. It computes the intersection + * of the sequences using only one common key and ensures that the original texts for the + * common key are identical. + */ + fun intersectAlignments(other: MatchSequence): MatchSequence { + requireAlignment() + other.requireAlignment() + + val intersectionKeys = keys intersect other.keys + require(intersectionKeys.size == 1) { + "Only intersection of two sequences with one same key is supported: ${keys}, ${other.keys}" + } + + val commonKey = intersectionKeys.first() + + require(originalTexts.getValue(commonKey) == other.originalTexts.getValue(commonKey)) { + "Texts are different for key $commonKey" + } + + val result = (keys union other.keys).associateWith { mutableListOf() } + + var pointer1 = 0 + var pointer2 = 0 + + while (pointer1 < size && pointer2 < other.size) { + val range1 = range(commonKey, pointer1) + val range2 = other.range(commonKey, pointer2) + + val first = maxOf(range1.first, range2.first) + + if (range1.last < first) { + pointer1 += 1 + continue + } + + if (range2.last < first) { + pointer2 += 1 + continue + } + + val last = minOf(range1.last, range2.last) + val length = last - first + 1 + + val delta1 = first - range1.first + val delta2 = first - range2.first + + ranges.forEach { (key, keyRanges) -> + val range = keyRanges[pointer1] + result.getValue(key) += range.first + delta1 until range.first + delta1 + length + } + + other.ranges.filter { it.key != commonKey }.forEach { (key, keyRanges) -> + val range = keyRanges[pointer2] + result.getValue(key) += range.first + delta2 until range.first + delta2 + length + } + + if (range1.last == last) { + pointer1 += 1 + } + + + if (range2.last == last) { + pointer2 += 1 + } + } + + return MatchSequence(result, originalTexts + other.originalTexts) + } +} + +enum class TextLabel { + BASE, + PARENT_1, + PARENT_2, + TARGET, + RESULT; +} diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/SplitText.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/SplitText.kt new file mode 100644 index 000000000000..bd3ed92ce8b7 --- /dev/null +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/conflictResolution/SplitText.kt @@ -0,0 +1,95 @@ +package com.intellij.cce.evaluable.conflictResolution + +interface SplitText { + val text: String + + val size: Int + + operator fun get(index: Int): T + + fun substring(range: IntRange): String + fun charOffset(index: Int): Int +} + +data class LineSplitText(override val text: String) : SplitText { + private val lines = text.lines() + private val offsets = lines.scanIndexed(0) { i, acc, line -> acc + line.length + (if (i == lines.lastIndex) 0 else 1) } + + override val size: Int get() = lines.size + + override fun get(index: Int): String = lines[index] + override fun substring(range: IntRange): String = lines.subList(range.first, range.last + 1).joinToString("\n") + override fun charOffset(index: Int): Int = offsets[index] +} + +data class CharacterSplitText(override val text: String) : SplitText { + override val size: Int = text.length + + override fun get(index: Int): Char = text[index] + override fun substring(range: IntRange): String = text.substring(range) + override fun charOffset(index: Int): Int = index +} + +fun calculateEditDistance(left: SplitText, right: SplitText): Array { + val dp = Array(left.size + 1) { IntArray(right.size + 1) } + + for (i in 0..left.size) { + for (j in 0..right.size) { + dp[i][j] = when { + i == 0 -> j + j == 0 -> i + else -> minOf( + dp[i - 1][j] + 1, + dp[i][j - 1] + 1, + dp[i - 1][j - 1] + (if (left[i - 1] == right[j - 1]) 0 else 2) + ) + } + } + } + + return dp +} + +fun retrieveAlignment(distances: Array): Pair, List> { + val leftRanges = mutableListOf() + val rightRanges = mutableListOf() + + var i = distances.size - 1 + var j = distances.first().size - 1 + var matchedLength = 0 + + fun commitSection() { + if (matchedLength > 0) { + leftRanges.add(i until i + matchedLength) + rightRanges.add(j until j + matchedLength) + matchedLength = 0 + } + } + + while (i > 0 && j > 0) { + when { + distances[i - 1][j - 1] <= distances[i - 1][j] && distances[i - 1][j - 1] <= distances[i][j - 1] -> { + if (distances[i - 1][j - 1] == distances[i][j]) { + matchedLength += 1 + } + else { + commitSection() + } + + i -= 1 + j -= 1 + } + distances[i - 1][j] <= distances[i][j - 1] -> { + commitSection() + i -= 1 + } + else -> { + commitSection() + j -= 1 + } + } + } + commitSection() + + return Pair(leftRanges.reversed(), rightRanges.reversed()) +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt index b99c86dae75b..6118c79682a3 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluable/standaloneExample/StandaloneExampleFeature.kt @@ -9,7 +9,6 @@ import com.intellij.cce.evaluable.EvaluationStrategy import com.intellij.cce.evaluable.StandaloneFeature import com.intellij.cce.evaluable.StrategySerializer import com.intellij.cce.filter.EvaluationFilter -import com.intellij.cce.interpreter.FeatureInvoker import com.intellij.cce.metric.Metric import com.intellij.cce.metric.PrecisionMetric import com.intellij.cce.metric.SessionsCountMetric @@ -24,16 +23,15 @@ class StandaloneExampleFeature : StandaloneFeature("standalone- } override fun getDataset(config: Config): EvaluationDataset { - val fileDataset = config.fileDataset ?: throw IllegalStateException("Required dataset config") + val fileDataset = config.csvDataset ?: throw IllegalStateException("Required dataset config") return CsvDataset( datasetRef = DatasetRef.parse(fileDataset.url), chunkSize = fileDataset.chunkSize, - targetField = "Type" + targetField = "Type", + featureInvoker = StandaloneExampleInvoker(), ) } - override fun getFeatureInvoker(strategy: DatasetStrategy): FeatureInvoker = StandaloneExampleInvoker() - override fun getMetrics(): List = listOf( SessionsCountMetric(), PrecisionMetric() diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt index 6e3a5cef5c8a..7f2c410339cb 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ActionsInterpretationHandler.kt @@ -23,8 +23,7 @@ import kotlin.system.measureTimeMillis class ActionsInterpretationHandler( private val config: Config, - private val datasetContext: DatasetContext, - private val featureInvoker: FeatureInvoker, + private val datasetContext: DatasetContext ) { companion object { @@ -64,7 +63,7 @@ class ActionsInterpretationHandler( workspace.fullLineLogsStorage.enableLogging(chunk.name) try { val sessions = logsSaver.invokeRememberingLogs { - chunk.evaluate(featureInvoker, handler, filter, interpretationConfig.order) { session -> + chunk.evaluate(handler, filter, interpretationConfig.order) { session -> featuresStorage.saveSession(session, chunk.name) } } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt index 6813b0e5442a..b43c4b451a88 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/BackgroundStepFactory.kt @@ -20,14 +20,14 @@ class BackgroundStepFactory( override fun generateActionsStep(): EvaluationStep = DatasetPreparationStep(environment.dataset, datasetContext) override fun interpretActionsStep(): EvaluationStep = - ActionsInterpretationStep(config, environment.dataset, datasetContext, environment.featureInvoker, newWorkspace = false) + ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = false) override fun generateReportStep(): EvaluationStep = ReportGenerationStep(inputWorkspacePaths?.map { EvaluationWorkspace.open(it, SetupStatsCollectorStep.statsCollectorLogsDirectory) }, config.reports.sessionsFilters, config.reports.comparisonFilters, feature) override fun interpretActionsOnNewWorkspaceStep(): EvaluationStep = - ActionsInterpretationStep(config, environment.dataset, datasetContext, environment.featureInvoker, newWorkspace = true) + ActionsInterpretationStep(config, environment.dataset, datasetContext, newWorkspace = true) override fun reorderElements(): EvaluationStep = ReorderElementsStep(config) diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt index 899490e99ae6..3cc045f6a582 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/EvaluationEnvironment.kt @@ -12,7 +12,6 @@ import com.intellij.cce.workspace.EvaluationWorkspace */ interface EvaluationEnvironment : AutoCloseable { val dataset: EvaluationDataset - val featureInvoker: FeatureInvoker fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? } @@ -22,7 +21,6 @@ interface EvaluationEnvironment : AutoCloseable { */ class StandaloneEnvironment( override val dataset: EvaluationDataset, - override val featureInvoker: FeatureInvoker ) : EvaluationEnvironment { override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? = step.runInIntellij(null, workspace) diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt index a342591510ab..3c75662ab10f 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/ProjectEnvironment.kt @@ -14,7 +14,6 @@ import java.nio.file.FileSystems class ProjectEnvironment( val project: Project, override val dataset: EvaluationDataset, - override val featureInvoker: FeatureInvoker, ) : EvaluationEnvironment { override fun execute(step: EvaluationStep, workspace: EvaluationWorkspace): EvaluationWorkspace? = @@ -43,8 +42,7 @@ class ProjectEnvironment( return ProjectEnvironment( project, - environment.dataset, - environment.featureInvoker + environment.dataset ) } } diff --git a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt index 3e5e20935dfb..cf793b88f3c6 100644 --- a/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt +++ b/plugins/evaluation-plugin/src/com/intellij/cce/evaluation/step/ActionsInterpretationStep.kt @@ -13,7 +13,6 @@ class ActionsInterpretationStep( private val config: Config, private val dataset: EvaluationDataset, private val datasetContext: DatasetContext, - private val featureInvoker: FeatureInvoker, private val newWorkspace: Boolean ) : BackgroundEvaluationStep { override val name: String = "Actions interpreting" @@ -24,7 +23,7 @@ class ActionsInterpretationStep( val resultWorkspace = if (newWorkspace) EvaluationWorkspace.create(config, SetupStatsCollectorStep.statsCollectorLogsDirectory) else workspace - ActionsInterpretationHandler(config, datasetContext, featureInvoker).invoke(dataset, resultWorkspace, progress) + ActionsInterpretationHandler(config, datasetContext).invoke(dataset, resultWorkspace, progress) return resultWorkspace } } \ No newline at end of file diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/MatchSequenceTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/MatchSequenceTest.kt new file mode 100644 index 000000000000..27bc5cfd44de --- /dev/null +++ b/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/MatchSequenceTest.kt @@ -0,0 +1,104 @@ +package com.intellij.cce.evaluable.conflictResolution + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import kotlin.math.absoluteValue +import kotlin.math.sign + +class MatchSequenceTest { + + @Test + fun `test complement`() { + assertEquals(listOf(0..9), complement(10)) + assertEquals(listOf(), complement(10, 0..9)) + assertEquals(listOf(5..9), complement(10, 0..4)) + assertEquals(listOf(0..4), complement(10, 5..9)) + assertEquals(listOf(2..3, 6..7), complement(10, 0..1, 4..5, 8..9)) + assertEquals(listOf(0..1, 4..5, 8..9), complement(10, 2..3, 6..7)) + } + + @Test + fun `test intersectAlignments`() { + assertEquals(listOf(), intersect(listOf(), listOf())) + assertEquals(listOf(0..9), intersect(listOf(0..9), listOf(0..9))) + assertEquals(listOf(), intersect(listOf(0..9), listOf())) + assertEquals(listOf(0..2), intersect(listOf(0..9), listOf(0..2))) + assertEquals(listOf(3..6), intersect(listOf(0..9), listOf(3..6))) + assertEquals(listOf(7..9), intersect(listOf(0..9), listOf(7..9))) + assertEquals(listOf(1..2, 7..8), intersect(listOf(0..2, 7..9), listOf(1..8))) + assertEquals(listOf(1..2, 7..8), intersect(listOf(0..2, 7..9), listOf(1..3, 6..8))) + } + + private fun complement(size: Int, vararg ranges: IntRange): List { + val offset = 10 + val scale = 2 + + val sequence = MatchSequence( + mapOf( + TextLabel.BASE to ranges.toList(), + TextLabel.PARENT_1 to ranges.map { (it.first + offset)..(it.last + offset) }, + TextLabel.PARENT_2 to ranges.map { (it.first * scale)..(it.last * scale) } + ), + mapOf( + TextLabel.BASE to CharacterSplitText("a".repeat(size)), + TextLabel.PARENT_1 to CharacterSplitText("b".repeat(size + offset)), + TextLabel.PARENT_2 to CharacterSplitText("c".repeat(size * scale)) + ) + ) + + val complement = sequence.complement() + + val baseRanges = complement.ranges(TextLabel.BASE) + + assertEquals(baseRanges, complement.ranges(TextLabel.PARENT_1).map { maxOf(0, it.first - offset)..(it.last - offset) }) + assertEquals(baseRanges, complement.ranges(TextLabel.PARENT_2).map { + val start = it.first.floorDiv(scale) + it.first.rem(scale).sign.absoluteValue // round up division + start..(if (it.last < it.first) start - 1 else it.last / scale) + }) + + return baseRanges + .filterNot { it.isEmpty() } // because of offset, there can appear unexpected empty intervals + } + + private fun intersect(ranges1: List, ranges2: List): List { + val offset1 = 10 + val offset2 = 100 + + fun size(ranges: List) = ranges.lastOrNull()?.last?.plus(1) ?: 0 + + val baseSize = maxOf(size(ranges1), size(ranges2)) + val sequence1 = MatchSequence( + mapOf( + TextLabel.BASE to ranges1, + TextLabel.PARENT_1 to ranges1.map { (it.first + offset1)..(it.last + offset1) }, + ), + mapOf( + TextLabel.BASE to CharacterSplitText("a".repeat(baseSize)), + TextLabel.PARENT_1 to CharacterSplitText("b".repeat(baseSize + offset1)), + ) + ) + val sequence2 = MatchSequence( + mapOf( + TextLabel.BASE to ranges2, + TextLabel.PARENT_2 to ranges2.map { (it.first + offset2)..(it.last + offset2) }, + ), + mapOf( + TextLabel.BASE to CharacterSplitText("a".repeat(baseSize)), + TextLabel.PARENT_2 to CharacterSplitText("b".repeat(baseSize + offset2)), + ) + ) + + val intersection = sequence1.intersectAlignments(sequence2) + + intersection.requireAlignment() + + val baseRanges = intersection.ranges(TextLabel.BASE) + + assertEquals(baseRanges, intersection.ranges(TextLabel.PARENT_1).map { maxOf(0, it.first - offset1)..(it.last - offset1) }) + assertEquals(baseRanges, intersection.ranges(TextLabel.PARENT_2).map { maxOf(0, it.first - offset2)..(it.last - offset2) }) + + return baseRanges + } + + private fun MatchSequence.ranges(label: TextLabel) = (0 until size).map { range(label, it) } +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/SplitTextTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/SplitTextTest.kt new file mode 100644 index 000000000000..deba81b183f9 --- /dev/null +++ b/plugins/evaluation-plugin/test/com/intellij/cce/evaluable/conflictResolution/SplitTextTest.kt @@ -0,0 +1,39 @@ +package com.intellij.cce.evaluable.conflictResolution + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class SplitTextTest { + + private val matrix1 = listOf( + listOf(0, 1, 2, 3), + listOf(1, 0, 1, 2), + listOf(2, 1, 0, 1), + listOf(3, 2, 1, 0), + ) + + private val matrix2 = listOf( + listOf(0, 1, 2, 3, 4, 5), + listOf(1, 2, 1, 2, 3, 4), + listOf(2, 3, 2, 3, 4, 5), + listOf(3, 4, 3, 4, 3, 4), + ) + + @Test + fun `test editDistanceMatrix`() { + assertEquals(matrix1, editDistance("abc", "abc")) + assertEquals(matrix2, editDistance("abc", "_a_c_")) + } + + @Test + fun `test retrieveAlignment`() { + assertEquals(Pair(listOf(0..2), listOf(0..2)), alignment(matrix1)) + assertEquals(Pair(listOf(0..0, 2..2), listOf(1..1, 3..3)), alignment(matrix2)) + } + + private fun editDistance(s1: String, s2: String) = + calculateEditDistance(CharacterSplitText(s1), CharacterSplitText(s2)).map { it.toList() } + + private fun alignment(matrix: List>) = + retrieveAlignment(matrix.map { it.toIntArray() }.toTypedArray()) +} \ No newline at end of file diff --git a/plugins/evaluation-plugin/test/com/intellij/cce/workspace/ConfigFactoryTest.kt b/plugins/evaluation-plugin/test/com/intellij/cce/workspace/ConfigFactoryTest.kt index bb540fe5e847..48eefb9ab47b 100644 --- a/plugins/evaluation-plugin/test/com/intellij/cce/workspace/ConfigFactoryTest.kt +++ b/plugins/evaluation-plugin/test/com/intellij/cce/workspace/ConfigFactoryTest.kt @@ -91,15 +91,15 @@ class ConfigFactoryTest { { "outputDir": "test_outputDir", "strategy": {}, - "fileDataset": { + "csvDataset": { "url": "test_url", "chunkSize": 300 } } """.trimIndent() ).also { - assertEquals("test_url", it.fileDataset?.url) - assertEquals(300, it.fileDataset?.chunkSize) + assertEquals("test_url", it.csvDataset?.url) + assertEquals(300, it.csvDataset?.chunkSize) } }