diff --git a/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/EAPUsageCollector.java b/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/EAPUsageCollector.java index cf510981eece..c3354854bc6f 100644 --- a/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/EAPUsageCollector.java +++ b/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/EAPUsageCollector.java @@ -17,7 +17,7 @@ import java.util.*; * @author Eugene Zhuravlev */ public final class EAPUsageCollector extends ApplicationUsagesCollector { - private static final EventLogGroup GROUP = new EventLogGroup("user.advanced.info", 5); + private static final EventLogGroup GROUP = new EventLogGroup("user.advanced.info", 6); private static final EventId1 BUILD = GROUP.registerEvent("build", EventFields.Enum("value", BuildType.class)); private static final EnumEventField LICENSE_VALUE = EventFields.Enum("value", LicenceType.class); private static final StringEventField METADATA = EventFields.StringValidatedByRegexpReference("metadata", "license_metadata"); diff --git a/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/FileEditorCollector.kt b/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/FileEditorCollector.kt index 9bca6d06b820..642321bffb52 100644 --- a/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/FileEditorCollector.kt +++ b/platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/FileEditorCollector.kt @@ -9,7 +9,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.vfs.VirtualFile object FileEditorCollector : CounterUsagesCollector() { - private val GROUP = EventLogGroup("file.editor", 5) + private val GROUP = EventLogGroup("file.editor", 6) private val FILE_EDITOR_FIELD = EventFields.Class("fileEditor") private val ALTERNATIVE_FILE_EDITOR_SELECTED = GROUP.registerVarargEvent("alternative.file.editor.selected", FILE_EDITOR_FIELD, diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/collectors/fus/fileTypes/FileTypeUsageCounterCollector.java b/platform/platform-impl/src/com/intellij/internal/statistic/collectors/fus/fileTypes/FileTypeUsageCounterCollector.java index 766e13bafcb5..cc479e1267a1 100644 --- a/platform/platform-impl/src/com/intellij/internal/statistic/collectors/fus/fileTypes/FileTypeUsageCounterCollector.java +++ b/platform/platform-impl/src/com/intellij/internal/statistic/collectors/fus/fileTypes/FileTypeUsageCounterCollector.java @@ -49,7 +49,7 @@ public final class FileTypeUsageCounterCollector extends CounterUsagesCollector private static final ExtensionPointName> EP = new ExtensionPointName<>("com.intellij.fileTypeUsageSchemaDescriptor"); - private static final EventLogGroup GROUP = new EventLogGroup("file.types.usage", 69); + private static final EventLogGroup GROUP = new EventLogGroup("file.types.usage", 70); private static final ClassEventField FILE_EDITOR = EventFields.Class("file_editor"); private static final EventField SCHEMA = EventFields.StringValidatedByCustomRule("schema", FileTypeSchemaValidator.class); diff --git a/platform/platform-tests/testData/fus/serialization/SerializationAnonymizedField.json b/platform/platform-tests/testData/fus/serialization/SerializationAnonymizedField.json new file mode 100644 index 000000000000..6327d20e3e10 --- /dev/null +++ b/platform/platform-tests/testData/fus/serialization/SerializationAnonymizedField.json @@ -0,0 +1,38 @@ +{ + "commitHash": "commitHash", + "buildNumber": "buildNumber", + "scheme": [ + { + "id": "testId", + "type": "counter", + "version": 1, + "schema": [ + { + "event": "testEvent", + "fields": [ + { + "path": "test_id_array", + "value": [ + "{regexp#hash}" + ], + "shouldBeAnonymized": true, + "dataType": "ARRAY" + }, + { + "path": "test_id", + "value": [ + "{regexp#hash}" + ], + "shouldBeAnonymized": true + } + ] + } + ], + "className": "classNameTest", + "recorder": "recorderTest", + "plugin": { + "id": "pluginIdTest" + } + } + ] +} \ No newline at end of file diff --git a/platform/platform-tests/testSrc/com/intellij/internal/statistics/serialization/SerializationHelperTest.kt b/platform/platform-tests/testSrc/com/intellij/internal/statistics/serialization/SerializationHelperTest.kt index 11aef26f1183..b2dad45a08df 100644 --- a/platform/platform-tests/testSrc/com/intellij/internal/statistics/serialization/SerializationHelperTest.kt +++ b/platform/platform-tests/testSrc/com/intellij/internal/statistics/serialization/SerializationHelperTest.kt @@ -17,17 +17,14 @@ import com.jetbrains.fus.reporting.model.lion3.LogEventAction import com.jetbrains.fus.reporting.model.lion3.LogEventGroup import com.jetbrains.fus.reporting.model.metadata.EventGroupRemoteDescriptors import org.junit.jupiter.api.Assertions -import org.junit.jupiter.api.Test import java.io.BufferedReader import java.io.File import java.io.StringReader import java.io.StringWriter -@Suppress("JUnitMixedFramework") internal class SerializationHelperTest : BasePlatformTestCase() { private fun getTestDataRoot() = PlatformTestUtil.getPlatformTestDataPath() + "fus/serialization/" - @org.junit.Test fun testSerializationGroupRemoteRule() { val enums = mapOf("__event_id1" to setOf("loading.config.failed", "logs.send", "metadata.loaded"), "__event_id2" to setOf("metadata.updated", "metadata.load.failed", "metadata.update.failed")) @@ -54,7 +51,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @org.junit.Test fun testDeserializationGroupRemoteRule() { val validationRule = File(getTestDataRoot() + "SerializationGroupRemoteRule.json").readText(Charsets.UTF_8) @@ -64,7 +60,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(validationRule, serializationText) } - @org.junit.Test fun testSerializationGroupDescriptor() { val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}")) val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor)) @@ -76,7 +71,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @org.junit.Test fun testDeserializationGroupDescriptor() { val groupDescriptor = File(getTestDataRoot() + "SerializationGroupDescriptor.json").readText(Charsets.UTF_8) @@ -86,7 +80,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(groupDescriptor, serializationText) } - @org.junit.Test fun testSerializationFeatureUsageData() { val data = FeatureUsageData("FUS") data.addData("durationMs", 1) @@ -99,7 +92,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @Test fun testSerializationEventLogExternalSettings() { val eventLogMajorVersionBorders = EventLogMajorVersionBorders() eventLogMajorVersionBorders.from = "2019.2" @@ -126,7 +118,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @Test fun testDeserializationEventLogExternalSettings() { val config = File(getTestDataRoot() + "SerializationEventLogExternalSettings.json").readText(Charsets.UTF_8) val reader = BufferedReader(StringReader(config)) @@ -136,7 +127,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(serializationText, config) } - @Test fun testSerializationEventGroupRemoteDescriptors() { val eventGroupRemoteDescriptors = EventGroupRemoteDescriptors() @@ -172,7 +162,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText.toString()) } - @Test fun testDeserializationEventGroupRemoteDescriptors() { val eventGroupRemoteDescriptors = File(getTestDataRoot() + "SerializationEventGroupRemoteDescriptors.json").readText(Charsets.UTF_8) @@ -182,7 +171,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(eventGroupRemoteDescriptors, serializationText) } - @Test fun testCustomSerializationLogEvent() { val logEvent = LogEvent("session", "build", "bucket", 1L, LogEventGroup("id", "version"), "recorderVersion", LogEventAction("id", true, mutableMapOf("string" to "any"), 2)) @@ -192,7 +180,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(serializationText, realText) } - @Test fun testCustomDeserializationLogEvent() { val logEvent = File(getTestDataRoot() + "CustomSerializationLogEvent.json").readText(Charsets.UTF_8) @@ -202,7 +189,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(logEvent, serializationText) } - @Test fun testSerializationLogEventRecordRequest() { val logEvent = LogEvent("session", "build", "bucket", 1L, LogEventGroup("id", "version"), "recorderVersion", LogEventAction("id", true, mutableMapOf("string" to "any"), 2)) @@ -215,7 +201,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(serializationText, realText) } - @Test fun testDeserializationLogEventRecordRequest() { val logEventRecordRequest = File(getTestDataRoot() + "SerializationLogEventRecordRequest.json").readText(Charsets.UTF_8) @@ -225,7 +210,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(logEventRecordRequest, serializationText) } - @Test fun testSerializationEventsSchemePrimitive() { val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}")) val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor)) @@ -239,7 +223,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @Test fun testDeserializationEventsSchemePrimitive() { val eventsScheme = File(getTestDataRoot() + "SerializationEventsSchemePrimitive.json").readText(Charsets.UTF_8) @@ -249,9 +232,8 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(eventsScheme, serializationText) } - @Test fun testSerializationEventsSchemeArray() { - val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"), FieldDataType.ARRAY) + val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"), dataType = FieldDataType.ARRAY) val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor)) val groupDescriptor = GroupDescriptor("testId", "counter", 1, setOf(eventSchemeDescriptor, eventSchemeDescriptor), "classNameTest", "recorderTest", PluginSchemeDescriptor("pluginIdTest")) @@ -263,7 +245,20 @@ internal class SerializationHelperTest : BasePlatformTestCase() { Assertions.assertEquals(realText, serializationText) } - @Test + fun testSerializationAnonymizedField() { + val anonymizedField = FieldDescriptor("test_id", setOf("{regexp#hash}"), shouldBeAnonymized = true) + val anonymizedArray = FieldDescriptor("test_id_array", setOf("{regexp#hash}"), shouldBeAnonymized = true, dataType = FieldDataType.ARRAY) + val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(anonymizedArray, anonymizedField)) + val groupDescriptor = GroupDescriptor("testId", "counter", 1, setOf(eventSchemeDescriptor), + "classNameTest", "recorderTest", PluginSchemeDescriptor("pluginIdTest")) + val eventsScheme = EventsScheme("commitHash", "buildNumber", listOf(groupDescriptor)) + + val serializationText = SerializationHelper.serialize(eventsScheme) + val realText = File(getTestDataRoot() + "SerializationAnonymizedField.json").readText(Charsets.UTF_8) + + Assertions.assertEquals(realText, serializationText) + } + fun testDeserializationEventsSchemeArray() { val eventsScheme = File(getTestDataRoot() + "SerializationEventsSchemeArray.json").readText(Charsets.UTF_8) diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/BaseEventFields.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/BaseEventFields.kt index 01cd6eb7e843..44b54adc3d79 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/BaseEventFields.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/BaseEventFields.kt @@ -251,6 +251,15 @@ data class AnonymizedEventField(@NonNls @EventFieldName override val name: Strin } } +data class AnonymizedListEventField(@NonNls @EventFieldName override val name: String) : StringListEventField(name) { + override val shouldBeAnonymized: Boolean + get() = true + + override val validationRule: List + get() = listOf("{regexp#hash}") +} + + internal data class ShortAnonymizedEventField(@NonNls @EventFieldName override val name: String) : PrimitiveEventField() { override val shouldBeAnonymized: Boolean = true diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/EventFields.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/EventFields.kt index 78a326e03c91..72896ddddc0f 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/EventFields.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/EventFields.kt @@ -479,6 +479,12 @@ object EventFields { @JvmStatic fun AnonymizedField(@NonNls @EventFieldName name: String): EventField = AnonymizedEventField(name) + /** + * Can be used to report unique identifiers safely by anonymizing them using hash function and local salt + * */ + @JvmStatic + fun AnonymizedList(@NonNls @EventFieldName name: String): AnonymizedListEventField = AnonymizedListEventField(name) + /** * Can be used to report unique identifiers safely by anonymizing them using hash function and local salt * diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsScheme.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsScheme.kt index c5ab5cdd3d17..d9d34065d89c 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsScheme.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsScheme.kt @@ -16,8 +16,11 @@ class FieldDataTypeIncludeFilter { return other is FieldDataType && FieldDataType.PRIMITIVE == other } } + data class FieldDescriptor(val path: String, val value: Set, + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + val shouldBeAnonymized: Boolean = false, @JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = FieldDataTypeIncludeFilter::class) val dataType: FieldDataType = FieldDataType.PRIMITIVE) diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsSchemeBuilder.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsSchemeBuilder.kt index d3c7a9f0f4d8..89c22670c837 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsSchemeBuilder.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsSchemeBuilder.kt @@ -11,9 +11,9 @@ import java.util.regex.Pattern object EventsSchemeBuilder { val pluginInfoFields = setOf( - FieldDescriptor("plugin", setOf("{util#plugin}")), - FieldDescriptor("plugin_type", setOf("{util#plugin_type}")), - FieldDescriptor("plugin_version", setOf("{util#plugin_version}")) + FieldDescriptor("plugin", setOf("{util#plugin}"), false), + FieldDescriptor("plugin_type", setOf("{util#plugin_type}"), false), + FieldDescriptor("plugin_version", setOf("{util#plugin_version}"), false) ) private val classValidationRuleNames = setOf("class_name", "dialog_class", "quick_fix_class_name", @@ -38,7 +38,7 @@ object EventsSchemeBuilder { if (field is StringListEventField.ValidatedByInlineRegexp) { validateRegexp(field.regexp) } - buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.ARRAY) + buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.ARRAY, field.shouldBeAnonymized) } is PrimitiveEventField -> { if (field is StringEventField.ValidatedByInlineRegexp) { @@ -48,13 +48,16 @@ object EventsSchemeBuilder { validateRegexp(field.regexp) } - buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.PRIMITIVE) + buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.PRIMITIVE, field.shouldBeAnonymized) } } } - private fun buildFieldDescriptors(fieldName: String, validationRules: List, fieldDataType: FieldDataType): Set { - val fields = mutableSetOf(FieldDescriptor(fieldName, validationRules.toSet(), fieldDataType)) + private fun buildFieldDescriptors(fieldName: String, + validationRules: List, + fieldDataType: FieldDataType, + shouldBeAnonymized: Boolean): Set { + val fields = mutableSetOf(FieldDescriptor(fieldName, validationRules.toSet(), shouldBeAnonymized, fieldDataType)) if (validationRules.any { it in classValidationRules }) { fields.addAll(pluginInfoFields) } @@ -163,11 +166,21 @@ object EventsSchemeBuilder { .groupBy { it.path } .map { (name, values) -> val type = defineDataType(values, name, eventName, groupId) - FieldDescriptor(name, values.flatMap { it.value }.toSet(), type) + val shouldBeAnonymized = defineShouldBeAnonymized(values, name, eventName, groupId) + FieldDescriptor(name, values.flatMap { it.value }.toSet(), shouldBeAnonymized, type) } .toSet() } + private fun defineShouldBeAnonymized(values: List, name: String, eventName: String, groupId: String): Boolean { + val shouldBeAnonymized = values.first().shouldBeAnonymized + return if (values.any { it.shouldBeAnonymized != shouldBeAnonymized }) + throw IllegalMetadataSchemeStateException("Field couldn't be defined twice with different shouldBeAnonymized value (group=$groupId, event=$eventName, field=$name)") + else { + shouldBeAnonymized + } + } + private fun defineDataType(values: List, name: String, eventName: String, groupId: String): FieldDataType { val dataType = values.first().dataType return if (values.any { it.dataType != dataType }) diff --git a/platform/vcs-log/impl/src/com/intellij/vcs/log/statistics/VcsLogIndexStatisticsCollectors.kt b/platform/vcs-log/impl/src/com/intellij/vcs/log/statistics/VcsLogIndexStatisticsCollectors.kt index 2964fdc59b61..9b03869584f3 100644 --- a/platform/vcs-log/impl/src/com/intellij/vcs/log/statistics/VcsLogIndexStatisticsCollectors.kt +++ b/platform/vcs-log/impl/src/com/intellij/vcs/log/statistics/VcsLogIndexStatisticsCollectors.kt @@ -52,7 +52,7 @@ internal class VcsLogIndexApplicationStatisticsCollector : ApplicationUsagesColl } internal class VcsLogIndexProjectStatisticsCollector : ProjectUsagesCollector() { - private val GROUP = EventLogGroup("vcs.log.index.project", 4) + private val GROUP = EventLogGroup("vcs.log.index.project", 5) private val INDEXING_TIME = GROUP.registerEvent("indexing.time.minutes", EventFields.Count) private val IS_PAUSED = EventFields.Boolean("is_paused") private val INDEXING_TIME_BY_ROOT = GROUP.registerEvent("indexing.time.by.root", diff --git a/plugins/changeReminder/src/com/jetbrains/changeReminder/stats/ChangeReminderStatsCollector.kt b/plugins/changeReminder/src/com/jetbrains/changeReminder/stats/ChangeReminderStatsCollector.kt index b2e02894ef8d..5b2a2762b2b6 100644 --- a/plugins/changeReminder/src/com/jetbrains/changeReminder/stats/ChangeReminderStatsCollector.kt +++ b/plugins/changeReminder/src/com/jetbrains/changeReminder/stats/ChangeReminderStatsCollector.kt @@ -40,12 +40,12 @@ internal fun getPredictionData(predictionData: PredictionData): List("empty_reason") { it.name.lowercase(Locale.ENGLISH) } diff --git a/plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt b/plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt index d77e9d1ef80b..be7fb7603d6d 100644 --- a/plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt +++ b/plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt @@ -9,7 +9,7 @@ import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesColle import com.intellij.openapi.project.Project internal object FileNavigationLogger : CounterUsagesCollector() { - private val GROUP = EventLogGroup("file.prediction", 14) + private val GROUP = EventLogGroup("file.prediction", 15) private var session: IntEventField = EventFields.Int("session") private var performance: LongListEventField = EventFields.LongList("performance") diff --git a/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt b/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt index bd5518a8bd21..72c03dce4d4a 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt @@ -23,7 +23,7 @@ import org.jetbrains.plugins.github.util.GHEnterpriseServerMetadataLoader import java.util.* internal object GHPRStatisticsCollector: CounterUsagesCollector() { - private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 6) + private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 7) override fun getGroup() = COUNTERS_GROUP diff --git a/plugins/kotlin/base/statistics/src/org/jetbrains/kotlin/idea/statistics/KotlinLanguageFeaturesFUSCollector.kt b/plugins/kotlin/base/statistics/src/org/jetbrains/kotlin/idea/statistics/KotlinLanguageFeaturesFUSCollector.kt index 9ea53bb8c42d..19187f975ba9 100644 --- a/plugins/kotlin/base/statistics/src/org/jetbrains/kotlin/idea/statistics/KotlinLanguageFeaturesFUSCollector.kt +++ b/plugins/kotlin/base/statistics/src/org/jetbrains/kotlin/idea/statistics/KotlinLanguageFeaturesFUSCollector.kt @@ -17,7 +17,7 @@ object KotlinLanguageFeaturesFUSCollector : CounterUsagesCollector() { override fun getGroup(): EventLogGroup = GROUP // Collector ID - private val GROUP = EventLogGroup("kotlin.ide.inspections", 2) + private val GROUP = EventLogGroup("kotlin.ide.inspections", 3) val inspectionTypeField = EventFields.Enum("inspection_type") { it.name.lowercase() } val kotlinLanguageVersionField = EventFields.StringValidatedByRegexpReference("kotlin_language_version", "version_lang_api")