MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2025-12-15 02:59:33 +07:00
Code Issues Packages Projects Releases Wiki Activity

[fus] AP-5930 add shouldBeAnonymized to generated scheme, bump group versions

Browse Source 
GitOrigin-RevId: e628599f14cf152640623f84837cd26bd5db7572
This commit is contained in:
Anastasia.Ivanova
2024-01-29 13:57:20 +01:00
committed by intellij-monorepo-bot
parent ddeefd7ffd
commit fe01da593f
14 changed files with 105 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/EAPUsageCollector.java
View File

@@ -17,7 +17,7 @@ import java.util.*;
* @author Eugene Zhuravlev
*/
public final class EAPUsageCollector extends ApplicationUsagesCollector {
private static final EventLogGroup GROUP = new EventLogGroup("user.advanced.info", 5);
private static final EventLogGroup GROUP = new EventLogGroup("user.advanced.info", 6);
private static final EventId1<BuildType> BUILD = GROUP.registerEvent("build", EventFields.Enum("value", BuildType.class));
private static final EnumEventField<LicenceType> LICENSE_VALUE = EventFields.Enum("value", LicenceType.class);
private static final StringEventField METADATA = EventFields.StringValidatedByRegexpReference("metadata", "license_metadata");

2
platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/FileEditorCollector.kt
View File

@@ -9,7 +9,7 @@ import com.intellij.openapi.project.Project
import com.intellij.openapi.vfs.VirtualFile
object FileEditorCollector : CounterUsagesCollector() {
private val GROUP = EventLogGroup("file.editor", 5)
private val GROUP = EventLogGroup("file.editor", 6)
private val FILE_EDITOR_FIELD = EventFields.Class("fileEditor")
private val ALTERNATIVE_FILE_EDITOR_SELECTED = GROUP.registerVarargEvent("alternative.file.editor.selected",
FILE_EDITOR_FIELD,

2
platform/platform-impl/src/com/intellij/internal/statistic/collectors/fus/fileTypes/FileTypeUsageCounterCollector.java
View File

@@ -49,7 +49,7 @@ public final class FileTypeUsageCounterCollector extends CounterUsagesCollector
private static final ExtensionPointName<FileTypeUsageSchemaDescriptorEP<FileTypeUsageSchemaDescriptor>> EP =
new ExtensionPointName<>("com.intellij.fileTypeUsageSchemaDescriptor");
private static final EventLogGroup GROUP = new EventLogGroup("file.types.usage", 69);
private static final EventLogGroup GROUP = new EventLogGroup("file.types.usage", 70);
private static final ClassEventField FILE_EDITOR = EventFields.Class("file_editor");
private static final EventField<String> SCHEMA = EventFields.StringValidatedByCustomRule("schema", FileTypeSchemaValidator.class);

38
platform/platform-tests/testData/fus/serialization/SerializationAnonymizedField.json Normal file
View File

@@ -0,0 +1,38 @@
{
"commitHash": "commitHash",
"buildNumber": "buildNumber",
"scheme": [
{
"id": "testId",
"type": "counter",
"version": 1,
"schema": [
{
"event": "testEvent",
"fields": [
{
"path": "test_id_array",
"value": [
"{regexp#hash}"
],
"shouldBeAnonymized": true,
"dataType": "ARRAY"
},
{
"path": "test_id",
"value": [
"{regexp#hash}"
],
"shouldBeAnonymized": true
}
]
}
],
"className": "classNameTest",
"recorder": "recorderTest",
"plugin": {
"id": "pluginIdTest"
}
}
]
}

35
platform/platform-tests/testSrc/com/intellij/internal/statistics/serialization/SerializationHelperTest.kt
View File

@@ -17,17 +17,14 @@ import com.jetbrains.fus.reporting.model.lion3.LogEventAction
import com.jetbrains.fus.reporting.model.lion3.LogEventGroup
import com.jetbrains.fus.reporting.model.metadata.EventGroupRemoteDescriptors
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.Test
import java.io.BufferedReader
import java.io.File
import java.io.StringReader
import java.io.StringWriter
@Suppress("JUnitMixedFramework")
internal class SerializationHelperTest : BasePlatformTestCase() {
private fun getTestDataRoot() = PlatformTestUtil.getPlatformTestDataPath() + "fus/serialization/"
@org.junit.Test
fun testSerializationGroupRemoteRule() {
val enums = mapOf("__event_id1" to setOf("loading.config.failed", "logs.send", "metadata.loaded"),
"__event_id2" to setOf("metadata.updated", "metadata.load.failed", "metadata.update.failed"))
@@ -54,7 +51,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@org.junit.Test
fun testDeserializationGroupRemoteRule() {
val validationRule = File(getTestDataRoot() + "SerializationGroupRemoteRule.json").readText(Charsets.UTF_8)
@@ -64,7 +60,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(validationRule, serializationText)
}
@org.junit.Test
fun testSerializationGroupDescriptor() {
val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"))
val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor))
@@ -76,7 +71,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@org.junit.Test
fun testDeserializationGroupDescriptor() {
val groupDescriptor = File(getTestDataRoot() + "SerializationGroupDescriptor.json").readText(Charsets.UTF_8)
@@ -86,7 +80,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(groupDescriptor, serializationText)
}
@org.junit.Test
fun testSerializationFeatureUsageData() {
val data = FeatureUsageData("FUS")
data.addData("durationMs", 1)
@@ -99,7 +92,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@Test
fun testSerializationEventLogExternalSettings() {
val eventLogMajorVersionBorders = EventLogMajorVersionBorders()
eventLogMajorVersionBorders.from = "2019.2"
@@ -126,7 +118,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@Test
fun testDeserializationEventLogExternalSettings() {
val config = File(getTestDataRoot() + "SerializationEventLogExternalSettings.json").readText(Charsets.UTF_8)
val reader = BufferedReader(StringReader(config))
@@ -136,7 +127,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(serializationText, config)
}
@Test
fun testSerializationEventGroupRemoteDescriptors() {
val eventGroupRemoteDescriptors = EventGroupRemoteDescriptors()
@@ -172,7 +162,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText.toString())
}
@Test
fun testDeserializationEventGroupRemoteDescriptors() {
val eventGroupRemoteDescriptors = File(getTestDataRoot() + "SerializationEventGroupRemoteDescriptors.json").readText(Charsets.UTF_8)
@@ -182,7 +171,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(eventGroupRemoteDescriptors, serializationText)
}
@Test
fun testCustomSerializationLogEvent() {
val logEvent = LogEvent("session", "build", "bucket", 1L, LogEventGroup("id", "version"),
"recorderVersion", LogEventAction("id", true, mutableMapOf("string" to "any"), 2))
@@ -192,7 +180,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(serializationText, realText)
}
@Test
fun testCustomDeserializationLogEvent() {
val logEvent = File(getTestDataRoot() + "CustomSerializationLogEvent.json").readText(Charsets.UTF_8)
@@ -202,7 +189,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(logEvent, serializationText)
}
@Test
fun testSerializationLogEventRecordRequest() {
val logEvent = LogEvent("session", "build", "bucket", 1L, LogEventGroup("id", "version"),
"recorderVersion", LogEventAction("id", true, mutableMapOf("string" to "any"), 2))
@@ -215,7 +201,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(serializationText, realText)
}
@Test
fun testDeserializationLogEventRecordRequest() {
val logEventRecordRequest = File(getTestDataRoot() + "SerializationLogEventRecordRequest.json").readText(Charsets.UTF_8)
@@ -225,7 +210,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(logEventRecordRequest, serializationText)
}
@Test
fun testSerializationEventsSchemePrimitive() {
val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"))
val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor))
@@ -239,7 +223,6 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@Test
fun testDeserializationEventsSchemePrimitive() {
val eventsScheme = File(getTestDataRoot() + "SerializationEventsSchemePrimitive.json").readText(Charsets.UTF_8)
@@ -249,9 +232,8 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(eventsScheme, serializationText)
}
@Test
fun testSerializationEventsSchemeArray() {
val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"), FieldDataType.ARRAY)
val filedDescriptor = FieldDescriptor("plugin", setOf("{util#class_name}", "{util#plugin}"), dataType = FieldDataType.ARRAY)
val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(filedDescriptor, filedDescriptor))
val groupDescriptor = GroupDescriptor("testId", "counter", 1, setOf(eventSchemeDescriptor, eventSchemeDescriptor),
"classNameTest", "recorderTest", PluginSchemeDescriptor("pluginIdTest"))
@@ -263,7 +245,20 @@ internal class SerializationHelperTest : BasePlatformTestCase() {
Assertions.assertEquals(realText, serializationText)
}
@Test
fun testSerializationAnonymizedField() {
val anonymizedField = FieldDescriptor("test_id", setOf("{regexp#hash}"), shouldBeAnonymized = true)
val anonymizedArray = FieldDescriptor("test_id_array", setOf("{regexp#hash}"), shouldBeAnonymized = true, dataType = FieldDataType.ARRAY)
val eventSchemeDescriptor = EventDescriptor("testEvent", setOf(anonymizedArray, anonymizedField))
val groupDescriptor = GroupDescriptor("testId", "counter", 1, setOf(eventSchemeDescriptor),
"classNameTest", "recorderTest", PluginSchemeDescriptor("pluginIdTest"))
val eventsScheme = EventsScheme("commitHash", "buildNumber", listOf(groupDescriptor))
val serializationText = SerializationHelper.serialize(eventsScheme)
val realText = File(getTestDataRoot() + "SerializationAnonymizedField.json").readText(Charsets.UTF_8)
Assertions.assertEquals(realText, serializationText)
}
fun testDeserializationEventsSchemeArray() {
val eventsScheme = File(getTestDataRoot() + "SerializationEventsSchemeArray.json").readText(Charsets.UTF_8)

9
platform/statistics/src/com/intellij/internal/statistic/eventLog/events/BaseEventFields.kt
View File

@@ -251,6 +251,15 @@ data class AnonymizedEventField(@NonNls @EventFieldName override val name: Strin
}
}
data class AnonymizedListEventField(@NonNls @EventFieldName override val name: String) : StringListEventField(name) {
override val shouldBeAnonymized: Boolean
get() = true
override val validationRule: List<String>
get() = listOf("{regexp#hash}")
}
internal data class ShortAnonymizedEventField(@NonNls @EventFieldName override val name: String) : PrimitiveEventField<String?>() {
override val shouldBeAnonymized: Boolean = true

6
platform/statistics/src/com/intellij/internal/statistic/eventLog/events/EventFields.kt
View File

@@ -479,6 +479,12 @@ object EventFields {
@JvmStatic
fun AnonymizedField(@NonNls @EventFieldName name: String): EventField<String?> = AnonymizedEventField(name)
/**
* Can be used to report unique identifiers safely by anonymizing them using hash function and local salt
* */
@JvmStatic
fun AnonymizedList(@NonNls @EventFieldName name: String): AnonymizedListEventField = AnonymizedListEventField(name)
/**
* Can be used to report unique identifiers safely by anonymizing them using hash function and local salt
*

3
platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsScheme.kt
View File

@@ -16,8 +16,11 @@ class FieldDataTypeIncludeFilter {
return other is FieldDataType && FieldDataType.PRIMITIVE == other
}
}
data class FieldDescriptor(val path: String,
val value: Set<String>,
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
val shouldBeAnonymized: Boolean = false,
@JsonInclude(value = JsonInclude.Include.CUSTOM, valueFilter = FieldDataTypeIncludeFilter::class)
val dataType: FieldDataType = FieldDataType.PRIMITIVE)

29
platform/statistics/src/com/intellij/internal/statistic/eventLog/events/scheme/EventsSchemeBuilder.kt
View File

@@ -11,9 +11,9 @@ import java.util.regex.Pattern
object EventsSchemeBuilder {
val pluginInfoFields = setOf(
FieldDescriptor("plugin", setOf("{util#plugin}")),
FieldDescriptor("plugin_type", setOf("{util#plugin_type}")),
FieldDescriptor("plugin_version", setOf("{util#plugin_version}"))
FieldDescriptor("plugin", setOf("{util#plugin}"), false),
FieldDescriptor("plugin_type", setOf("{util#plugin_type}"), false),
FieldDescriptor("plugin_version", setOf("{util#plugin_version}"), false)
)
private val classValidationRuleNames = setOf("class_name", "dialog_class", "quick_fix_class_name",
@@ -38,7 +38,7 @@ object EventsSchemeBuilder {
if (field is StringListEventField.ValidatedByInlineRegexp) {
validateRegexp(field.regexp)
}
buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.ARRAY)
buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.ARRAY, field.shouldBeAnonymized)
}
is PrimitiveEventField -> {
if (field is StringEventField.ValidatedByInlineRegexp) {
@@ -48,13 +48,16 @@ object EventsSchemeBuilder {
validateRegexp(field.regexp)
}
buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.PRIMITIVE)
buildFieldDescriptors(fieldName, field.validationRule, FieldDataType.PRIMITIVE, field.shouldBeAnonymized)
}
}
}
private fun buildFieldDescriptors(fieldName: String, validationRules: List<String>, fieldDataType: FieldDataType): Set<FieldDescriptor> {
val fields = mutableSetOf(FieldDescriptor(fieldName, validationRules.toSet(), fieldDataType))
private fun buildFieldDescriptors(fieldName: String,
validationRules: List<String>,
fieldDataType: FieldDataType,
shouldBeAnonymized: Boolean): Set<FieldDescriptor> {
val fields = mutableSetOf(FieldDescriptor(fieldName, validationRules.toSet(), shouldBeAnonymized, fieldDataType))
if (validationRules.any { it in classValidationRules }) {
fields.addAll(pluginInfoFields)
}
@@ -163,11 +166,21 @@ object EventsSchemeBuilder {
.groupBy { it.path }
.map { (name, values) ->
val type = defineDataType(values, name, eventName, groupId)
FieldDescriptor(name, values.flatMap { it.value }.toSet(), type)
val shouldBeAnonymized = defineShouldBeAnonymized(values, name, eventName, groupId)
FieldDescriptor(name, values.flatMap { it.value }.toSet(), shouldBeAnonymized, type)
}
.toSet()
}
private fun defineShouldBeAnonymized(values: List<FieldDescriptor>, name: String, eventName: String, groupId: String): Boolean {
val shouldBeAnonymized = values.first().shouldBeAnonymized
return if (values.any { it.shouldBeAnonymized != shouldBeAnonymized })
throw IllegalMetadataSchemeStateException("Field couldn't be defined twice with different shouldBeAnonymized value (group=$groupId, event=$eventName, field=$name)")
else {
shouldBeAnonymized
}
}
private fun defineDataType(values: List<FieldDescriptor>, name: String, eventName: String, groupId: String): FieldDataType {
val dataType = values.first().dataType
return if (values.any { it.dataType != dataType })

2
platform/vcs-log/impl/src/com/intellij/vcs/log/statistics/VcsLogIndexStatisticsCollectors.kt
View File

@@ -52,7 +52,7 @@ internal class VcsLogIndexApplicationStatisticsCollector : ApplicationUsagesColl
}
internal class VcsLogIndexProjectStatisticsCollector : ProjectUsagesCollector() {
private val GROUP = EventLogGroup("vcs.log.index.project", 4)
private val GROUP = EventLogGroup("vcs.log.index.project", 5)
private val INDEXING_TIME = GROUP.registerEvent("indexing.time.minutes", EventFields.Count)
private val IS_PAUSED = EventFields.Boolean("is_paused")
private val INDEXING_TIME_BY_ROOT = GROUP.registerEvent("indexing.time.by.root",

12
plugins/changeReminder/src/com/jetbrains/changeReminder/stats/ChangeReminderStatsCollector.kt
View File

@@ -40,12 +40,12 @@ internal fun getPredictionData(predictionData: PredictionData): List<EventPair<*
object ChangeReminderStatsCollector : CounterUsagesCollector() {
override fun getGroup(): EventLogGroup = GROUP
internal val GROUP = EventLogGroup("vcs.change.reminder", 3)
internal val COMMITTED_FILES = EventFields.StringListValidatedByRegexp("committed_files", "hash")
internal val DISPLAYED_PREDICTION = EventFields.StringListValidatedByRegexp("displayed_prediction", "hash")
internal val CUR_MODIFIED_FILES = EventFields.StringListValidatedByRegexp("cur_modified_files", "hash")
internal val PREV_MODIFIED_FILES = EventFields.StringListValidatedByRegexp("prev_modified_files", "hash")
internal val PREDICTION_FOR_FILES = EventFields.StringListValidatedByRegexp("prediction_for_files", "hash")
internal val GROUP = EventLogGroup("vcs.change.reminder", 4)
internal val COMMITTED_FILES = EventFields.AnonymizedList("committed_files")
internal val DISPLAYED_PREDICTION = EventFields.AnonymizedList("displayed_prediction")
internal val CUR_MODIFIED_FILES = EventFields.AnonymizedList("cur_modified_files")
internal val PREV_MODIFIED_FILES = EventFields.AnonymizedList("prev_modified_files")
internal val PREDICTION_FOR_FILES = EventFields.AnonymizedList("prediction_for_files")
internal val EMPTY_REASON = EventFields.Enum<PredictionData.EmptyPredictionReason>("empty_reason") {
it.name.lowercase(Locale.ENGLISH)
}

2
plugins/filePrediction/src/com/intellij/filePrediction/logger/FileNavigationLogger.kt
View File

@@ -9,7 +9,7 @@ import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesColle
import com.intellij.openapi.project.Project
internal object FileNavigationLogger : CounterUsagesCollector() {
private val GROUP = EventLogGroup("file.prediction", 14)
private val GROUP = EventLogGroup("file.prediction", 15)
private var session: IntEventField = EventFields.Int("session")
private var performance: LongListEventField = EventFields.LongList("performance")

2
plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt
View File

@@ -23,7 +23,7 @@ import org.jetbrains.plugins.github.util.GHEnterpriseServerMetadataLoader
import java.util.*
internal object GHPRStatisticsCollector: CounterUsagesCollector() {
private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 6)
private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 7)
override fun getGroup() = COUNTERS_GROUP

2
plugins/kotlin/base/statistics/src/org/jetbrains/kotlin/idea/statistics/KotlinLanguageFeaturesFUSCollector.kt
View File

@@ -17,7 +17,7 @@ object KotlinLanguageFeaturesFUSCollector : CounterUsagesCollector() {
override fun getGroup(): EventLogGroup = GROUP
// Collector ID
private val GROUP = EventLogGroup("kotlin.ide.inspections", 2)
private val GROUP = EventLogGroup("kotlin.ide.inspections", 3)
val inspectionTypeField = EventFields.Enum<KotlinLanguageFeatureInspectionType>("inspection_type") { it.name.lowercase() }
val kotlinLanguageVersionField = EventFields.StringValidatedByRegexpReference("kotlin_language_version", "version_lang_api")