From 9be68d5046d51274df03097d13da524772bb656d Mon Sep 17 00:00:00 2001
From: Sergey Patrikeev <Sergey.Patrikeev@jetbrains.com>
Date: Sat, 2 May 2020 19:14:58 +0300
Subject: [PATCH] Indexing time diagnostic: count statistic values
 (min/max/mean/median), use millis instead of nano in presentations.

GitOrigin-RevId: 46a659987dfebe648aac0bc2275230d6fcfc2017
---
 .../diagnostic/IndexDiagnosticJson.kt         | 127 ++++++++++++------
 .../indexing/diagnostic/MaxNTimeBucket.kt     |  38 ++++++
 .../util/indexing/diagnostic/PerThreadTime.kt |  12 +-
 3 files changed, 130 insertions(+), 47 deletions(-)
 create mode 100644 platform/lang-impl/src/com/intellij/util/indexing/diagnostic/MaxNTimeBucket.kt

diff --git a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/IndexDiagnosticJson.kt b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/IndexDiagnosticJson.kt
index d0bd7d94927c..cdd4ae5a7ec5 100644
--- a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/IndexDiagnosticJson.kt
+++ b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/IndexDiagnosticJson.kt
@@ -8,37 +8,70 @@ import java.time.ZoneId
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 
-data class JsonPerThreadTime(
-  val perThreadTimes: List<TimeNano>,
-  val totalCpuTime: TimeNano
+private fun TimeNano.toMillis(): TimeMillis = this / 1_000_000
+
+data class JsonPerThreadTimeStats(
+  val minTime: TimeMillis,
+  val maxTime: TimeMillis,
+  val meanTime: TimeMillis,
+  val medianTime: TimeMillis
 )
 
-fun PerThreadTime.toJsonPerThreadTime(): JsonPerThreadTime =
-  JsonPerThreadTime(
-    threadIdToTime.values.sortedDescending(),
-    threadIdToTime.values.sum()
+fun PerThreadTime.toJsonPerThreadTime(): JsonPerThreadTimeStats {
+  val timeBuckets = threadIdToTimeBucket.values
+  if (timeBuckets.isEmpty()) {
+    return JsonPerThreadTimeStats(-1, -1, -1, -1)
+  }
+
+  val median = getMedianOfArray(timeBuckets.flatMap { it.maxNTimes })
+  val min = timeBuckets.map { it.minTime }.min()!!
+  val max = timeBuckets.map { it.maxTime }.max()!!
+  val mean = getMeanOfArray(timeBuckets.map { it.meanTime })
+
+  return JsonPerThreadTimeStats(
+    min.toMillis(),
+    max.toMillis(),
+    mean.toLong().toMillis(),
+    median.toLong().toMillis()
   )
+}
+
+private fun <N: Number> getMeanOfArray(elements: Collection<N>): Double {
+  require(elements.isNotEmpty())
+  return elements.map { it.toDouble() }.sum() / elements.size
+}
+
+private fun <N: Number> getMedianOfArray(elements: Collection<N>): Double {
+  require(elements.isNotEmpty())
+  val sorted = elements.map { it.toDouble() }.sorted()
+  return if (sorted.size % 2 == 0) {
+    (sorted[sorted.size / 2] + sorted[sorted.size / 2 - 1]) / 2.0
+  }
+  else {
+    sorted[sorted.size / 2]
+  }
+}
 
 data class JsonFileProviderIndexStatistics(
-  val indexableFilesProviderDebugName: String,
+  val providerName: String,
   // <total time> = <content loading time> + <indexing time> + <time spent on waiting for other indexing tasks to complete>
-  val totalTime: TimeNano,
-  val indexingTime: JsonPerThreadTime,
-  val contentLoadingTime: JsonPerThreadTime,
+  val totalTime: TimeMillis,
+  val indexingTimePerFile: JsonPerThreadTimeStats,
+  val contentLoadingTimePerFile: JsonPerThreadTimeStats,
   val numberOfFilesPerFileType: List<FilesNumberPerFileType>,
   val timesPerFileType: List<TimePerFileType>,
   val timesPerIndexer: List<TimePerIndexer>
 ) {
 
-  data class TimePerIndexer(val indexId: String, val time: JsonPerThreadTime)
-  data class TimePerFileType(val fileType: String, val time: JsonPerThreadTime)
+  data class TimePerIndexer(val indexId: String, val time: JsonPerThreadTimeStats)
+  data class TimePerFileType(val fileType: String, val time: JsonPerThreadTimeStats)
   data class FilesNumberPerFileType(val fileType: String, val filesNumber: Int)
 }
 
 fun FileProviderIndexStatistics.convertToJson(): JsonFileProviderIndexStatistics =
   JsonFileProviderIndexStatistics(
     providerDebugName,
-    totalTime,
+    totalTime.toMillis(),
     indexingStatistics.indexingTime.toJsonPerThreadTime(),
     indexingStatistics.contentLoadingTime.toJsonPerThreadTime(),
     indexingStatistics.numberOfFilesPerFileType
@@ -47,50 +80,58 @@ fun FileProviderIndexStatistics.convertToJson(): JsonFileProviderIndexStatistics
     ,
     indexingStatistics.timesPerFileType
       .map { JsonFileProviderIndexStatistics.TimePerFileType(it.key, it.value.toJsonPerThreadTime()) }
-      .sortedByDescending { it.time.totalCpuTime }
+      .sortedByDescending { it.time.meanTime }
     ,
     indexingStatistics.timesPerIndexer
       .map { JsonFileProviderIndexStatistics.TimePerIndexer(it.key, it.value.toJsonPerThreadTime()) }
-      .sortedByDescending { it.time.totalCpuTime }
+      .sortedByDescending { it.time.meanTime }
   )
 
+typealias PresentableTime = String
+
+private fun TimeMillis.toPresentableTime(): PresentableTime =
+  DateFormatUtil.getIso8601Format().format(this)
+
 @Suppress("unused", "used for JSON")
 data class JsonProjectIndexingHistoryTimes(
-  val startIndexing: TimeMillis,
-  val endIndexing: TimeMillis,
+  val startIndexing: PresentableTime,
+  val endIndexing: PresentableTime,
+  val indexingTime: TimeMillis,
 
-  val startPushProperties: TimeMillis,
-  val endPushProperties: TimeMillis,
+  val startPushProperties: PresentableTime,
+  val endPushProperties: PresentableTime,
+  val pushPropertiesTime: TimeMillis,
 
-  val startIndexExtensions: TimeMillis,
-  val endIndexExtensions: TimeMillis,
+  val startIndexExtensions: PresentableTime,
+  val endIndexExtensions: PresentableTime,
+  val indexExtensionsTime: TimeMillis,
 
-  val startScanFiles: TimeMillis,
-  val endScanFiles: TimeMillis
-) {
-  val presentableStartIndexingTime: String get() = DateFormatUtil.formatTimeWithSeconds(startIndexing)
-  val indexingTime: TimeMillis get() = endIndexing - startIndexing
-  val pushPropertiesTime: TimeMillis get() = endPushProperties - startPushProperties
-  val indexExtensionsTime: TimeMillis get() = endIndexExtensions - startIndexExtensions
-  val scanFilesTime: TimeMillis get() = endScanFiles - startScanFiles
-}
+  val startScanFiles: PresentableTime,
+  val endScanFiles: PresentableTime,
+  val scanFilesTime: TimeMillis
+)
 
-fun ProjectIndexingHistory.IndexingTimes.convertToJson(): JsonProjectIndexingHistoryTimes =
-  JsonProjectIndexingHistoryTimes(
-    startIndexing,
-    endIndexing,
-    startPushProperties,
-    endPushProperties,
-    startIndexExtensions,
-    endIndexExtensions,
-    startScanFiles,
-    endScanFiles
+fun ProjectIndexingHistory.IndexingTimes.convertToJson(): JsonProjectIndexingHistoryTimes {
+  return JsonProjectIndexingHistoryTimes(
+    startIndexing.toPresentableTime(),
+    endIndexing.toPresentableTime(),
+    endIndexing - startIndexing,
+    startPushProperties.toPresentableTime(),
+    endPushProperties.toPresentableTime(),
+    endPushProperties - startPushProperties,
+    startIndexExtensions.toPresentableTime(),
+    endIndexExtensions.toPresentableTime(),
+    endIndexExtensions - startIndexExtensions,
+    startScanFiles.toPresentableTime(),
+    endScanFiles.toPresentableTime(),
+    endScanFiles - startScanFiles
   )
+}
 
 data class JsonProjectIndexingHistory(
   val projectName: String,
   val indexingTimes: JsonProjectIndexingHistoryTimes,
-  val providerStatistics: List<JsonFileProviderIndexStatistics>
+  val fileProviderStatistics: List<JsonFileProviderIndexStatistics>
 )
 
 fun ProjectIndexingHistory.convertToJson(): JsonProjectIndexingHistory {
@@ -99,7 +140,7 @@ fun ProjectIndexingHistory.convertToJson(): JsonProjectIndexingHistory {
     times.convertToJson(),
     providerStatistics
       .map { it.convertToJson() }
-      .sortedByDescending { it.indexingTime.totalCpuTime }
+      .sortedByDescending { it.indexingTimePerFile.meanTime }
   )
 }
 
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/MaxNTimeBucket.kt b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/MaxNTimeBucket.kt
new file mode 100644
index 000000000000..c0013e501ee2
--- /dev/null
+++ b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/MaxNTimeBucket.kt
@@ -0,0 +1,38 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.util.indexing.diagnostic
+
+import java.util.*
+
+/**
+ * Bucket of times with size of at most [sizeLimit], where the first add time is [firstTime]
+ * Maintains min, max, mean of all added times.
+ */
+class MaxNTimeBucket(private val sizeLimit: Int, firstTime: TimeNano) {
+
+  private val _maxNTimes: PriorityQueue<TimeNano> = PriorityQueue()
+  private var _count: Long = 1
+  private var _minTime: TimeNano = firstTime
+  private var _maxTime: TimeNano = firstTime
+  private var _sum: TimeNano = firstTime
+
+  @Synchronized
+  fun addTime(time: TimeNano) {
+    _maxNTimes += time
+
+    if (_maxTime < time) _maxTime = time
+    if (_minTime > time) _minTime = time
+    _sum += time
+    _count++
+
+    while (_maxNTimes.size > sizeLimit) {
+      _maxNTimes.poll()
+    }
+  }
+
+  val count: Long @Synchronized get() = _count
+  val totalTime: TimeNano @Synchronized get() = _sum
+  val minTime: TimeNano @Synchronized get() = _minTime
+  val maxTime: TimeNano @Synchronized get() = _maxTime
+  val meanTime: Double @Synchronized get() = _sum.toDouble() / _count
+  val maxNTimes: List<TimeNano> @Synchronized get() = _maxNTimes.toList()
+}
\ No newline at end of file
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/PerThreadTime.kt b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/PerThreadTime.kt
index b5abfd13d067..b905b3c212b0 100644
--- a/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/PerThreadTime.kt
+++ b/platform/lang-impl/src/com/intellij/util/indexing/diagnostic/PerThreadTime.kt
@@ -5,13 +5,17 @@ import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.ConcurrentMap
 
 class PerThreadTime {
-  val threadIdToTime: Map<Long, TimeNano>
-    get() = HashMap(_threadIdToTime)
+  private val timeBucketSize = 1024
 
-  private val _threadIdToTime: ConcurrentMap<Long, TimeNano> = ConcurrentHashMap()
+  val threadIdToTimeBucket: Map<Long, MaxNTimeBucket>
+    get() = HashMap(_threadIdToTimeBucket)
+
+  private val _threadIdToTimeBucket: ConcurrentMap<Long, MaxNTimeBucket> = ConcurrentHashMap()
 
   fun addTimeSpentInCurrentThread(nanoTime: TimeNano) {
     val currentThread = Thread.currentThread()
-    _threadIdToTime.compute(currentThread.id) { _, currentTime -> (currentTime ?: 0L) + nanoTime }
+    _threadIdToTimeBucket
+      .computeIfAbsent(currentThread.id) { MaxNTimeBucket(timeBucketSize, nanoTime) }
+      .addTime(nanoTime)
   }
 }
\ No newline at end of file