Remove unused plugins from monorepo

These plugins were extracted from monorepo to a separate repository. The last updated source code is available in the `alter-ego-dev` branch

GitOrigin-RevId: 26946fe048c85fedf4e924f0b8bf67a7a1718c6f
This commit is contained in:
Sasha Blashenkov
2025-02-01 01:28:04 +00:00
committed by intellij-monorepo-bot
parent c452f718f8
commit ef9153fbad
64 changed files with 0 additions and 3816 deletions
@@ -634,21 +634,6 @@ object CommunityRepositoryModules {
}
}
fun aeDatabasePlugin(mainModuleName: String, extraModules: List<String> = emptyList()): PluginLayout {
return plugin(mainModuleName) { spec ->
spec.directoryName = "ae-database"
spec.mainJarName = "ae-database.jar"
spec.withModules(listOf(
"intellij.ae.database.core",
"intellij.ae.database.counters.community"
))
spec.bundlingRestrictions.includeInDistribution = PluginDistribution.ALL
if (extraModules.isNotEmpty()) {
spec.withModules(extraModules)
}
}
}
fun githubPlugin(mainModuleName: String, productCode: String): PluginLayout {
return plugin(mainModuleName) { spec ->
spec.directoryName = "vcs-github-$productCode"
@@ -1,53 +0,0 @@
### auto-generated section `build intellij.ae.database.core` start
load("@rules_jvm//:jvm.bzl", "jvm_library", "jvm_resources", "jvm_test")
jvm_resources(
name = "ae-database-core_resources",
files = glob(["resources/**/*"]),
strip_prefix = "resources"
)
jvm_library(
name = "ae-database-core",
module_name = "intellij.ae.database.core",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True),
deps = [
"@lib//:kotlin-stdlib",
"//platform/util-ex",
"//platform/projectModel-api:projectModel",
"//platform/statistics",
"//platform/core-api:core",
"//platform/sqlite",
"//platform/util",
],
runtime_deps = [":ae-database-core_resources"]
)
jvm_library(
name = "ae-database-core_test_lib",
visibility = ["//visibility:public"],
srcs = glob(["tests/**/*.kt", "tests/**/*.java"], allow_empty = True),
associates = [":ae-database-core"],
deps = [
"@lib//:kotlin-stdlib",
"//platform/util-ex",
"//platform/projectModel-api:projectModel",
"//platform/statistics",
"//platform/statistics:statistics_test_lib",
"//platform/core-api:core",
"//platform/sqlite",
"//platform/sqlite:sqlite_test_lib",
"//platform/util",
"//platform/testFramework",
"//platform/testFramework:testFramework_test_lib",
"@lib//:assert_j",
],
runtime_deps = [":ae-database-core_resources"]
)
jvm_test(
name = "ae-database-core_test",
runtime_deps = [":ae-database-core_test_lib"]
)
### auto-generated section `build intellij.ae.database.core` end
@@ -1,22 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/tests" isTestSource="true" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.util.ex" />
<orderEntry type="module" module-name="intellij.platform.projectModel" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="module" module-name="intellij.platform.core" />
<orderEntry type="module" module-name="intellij.platform.sqlite" />
<orderEntry type="module" module-name="intellij.platform.util" />
<orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
<orderEntry type="library" scope="TEST" name="assertJ" level="project" />
</component>
</module>
@@ -1,13 +0,0 @@
<idea-plugin package="com.intellij.ae.database.core">
<extensionPoints>
<extensionPoint qualifiedName="com.intellij.ae.database.fusEventCatcher"
interface="com.intellij.ae.database.core.baseEvents.fus.FusEventCatcher$Factory"
dynamic="false"
/>
</extensionPoints>
<extensions defaultExtensionNs="com.intellij">
<statistic.eventLog.externalListenerProvider implementation="com.intellij.ae.database.core.baseEvents.fus.FusExternalEventLogListenerProvider"/>
<postStartupActivity implementation="com.intellij.ae.database.core.IdServiceTempPreloader"/>
</extensions>
</idea-plugin>
@@ -1,25 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.project.Project
import kotlinx.coroutines.CoroutineScope
@Service
internal class AEDatabaseLifetime(private val coroutineScope: CoroutineScope) : Disposable {
companion object {
fun getScope() = ApplicationManager.getApplication().service<AEDatabaseLifetime>().coroutineScope
fun getDisposable(): Disposable = ApplicationManager.getApplication().service<AEDatabaseLifetime>()
fun getDisposable(project: Project): Disposable = project.service<AEDatabaseProjectLifetime>()
}
override fun dispose() {}
}
@Service(Service.Level.PROJECT)
internal class AEDatabaseProjectLifetime : Disposable {
override fun dispose() {}
}
@@ -1,57 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core
import com.intellij.ae.database.core.dbs.SqliteDatabaseMetadata
import com.intellij.internal.statistic.eventLog.fus.MachineIdManager
import com.intellij.openapi.application.ApplicationInfo
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.*
import com.intellij.openapi.diagnostic.logger
import java.util.*
private val logger = logger<IdService>()
@Service(Service.Level.APP)
@State(name = "AEIdeId", storages = [Storage("aeIdeId.xml", roamingType = RoamingType.DISABLED)], reportStatistic = false)
class IdService : PersistentStateComponentWithModificationTracker<IdService.State> {
class State: BaseState() {
var id: String? by string("undefined")
}
companion object {
fun getInstance() = ApplicationManager.getApplication().service<IdService>()
suspend fun getInstanceAsync() = ApplicationManager.getApplication().serviceAsync<IdService>()
}
val id get() = state.id ?: run {
logger.error("id was not defined")
"undefined"
}
val machineId by lazy { MachineIdManager.getAnonymizedMachineId("com.intellij.platform.ae.database") ?: "undefined" }
fun getDatabaseId(metadata: SqliteDatabaseMetadata) = metadata.ideId
val ideCode by lazy {
ApplicationInfo.getInstance().build.productCode.ifBlank {
if (ApplicationManager.getApplication().isUnitTestMode) "unittest"
else {
logger.warn("productCode is blank, will use 'unknown' value")
"unknown"
}
}
}
private var myState = State()
override fun getState() = myState
override fun getStateModificationCount() = state.modificationCount
override fun loadState(state: State) {
myState = state
}
override fun noStateLoaded() {
myState.id = UUID.randomUUID().toString()
}
}
@@ -1,10 +0,0 @@
package com.intellij.ae.database.core
import com.intellij.openapi.project.Project
import com.intellij.openapi.startup.ProjectActivity
private class IdServiceTempPreloader : ProjectActivity {
override suspend fun execute(project: Project) {
IdService.getInstanceAsync().id
}
}
@@ -1,19 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.activities
import com.intellij.ae.database.core.dbs.counter.CounterUserActivityDatabase
import com.intellij.ae.database.core.utils.InstantUtils
import java.time.Instant
interface CounterUserActivity : UserActivity
abstract class DatabaseBackedCounterUserActivity : CounterUserActivity {
protected suspend fun getDatabase() = CounterUserActivityDatabase.getInstanceAsync()
}
abstract class WritableDatabaseBackedCounterUserActivity : DatabaseBackedCounterUserActivity() {
protected suspend fun submit(diff: Int, eventTime: Instant = InstantUtils.Now) {
getDatabase().submit(this, diff, eventTime)
}
}
@@ -1,27 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.activities
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabase
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import java.time.Instant
interface TimeSpanUserActivity : UserActivity
abstract class DatabaseBackedTimeSpanUserActivity : TimeSpanUserActivity {
protected val coroutineScope get() = com.intellij.ae.database.core.AEDatabaseLifetime.getScope()
protected suspend fun getDatabase() = TimeSpanUserActivityDatabase.getInstanceAsync()
}
abstract class WritableDatabaseBackedTimeSpanUserActivity : DatabaseBackedTimeSpanUserActivity() {
abstract val canBeStale: Boolean
protected suspend fun submitManual(id: String, kind: TimeSpanUserActivityDatabaseManualKind, extra: Map<String, String>?, moment: Instant? = null) {
getDatabase().submitManual(this, id, kind, canBeStale, extra, moment)
}
protected suspend fun submitPeriodic(id: String, extra: Map<String, String>? = null) {
getDatabase().submitPeriodicEvent(this, id, extra)
}
}
fun DatabaseBackedTimeSpanUserActivity.toKey(extraKey: String) = "${id}_${extraKey}"
@@ -1,45 +0,0 @@
package com.intellij.ae.database.core.activities
/**
* Base interface for user activity – nothing but an ID.
*
* Any class with [UserActivity] interface should be:
* 1. a Kotlin object
* 2. stateless
*
* You don't want to implement this interface. Instead, implement:
* * [WritableDatabaseBackedCounterUserActivity]: a counter user activity that can be stored in a database
* * [WritableDatabaseBackedTimeSpanUserActivity]: a time span user activity that can be stored in a database
* * [ReadableUserActivity]: if your activity can return a result
* * [DatabaseBackedCounterUserActivity] / [DatabaseBackedTimeSpanUserActivity]: useful for readable user activities,
* if you need to access database (it most cases you would like to do so)
*
* =-=-=-=-=-=-=-=-=-=-=-=-=
*
* Implementation example 1:
*
* For an upcoming campaign you need to implement a feature counter: 'The longest streak of IDE opening'.
* To achieve the goal you need to implement the following objects:
* 1. a [WritableDatabaseBackedTimeSpanUserActivity] that records each time IDE is started. This activity doesn't return any information, it's
* write only. Question to careful readers: what should it return?
* 2. a [ReadableUserActivity] that calculates the longest streak of IDE opening based on data from activity (1). This activity by itself
* doesn't store any data
*
* When the developer will be asked to calculate some new value and it can be calculated with data about when IDE was running, they can easily
* reuse data from user activity (1)
*/
sealed interface UserActivity {
val id: String
}
/**
* Represents an activity that can return some sort of value or several values.
*/
interface ReadableUserActivity<TResult> : UserActivity {
/**
* Returns a value calculated for this activity. You are free to implement other
* 'getter' methods in your class, but this method should return some sort of
* default value
*/
suspend fun getActivityValue(): TResult
}
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.activities;
import org.jetbrains.annotations.ApiStatus;
@@ -1,60 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.baseEvents
import com.intellij.ae.database.core.activities.WritableDatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.baseEvents.fus.FusEventCatcher
import com.intellij.ae.database.core.baseEvents.fus.FusEventDefinitionBuilder
import java.time.Instant
/**
* FUS-based counter user activity. You should define the event with [define] method.
*
* You need to register your activity in XML
*
* @see com.intellij.ae.database.baseEvents.fus.FusEventCatcher
*/
abstract class FusBasedCounterUserActivity : WritableDatabaseBackedCounterUserActivity() {
protected val catcher = object : FusEventCatcher() {
override fun define(): FusEventDefinitionBuilder {
return this@FusBasedCounterUserActivity.define()
}
override suspend fun onEvent(fields: Map<String, Any>, eventTime: Instant) {
getDatabase().submit(this@FusBasedCounterUserActivity, getIncrementValue(fields), eventTime)
}
}
final override val id: String by lazy { catcher.definition.id }
/**
* Override this method if your need to do custom calculations over FUS data
*
* Example 1:
* You want to write down every debug session start. In this case '1' is good for you here, no need to override
*
* Example 2:
* You want to calculate how many characters completion saved. You need to subtract 'typing' field from 'token_length' field.
* Of course, don't forget null checks. The resulting method should look something like this:
* ```
* val tokenLen = fields["token_length"] as? Int
* val typing = fields["typing"] as? Int
* if (tokenLen == null || typing == null) {
* thisLogger().error("One of required fields is null")
* return 1
* }
* return tokenLen - typing
* ```
*/
protected open fun getIncrementValue(fields: Map<String, Any>) = 1
/**
* @see com.intellij.ae.database.baseEvents.fus.FusEventCatcher.define
*/
protected abstract fun define(): FusEventDefinitionBuilder
/**
* @see com.intellij.ae.database.baseEvents.fus.FusEventCatcher.definition
*/
protected fun definition(id: String, x: FusEventDefinitionBuilder.() -> Unit) = catcher.definition(id, x)
}
@@ -1,105 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.baseEvents.fus
import com.intellij.openapi.extensions.ExtensionPointName
import java.time.Instant
data class FusEventDefinitionField<T>(
val type: Class<T>,
val comparator: (T) -> Boolean
)
internal data class FusEventDefinition(
val id: String,
val group: String,
val event: String,
val fields: Map<String, FusEventDefinitionField<Any>>
)
class FusEventDefinitionBuilder(private val id: String) {
val fields = mutableMapOf<String, FusEventDefinitionField<*>>()
private var myGroup: String? = null
private var myEvent: String? = null
inner class FieldsBuilder {
inline fun <reified T> field(name: String, value: T) = field<T>(name) { it == value }
inline fun <reified T> field(name: String, noinline comparator: (T) -> Boolean) {
fields[name] = FusEventDefinitionField(T::class.java, comparator)
}
}
fun event(group: String, event: String, x: FieldsBuilder.() -> Unit) {
if (myGroup != null) error("event() can be called only once (at least for now)")
myGroup = group
myEvent = event
x(FieldsBuilder())
}
fun event(group: String, event: String) {
event(group, event) {}
}
internal fun build(): FusEventDefinition {
val group = myGroup ?: error("Group is not defined")
val event = myEvent ?: error("Event is not defined")
@Suppress("UNCHECKED_CAST")
return FusEventDefinition(id, group, event, fields as Map<String, FusEventDefinitionField<Any>>)
}
}
/**
* A class that allows to catch specific FUS event. You should define the event with [define] method
*
* You need to register your activity in XML:
* ```
* <fusEventCatcher implementation="com.intellij.ae.database.v2.events.SampleFusBasedUserActivity$Factory"/>
* ```
*
* `implementation` is a path to [Factory] class. Note the dollar symbol at the end.
*/
abstract class FusEventCatcher {
companion object {
val EP_NAME = ExtensionPointName.create<Factory>("com.intellij.ae.database.fusEventCatcher")
}
interface Factory {
fun getInstance(): FusEventCatcher
}
internal val definition: FusEventDefinition by lazy { define().build() }
fun definition(id: String, x: FusEventDefinitionBuilder.() -> Unit): FusEventDefinitionBuilder {
return FusEventDefinitionBuilder(id).apply(x)
}
/**
* A definition of event.
*
* Starts with [definition] function. It accepts 'id' as an argument – it will be used in [id] object field; and a lambda.
* Lambda contains definition of FUS event. You should call [FusEventDefinitionBuilder.event] method and pass event group, event id and a lambda.
* This lambda contains a list of fields that should be present in an event.
*
* Example:
* ```
* definition("sampleEvent") {
* event("toolwindow", "activated") {
* field("id", "Project")
* field<Int>("invocation") { it > 10 }
* }
* }
* ```
*
* It defines user activity "sampleEvent", which is FUS event "toolwindow.activated" with fields "id" = "Project" and "invocation" > 10.
* Note that logical condition between fields is 'AND' – all fields should satisfy the condition
*/
protected abstract fun define(): FusEventDefinitionBuilder
/**
* Perform an action when FUS event from [define] occurs
*/
abstract suspend fun onEvent(fields: Map<String, Any>, eventTime: Instant)
}
@@ -1,111 +0,0 @@
package com.intellij.ae.database.core.baseEvents.fus
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.internal.statistic.eventLog.EventLogListenersManager
import com.intellij.internal.statistic.eventLog.ExternalEventLogListenerProviderExtension
import com.intellij.internal.statistic.eventLog.StatisticsEventLogListener
import com.intellij.openapi.Disposable
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.components.serviceIfCreated
import com.intellij.util.asSafely
import com.jetbrains.fus.reporting.model.lion3.LogEvent
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.time.Instant
/**
* Service that receives all FUS events and tries to find suitable [FusEventCatcher]
*
* TODO: lifetime of this service is longer than lifetime of [FusEventCatcher], because a catcher can be implemented inside a plugin,
* TODO: but an EP is marked is marked as non-dynamic and there is no catchers outside impl-detail plugin
*/
@Service(Service.Level.APP)
internal class FusEventCatcherService(cs: CoroutineScope) {
companion object {
internal fun getInstance() = ApplicationManager.getApplication().service<FusEventCatcherService>()
}
private data class EventSubmission(val group: String, val event: String, val fields: Map<String, Any>, val time: Instant)
private val eventsSubmitFlow = MutableSharedFlow<EventSubmission>(replay = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST)
// group to event
private val catchers = mutableMapOf<Pair<String, String>, MutableList<FusEventCatcher>>()
private val catchersLock = Mutex()
init {
cs.launch {
eventsSubmitFlow.collect {
val catchers = find(it.group, it.event, it.fields)
for (catcher in catchers) {
catcher.onEvent(it.fields, it.time)
}
}
}
}
fun submit(group: String, event: String, fields: Map<String, Any>) {
val time = fields["created"]?.asSafely<Long>()?.let { Instant.ofEpochMilli(it) } ?: InstantUtils.Now
eventsSubmitFlow.tryEmit(EventSubmission(group, event, fields, time))
}
private suspend fun find(group: String, event: String, fields: Map<String, Any>): List<FusEventCatcher> {
val suitableCatchers = catchersLock.withLock {
if (catchers.isEmpty()) {
initCatchers()
}
catchers[group to event]
}
return suitableCatchers?.filter { catcher ->
catcher.definition.fields.all { definitionField ->
val value = fields[definitionField.key] ?: return@all false
val definition = definitionField.value
if (definition.type != value::class.java) return@all false
definition.comparator(value)
}
} ?: emptyList()
}
// must run under [catchersLock]
private fun initCatchers() {
assert(catchersLock.isLocked)
for (catcher in FusEventCatcher.EP_NAME.extensionList.map { it.getInstance() }) {
val key = catcher.definition.group to catcher.definition.event
catchers.getOrPut(key) { mutableListOf() }.add(catcher)
}
}
}
private class Listener : StatisticsEventLogListener {
override fun onLogEvent(validatedEvent: LogEvent, rawEventId: String?, rawData: Map<String, Any>?) {
// rawEventId and rawData can't be used
val group = validatedEvent.group.id
val event = validatedEvent.event.id
val fields = validatedEvent.event.data
FusEventCatcherService.getInstance().submit(group, event, fields)
}
}
internal class FusExternalEventLogListenerProvider : ExternalEventLogListenerProviderExtension {
override fun forceLoggingAlwaysEnabled(): Boolean {
return true
}
override fun getEventLogListener(recorderId: String): StatisticsEventLogListener? {
if (recorderId == "FUS") {
return Listener()
}
else return null
}
}
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.baseEvents;
import org.jetbrains.annotations.ApiStatus;
@@ -1,13 +0,0 @@
package com.intellij.ae.database.core.dbs
/**
* A layer that uses SQLite to store data.
*
* Be very careful with these APIs! You MUST NOT make any write operations to the database
*/
interface ISqliteBackedDatabaseLayer {
/**
* Table name for the current layer. Use this in your SQL queries
*/
val tableName: String
}
@@ -1,12 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.dbs
import org.jetbrains.sqlite.SqliteConnection
interface ISqliteExecutor {
suspend fun <T> execute(action: suspend (initDb: SqliteConnection) -> T): T?
}
interface ISqliteInternalExecutor {
suspend fun <T> execute(action: suspend (initDb: SqliteConnection, metadata: SqliteDatabaseMetadata) -> T): T?
}
@@ -1,3 +0,0 @@
package com.intellij.ae.database.core.dbs
interface IUserActivityDatabaseLayer
@@ -1,87 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.dbs
import com.intellij.ae.database.core.IdService
import com.intellij.ae.database.core.dbs.migrations.LAST_DB_VERSION
import com.intellij.ae.database.core.dbs.migrations.MIGRATIONS
import com.intellij.openapi.diagnostic.logger
import com.intellij.util.concurrency.ThreadingAssertions
import org.jetbrains.sqlite.ObjectBinderFactory
import org.jetbrains.sqlite.SqliteConnection
import org.jetbrains.sqlite.SqliteException
private val logger = logger<SqliteDatabaseMetadata>()
/**
* A class that checks that database is properly initialized, runs migrations and stores metadata information.
* Use it to run something in database once per application run
*
* If you want to add new migration, add text to [MIGRATIONS] array
*
* This class must be initialized on background thread
*/
class SqliteDatabaseMetadata internal constructor(private val connection: SqliteConnection, isNewFile: Boolean) {
/**
* ID from database that represents a pair of IDE ID [IdService.id]
* and machine ID [IdService.machineId] and IDE family [ApplicationInfo]
*/
val ideId: Int
init {
ThreadingAssertions.assertBackgroundThread()
if (isNewFile) {
logger.info("New database, executing migration")
executeMigrations(0)
}
else {
val version = try {
getVersion()
}
catch (t: SqliteException) {
if (t.message == "[SQLITE_ERROR] SQL error or missing database (no such table: meta)") {
logger.info("File exists, but database seem to be uninitialized")
0
}
else {
throw t
}
}
executeMigrations(version)
}
ideId = initIdeId()
}
private fun executeMigrations(fromVersion: Int) {
val migrations = MIGRATIONS.subList(maxOf(fromVersion, 0), LAST_DB_VERSION)
if (migrations.isEmpty()) {
return
}
for (migration in migrations) {
connection.execute(migration)
}
connection
.prepareStatement("UPDATE meta SET version = (?) WHERE true;", ObjectBinderFactory.create1<Int>())
.apply { binder.bind(LAST_DB_VERSION) }
.executeUpdate()
}
private fun getVersion(): Int {
return connection.selectInt("SELECT version FROM meta LIMIT 1") ?: -1
}
private fun initIdeId(): Int {
val potentialResult = connection.prepareStatement("SELECT id FROM ide WHERE ide_id = (?) AND machine_id = (?) LIMIT 1", ObjectBinderFactory.create2<String, String>()).use { statement ->
statement.binder.bind(IdService.getInstance().id, IdService.getInstance().machineId)
statement.selectInt()
}
if (potentialResult != null) return potentialResult
return connection.prepareStatement("insert into ide(ide_id, machine_id, family) values (?, ?, ?) RETURNING id;", ObjectBinderFactory.create3<String, String, String>()).use { statement ->
statement.binder.bind(IdService.getInstance().id, IdService.getInstance().machineId, IdService.getInstance().ideCode)
statement.selectInt() ?: error("Null was returned when not expected")
}
}
}
@@ -1,333 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("SqlResolve")
@file:OptIn(ExperimentalCoroutinesApi::class)
package com.intellij.ae.database.core.dbs
import com.intellij.ae.database.core.IdService
import com.intellij.diagnostic.dumpCoroutines
import com.intellij.openapi.application.ApplicationInfo
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.application.PathManager
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.components.serviceAsync
import com.intellij.openapi.diagnostic.logger
import com.intellij.util.PlatformUtils
import com.intellij.util.io.createDirectories
import kotlinx.coroutines.*
import org.jetbrains.annotations.VisibleForTesting
import org.jetbrains.sqlite.SqliteConnection
import java.nio.file.Path
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicReference
import kotlin.io.path.exists
import kotlin.io.path.isWritable
import kotlin.time.Duration
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds
private val logger = logger<SqliteLazyInitializedDatabase>()
private const val MAX_CONNECTION_RETRIES_ALLOWED = 5
/**
* This service provides access to an instance of [SqliteConnection]
*/
@Service
class SqliteLazyInitializedDatabase(private val cs: CoroutineScope) : ISqliteExecutor, ISqliteInternalExecutor {
companion object {
internal suspend fun getInstanceAsync() = serviceAsync<SqliteLazyInitializedDatabase>()
internal fun getInstance() = ApplicationManager.getApplication().service<SqliteLazyInitializedDatabase>()
}
private class ConnectionHolder(val connection: SqliteConnection, val metadata: SqliteDatabaseMetadata)
private sealed class State {
data object NotInitialized : State()
class Active(val def: Deferred<ConnectionHolder>) : State()
class Cancelling(val def: Deferred<ConnectionHolder>, val job: Job) : State()
data object Locked : State()
}
private val databasePath by lazy { createDatabasePath() }
private val connectionAttempts = AtomicInteger(0)
// private val lastAccessTime = MutableStateFlow<Long>(0)
private val myConnection = AtomicReference<State>(State.NotInitialized)
private val actionsBeforeDatabaseDisposal = mutableListOf<suspend (isFinal: Boolean) -> Unit>()
private val closeDispatcher = Dispatchers.IO.limitedParallelism(1)
private val loggingAttempt = AtomicInteger(0)
init {
if (System.getProperty("ae.database.fullLock")?.toBoolean() == true) {
myConnection.set(State.Locked)
}
cs.launch {
while (true) {
delay(6.minutes)
mainClosingLogic(15.seconds, "every 6 min", false)
}
}
cs.launch {
try {
awaitCancellation()
}
finally {
withContext(NonCancellable) {
mainClosingLogic(6.seconds, "scope cancellation", true)
}
}
}
}
private var isFinalClosed = false
private suspend fun CoroutineScope.mainClosingLogic(timeout: Duration, reason: String, isFinal: Boolean) {
val shouldLog = isFinal || loggingAttempt.getAndIncrement() % 10 == 0
if (isFinalClosed) {
logger.info("Close database for reason \"${reason}\" ignored: already closed.")
}
if (shouldLog) {
logger.info("Starting saving database (reason: $reason)")
}
try {
withTimeout(timeout) {
withContext(closeDispatcher) {
closeDatabaseImpl(isFinal, shouldLog)?.join()
if (shouldLog) {
logger.info("Saving completed (reason: $reason)")
}
}
}
}
catch (t: TimeoutCancellationException) {
if (PlatformUtils.isPyCharm()) {
// for some reason PyCharm and DataSpell installers don't start `closeDatabaseImpl()` in time, but only a minute later
// this is not critical and no data will be (hopefully) lost, but the fix will be impletented a bit later
logger.warn("Saving timeout (saving reason: $reason)\n${dumpCoroutines(this)}", t)
logger.warn(dumpCoroutines(this))
}
else {
logger.error("Saving timeout (saving reason: $reason)\n${dumpCoroutines(this)}", t)
}
}
catch (t: Throwable) {
logger.error("Saving failed (saving reason: $reason)\n${dumpCoroutines(this)}", t)
}
finally {
if (isFinal) isFinalClosed = true
}
}
fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
actionsBeforeDatabaseDisposal.add(action)
}
@VisibleForTesting
suspend fun closeDatabase() {
coroutineScope {
mainClosingLogic(16.seconds, "in test", true)
}
}
@Suppress("OPT_IN_USAGE")
private fun closeDatabaseImpl(isFinal: Boolean, shouldLog: Boolean): Job? {
// service scope is dead at this point, need to use GlobalScope
// todo: not true on temp close
fun launchJob(action: suspend CoroutineScope.() -> Unit): Job =
GlobalScope.launch(Dispatchers.IO, start = CoroutineStart.LAZY, action)
while (true) {
when (val state = myConnection.get()) {
is State.NotInitialized -> {
if (!isFinal) return null
return launchJob {
logger.info("Opening connection for final termination.")
val connection = getConn2(this)
if (connection != null) {
logger.info("Connection opened, performing termination.")
close(connection, isFinal = true, shouldLog = shouldLog)
} else {
logger.info("Connection open failure.")
}
}
}
is State.Active -> {
val job = launchJob {
close(state.def.await(), isFinal, shouldLog = shouldLog)
}
val newState = State.Cancelling(state.def, job)
if (myConnection.compareAndSet(state, newState)) {
job.invokeOnCompletion {
check(myConnection.compareAndSet(newState, State.NotInitialized))
}
return job
}
else {
job.cancel()
}
}
is State.Cancelling -> {
return state.job
}
is State.Locked -> {
return null
}
}
}
}
private suspend fun doExecuteBeforeConnectionClosed(isFinal: Boolean) {
for (action in actionsBeforeDatabaseDisposal) {
action(isFinal)
}
}
/**
* Allows executing code with [SqliteConnection]
*/
override suspend fun <T> execute(action: suspend (initDb: SqliteConnection, metadata: SqliteDatabaseMetadata) -> T): T? {
val conn = withContext(Dispatchers.IO) {
getConn2()
}
if (conn == null) {
return null
}
return action(conn.connection, conn.metadata)
}
private suspend fun getConn2(scope: CoroutineScope = cs): ConnectionHolder? {
while (true) {
when (val state = myConnection.get()) {
State.NotInitialized -> {
val connectionHolderDeferred = CompletableDeferred<ConnectionHolder>(parent = scope.coroutineContext.job)
if (!myConnection.compareAndSet(state, State.Active(connectionHolderDeferred))) {
connectionHolderDeferred.cancel()
continue
}
val conn = kotlin.runCatching {
val dbPath = databasePath
logger.info("Database path: $dbPath")
val isNewFile = dbPath == null || !dbPath.exists()
val newConnection = SqliteConnection(dbPath, false)
val newMetadata = SqliteDatabaseMetadata(newConnection, isNewFile)
ConnectionHolder(newConnection, newMetadata)
}
if (!connectionHolderDeferred.completeWith(conn)) {
if (conn.isSuccess) {
conn.getOrThrow().connection.close()
}
else {
if (connectionAttempts.getAndIncrement() >= MAX_CONNECTION_RETRIES_ALLOWED) {
myConnection.compareAndSet(state, State.Locked)
}
}
}
return connectionHolderDeferred.await()
}
is State.Active -> {
return state.def.await()
}
is State.Cancelling -> {
// during cancellation there might be calls to DB
return state.def.await()
}
is State.Locked -> {
return null
}
}
}
}
override suspend fun <T> execute(action: suspend (initDb: SqliteConnection) -> T): T? {
return execute { initDb, metadata ->
action(initDb)
}
}
private suspend fun close(connectionHolder: ConnectionHolder, isFinal: Boolean, shouldLog: Boolean) {
if (shouldLog) {
logger.info("close start")
}
connectionHolder.connection.use {
doExecuteBeforeConnectionClosed(isFinal)
}
check(connectionHolder.connection.isClosed)
if (shouldLog) {
logger.info("close end")
}
}
private fun createDatabasePath(): Path? {
val tempPath = System.getProperty("ae.database.path")
if (tempPath != null) {
return Path.of(tempPath)
}
val majorVersion = ApplicationInfo.getInstance().build.baselineVersion
// todo if settings are migrated, hash is the same
val fileName = "ae_${IdService.getInstance().id}-$majorVersion.db"
val fileMask = "ae_${IdService.getInstance().id}-*.db"
// Attempt 1: store db file in common folder for all ides
val attempt1 = try {
createDatabasePathStoreInCommonFolder(fileName, fileMask)
}
catch (t: Throwable) {
logger.error("Could not get path in common folder", t)
null
}
if (attempt1 != null) {
return attempt1
}
// Attempt 2: store db file in IDE's `config` directory
val attempt2 = createDatabasePathStoreInConfigFolder(fileName, fileMask)
return attempt2
}
private fun createDatabasePathStoreInCommonFolder(fileName: String, mask: String): Path? {
val commonFolder = PathManager.getCommonDataPath()
val intellijFolder = commonFolder.resolve("IntelliJ")
val desiredDatabasePath = intellijFolder.resolve(fileName)
if (System.getProperty("ae.database.forceConfigFolder")?.toBoolean() == true) {
return null
}
// folder where the file is located is required to be writable for journal creation
if (intellijFolder.exists() && !intellijFolder.isWritable()) {
logger.warn("Folder $intellijFolder exist, but not writable")
return null
}
if (!intellijFolder.exists() && !commonFolder.isWritable()) {
logger.warn("Folder $intellijFolder does not exist and $commonFolder is not writable")
return null
}
if (desiredDatabasePath.exists() && !desiredDatabasePath.isWritable()) {
logger.warn("Db file $desiredDatabasePath exists, but not writable")
return null
}
if (!intellijFolder.exists()) {
intellijFolder.createDirectories()
}
return desiredDatabasePath
}
private fun createDatabasePathStoreInConfigFolder(fileName: String, mask: String): Path {
val configFolder = PathManager.getConfigDir()
val databasePath = configFolder.resolve(fileName)
return databasePath
}
}
@@ -1,7 +0,0 @@
package com.intellij.ae.database.core.dbs
object SqliteNames {
const val META_TABLE = "meta"
const val IDE_TABLE = "ide"
const val COUNTER_USER_ACTIVITY_TABLE = "counterUserActivity"
}
@@ -1,95 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("SqlResolve")
package com.intellij.ae.database.core.dbs.counter
import com.intellij.ae.database.core.IdService
import com.intellij.ae.database.core.activities.DatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.dbs.*
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.components.serviceAsync
import com.intellij.openapi.diagnostic.thisLogger
import kotlinx.coroutines.CoroutineScope
import org.jetbrains.sqlite.ObjectBinderFactory
import org.jetbrains.sqlite.SqliteConnection
import java.time.Instant
/**
* Database for storing and retrieving counter-based events.
*
* All events are rounded to a minute, seconds and milliseconds are always 0
*/
@Service
class CounterUserActivityDatabase(cs: CoroutineScope) : ICounterUserActivityDatabase,
IUserActivityDatabaseLayer,
IReadOnlyCounterUserActivityDatabase,
IInternalCounterUserActivityDatabase,
ISqliteExecutor, ISqliteBackedDatabaseLayer {
companion object {
suspend fun getInstanceAsync() = serviceAsync<CounterUserActivityDatabase>()
fun getInstance() = ApplicationManager.getApplication().service<CounterUserActivityDatabase>()
}
private val throttler = CounterUserActivityDatabaseThrottler(cs, this, runBackgroundUpdater = !ApplicationManager.getApplication().isUnitTestMode)
/**
* Retrieves the activity for a user based on the provided activity ID and time range.
* Answers the question 'How many times did activity happen in given timeframe?'
*/
override suspend fun getActivitySum(activity: DatabaseBackedCounterUserActivity, from: Instant?, until: Instant?): Int {
val nnFrom = InstantUtils.formatForDatabase(from ?: InstantUtils.SomeTimeAgo)
val nnUntil = InstantUtils.formatForDatabase(until ?: InstantUtils.NowButABitLater)
return execute { connection ->
val getActivityStatement = connection.prepareStatement(
"SELECT sum(diff) FROM counterUserActivity WHERE activity_id = ? AND created_at >= ? AND created_at <= ?",
ObjectBinderFactory.create3<String, String, String>()
)
throttler.commitChanges()
getActivityStatement.binder.bind(activity.id, nnFrom, nnUntil)
getActivityStatement.selectInt() ?: 0
} ?: 0
}
/**
* Main entry point for submitting new event update
*
* This method doesn't submit to database, but first submits to throttler which
*/
override suspend fun submit(activity: DatabaseBackedCounterUserActivity, diff: Int, eventTime: Instant) {
thisLogger().info("${activity.id} = $diff")
throttler.submit(activity, diff, eventTime)
}
/**
* Writes event directly to database. Very internal API!
*/
override suspend fun submitDirect(activity: DatabaseBackedCounterUserActivity, diff: Int, instant: Instant, extra: Map<String, String>?) {
execute { database, metadata ->
val updateActivityStatement = database.prepareStatement(
"INSERT INTO counterUserActivity (activity_id, diff, created_at, ide_id) VALUES (?, ?, ?, ?)",
ObjectBinderFactory.create4<String, Int, String, Int>()
)
updateActivityStatement.binder.bind(activity.id, diff, InstantUtils.formatForDatabase(instant), IdService.getInstance().getDatabaseId(metadata))
updateActivityStatement.executeUpdate()
}
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
SqliteLazyInitializedDatabase.getInstance().executeBeforeConnectionClosed(action)
}
override suspend fun <T> execute(action: suspend (initDb: SqliteConnection) -> T): T? {
return SqliteLazyInitializedDatabase.getInstanceAsync().execute(action)
}
private suspend fun <T> execute(action: suspend (initDb: SqliteConnection, metadata: SqliteDatabaseMetadata) -> T): T? {
return SqliteLazyInitializedDatabase.getInstanceAsync().execute(action)
}
override val tableName: String = "counterUserActivity"
}
@@ -1,79 +0,0 @@
package com.intellij.ae.database.core.dbs.counter
import com.intellij.ae.database.core.activities.DatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.openapi.diagnostic.logger
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import java.time.Instant
import kotlin.time.Duration
import kotlin.time.Duration.Companion.minutes
private val logger = logger<CounterUserActivityDatabaseThrottler>()
private data class CounterUserActivityHits(
val firstOccurance: Instant,
var lastOccurance: Instant,
var count: Int
)
/**
* Collects events and posts it as a "one big event" to a database to avoid intensive write operations
*/
internal class CounterUserActivityDatabaseThrottler(private val cs: CoroutineScope,
private val database: IInternalCounterUserActivityDatabase,
private val updatePause: Duration = 2.minutes,
runBackgroundUpdater: Boolean = true) {
private val events = HashMap<DatabaseBackedCounterUserActivity, CounterUserActivityHits>()
private val eventsLock = Mutex()
init {
if (runBackgroundUpdater) {
cs.launch {
while (isActive) {
logger.trace("Updater was (re)started")
delay(updatePause)
commitChanges()
logger.trace("Updater finished")
}
}
}
database.executeBeforeConnectionClosed { isFinal ->
if (isFinal) {
commitChanges()
}
}
}
suspend fun submit(activity: DatabaseBackedCounterUserActivity, newValue: Int, eventTime: Instant = InstantUtils.Now) {
eventsLock.withLock {
val ev = events.getOrElse(activity) {
CounterUserActivityHits(eventTime, eventTime, 0)
}
ev.lastOccurance = eventTime
ev.count += newValue
events[activity] = ev
}
}
suspend fun commitChanges() {
eventsLock.withLock {
if (events.isNotEmpty()) {
events
.filter { it.value.count != 0 }
.apply { logger.info("Started committing changes ($size)") }
.forEach {
database.submitDirect(it.key, it.value.count, it.value.lastOccurance, null)
}
events.clear()
}
}
}
}
@@ -1,18 +0,0 @@
package com.intellij.ae.database.core.dbs.counter
import com.intellij.ae.database.core.activities.DatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.utils.InstantUtils
import java.time.Instant
interface ICounterUserActivityDatabase {
suspend fun submit(activity: DatabaseBackedCounterUserActivity, diff: Int, eventTime: Instant = InstantUtils.Now)
}
internal interface IInternalCounterUserActivityDatabase {
suspend fun submitDirect(activity: DatabaseBackedCounterUserActivity, diff: Int, instant: Instant, extra: Map<String, String>? = null)
fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit)
}
interface IReadOnlyCounterUserActivityDatabase {
suspend fun getActivitySum(activity: DatabaseBackedCounterUserActivity, from: Instant?, until: Instant?): Int
}
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.dbs.counter;
import org.jetbrains.annotations.ApiStatus;
@@ -1,4 +0,0 @@
package com.intellij.ae.database.core.dbs.migrations
val MIGRATIONS = listOf(MIGRATION_V1, MIGRATION_V2)
val LAST_DB_VERSION = MIGRATIONS.size
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.dbs.migrations;
import org.jetbrains.annotations.ApiStatus;
@@ -1,43 +0,0 @@
package com.intellij.ae.database.core.dbs.migrations
import org.intellij.lang.annotations.Language
@Language("SQLite")
internal val MIGRATION_V1 = """
CREATE TABLE "meta" (
version INTEGER NOT NULL
);
CREATE TABLE "ide" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
machine_id TEXT NOT NULL,
ide_id TEXT NOT NULL,
family TEXT NOT NULL
);
CREATE TABLE "counterUserActivity" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
activity_id TEXT NOT NULL,
ide_id TEXT NOT NULL,
created_at TEXT NOT NULL,
diff INTEGER NOT NULL,
extra TEXT,
FOREIGN KEY (ide_id) REFERENCES ide(id)
);
CREATE TABLE "timespanUserActivity" (
id INTEGER PRIMARY KEY AUTOINCREMENT,
activity_id TEXT NOT NULL,
ide_id TEXT NOT NULL,
started_at TEXT NOT NULL,
ended_at TEXT NOT NULL,
extra TEXT,
FOREIGN KEY (ide_id) REFERENCES ide(id)
);
INSERT INTO meta (version) VALUES (1);
"""
@@ -1,8 +0,0 @@
package com.intellij.ae.database.core.dbs.migrations
import org.intellij.lang.annotations.Language
@Language("SQLite")
internal val MIGRATION_V2 = """
ALTER TABLE "timespanUserActivity" ADD COLUMN is_finished INTEGER DEFAULT 0;
"""
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.dbs;
import org.jetbrains.annotations.ApiStatus;
@@ -1,55 +0,0 @@
package com.intellij.ae.database.core.dbs.timespan
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.activities.toKey
import java.time.Instant
data class EventDescriptor private constructor (
val activity: DatabaseBackedTimeSpanUserActivity,
val id: String,
val canBeStale: Boolean,
val isPeriodic: Boolean,
val startedAt: Instant,
var endAt: Instant? = null,
var databaseId: Int? = null,
val extra: Map<String, String>? = null,
) {
companion object {
internal fun manual(
activity: DatabaseBackedTimeSpanUserActivity,
id: String,
canBeStale: Boolean,
startedAt: Instant,
extra: Map<String, String>?,
): EventDescriptor = EventDescriptor(
activity,
id,
canBeStale,
isPeriodic = false,
startedAt = startedAt,
endAt = null,
databaseId = null,
extra = extra,
)
internal fun periodic(
activity: DatabaseBackedTimeSpanUserActivity,
id: String,
startedAt: Instant,
endAt: Instant,
canBeStale: Boolean,
extra: Map<String, String>?,
): EventDescriptor = EventDescriptor(
activity,
id,
canBeStale = canBeStale,
isPeriodic = true,
startedAt = startedAt,
endAt = endAt,
databaseId = null,
extra = extra,
)
}
}
internal fun EventDescriptor.toKey(): String = this.activity.toKey(id)
@@ -1,243 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@file:Suppress("SqlResolve")
package com.intellij.ae.database.core.dbs.timespan
import com.intellij.ae.database.core.IdService
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.*
import com.intellij.ae.database.core.formatString
import com.intellij.ae.database.core.models.TimeSpan
import com.intellij.ae.database.core.utils.BooleanUtils
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.serviceAsync
import com.intellij.openapi.diagnostic.logger
import kotlinx.coroutines.CoroutineScope
import org.jetbrains.annotations.TestOnly
import org.jetbrains.sqlite.ObjectBinderFactory
import org.jetbrains.sqlite.SqliteConnection
import java.time.Instant
enum class TimeSpanUserActivityDatabaseManualKind {
Start,
End
}
private val logger = logger<TimeSpanUserActivityDatabase>()
@Service
class TimeSpanUserActivityDatabase(cs: CoroutineScope) : IUserActivityDatabaseLayer,
IReadOnlyTimeSpanUserActivityDatabase,
ITimeSpanUserActivityDatabase,
IInternalTimeSpanUserActivityDatabase,
ISqliteBackedDatabaseLayer,
ISqliteExecutor {
companion object {
internal suspend fun getInstanceAsync() = serviceAsync<TimeSpanUserActivityDatabase>()
}
private val throttler = TimeSpanUserActivityDatabaseThrottler(cs, this)
@TestOnly
fun cancelBackgroundUpdate() {
throttler.cancelBackgroundUpdate()
}
override suspend fun getLongestActivity(activity: DatabaseBackedTimeSpanUserActivity, from: Instant?, until: Instant?): TimeSpan? {
return execute { database ->
throttler.commitChanges(false)
val longestActivityStatement = database.prepareStatement(
"""SELECT activity_id, started_at, ended_at FROM timespanUserActivity
|WHERE activity_id = ? AND (julianday(started_at) >= julianday(?) AND julianday(?) <= julianday(ended_at))
|ORDER BY (julianday(ended_at) - julianday(started_at)) DESC LIMIT ?""".trimMargin(),
ObjectBinderFactory.create4<String, String, String, Int>()
)
longestActivityStatement.binder.bind(activity.id, InstantUtils.formatForDatabase(from ?: InstantUtils.SomeTimeAgo),
InstantUtils.formatForDatabase(until ?: InstantUtils.NowButABitLater), 1)
longestActivityStatement.executeQuery().let {
val activityId = it.getString(0) ?: error("Required column activityId was not found")
val startedAt = it.getString(1) ?: error("Required column startedAt was not found")
val endedAt = it.getString(2) ?: error("Required column endedAt was not found")
TimeSpan(activity, activityId, InstantUtils.fromString(startedAt), InstantUtils.fromString(endedAt))
}
}
}
/**
* Some events may know when they end, so it's more preferable to use this method
*
* @param activity
* @param kind
* @param canBeStale if true, database will end event when coroutine scope (=application) dies
*/
override suspend fun submitManual(activity: DatabaseBackedTimeSpanUserActivity,
id: String,
kind: TimeSpanUserActivityDatabaseManualKind,
canBeStale: Boolean,
extra: Map<String, String>?,
moment: Instant?) {
throttler.submitManual(activity, id, kind, canBeStale, moment, extra)
}
override suspend fun endAllEvents(): Int {
return execute { db ->
val stmt = db.prepareStatement(
"""
UPDATE timespanUserActivity
SET ended_at = ?,
is_finished = ?
""".trimIndent(),
ObjectBinderFactory.create2<String, Int>(),
)
val dbEndedAt = InstantUtils.formatForDatabase(InstantUtils.Now)
val dbIsFinished = BooleanUtils.formatForDatabase(true)
stmt.binder.bind(dbEndedAt, dbIsFinished)
stmt.executeUpdate()
return@execute db.affectedRows()
} ?: 0
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
return execute { db ->
val stmt = db.prepareStatement(
"DELETE FROM timespanUserActivity WHERE activity_id = ?",
ObjectBinderFactory.create1<String>(),
)
stmt.binder.bind(activity.id)
stmt.executeUpdate()
return@execute db.affectedRows()
} ?: 0
}
override suspend fun endEventInternal(
databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?,
): Int {
val dbStartedAt = InstantUtils.formatForDatabase(startedAt)
val dbEndedAt = InstantUtils.formatForDatabase(endedAt)
val dbIsFinished = BooleanUtils.formatForDatabase(isFinished)
return execute { database, metadata ->
if (databaseId != null) {
val endEventUpdateStatement = database.prepareStatement(
"""
UPDATE timespanUserActivity
SET ended_at = ?,
is_finished = ?
WHERE id = ?
""".trimIndent(),
ObjectBinderFactory.create3<String, Int, Int>(),
)
endEventUpdateStatement.binder.bind(dbEndedAt, dbIsFinished, databaseId)
endEventUpdateStatement.executeUpdate()
// Return the current ID
databaseId
}
else {
val endEventInsertStatement = database.prepareStatement(
"""
INSERT INTO timespanUserActivity (
activity_id,
ide_id,
started_at,
ended_at,
is_finished,
extra
)
VALUES (?, ?, ?, ?, ?, ?)
RETURNING id;
""".trimIndent(),
ObjectBinderFactory.create6<String, Int, String, String, Int, String?>(),
)
val extraString = extra?.let { formatString(it) }
endEventInsertStatement.binder.bind(
activity.id,
IdService.getInstance().getDatabaseId(metadata),
dbStartedAt,
dbEndedAt,
dbIsFinished,
extraString,
)
// Return inserted ID
endEventInsertStatement
.executeQuery()
.getInt(0)
}
} ?: error("Could not execute because of null from database")
}
/**
* Use this method if you don't know when your event will end. Feel free to call this method as much as you want.
* Event is considered to be finished if no updates were submitted in +-2 minutes
*
* @param activity
* @param id
* @param extra the first submitted [extra] will be used
*/
override suspend fun submitPeriodicEvent(activity: WritableDatabaseBackedTimeSpanUserActivity, id: String, extra: Map<String, String>?) {
throttler.submitPeriodic(activity, id, activity.canBeStale, extra)
}
override suspend fun cancel(activity: WritableDatabaseBackedTimeSpanUserActivity, id: String) {
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
SqliteLazyInitializedDatabase.getInstance().executeBeforeConnectionClosed(action)
}
override suspend fun <T> execute(action: suspend (initDb: SqliteConnection) -> T): T? {
return SqliteLazyInitializedDatabase.getInstanceAsync().execute(action)
}
private suspend fun <T> execute(action: suspend (initDb: SqliteConnection, metadata: SqliteDatabaseMetadata) -> T): T? {
return SqliteLazyInitializedDatabase.getInstanceAsync().execute(action)
}
override val tableName: String = "timespanUserActivity"
}
interface IReadOnlyTimeSpanUserActivityDatabase {
suspend fun getLongestActivity(activity: DatabaseBackedTimeSpanUserActivity, from: Instant?, until: Instant?): TimeSpan?
}
interface ITimeSpanUserActivityDatabase : IReadOnlyTimeSpanUserActivityDatabase {
suspend fun submitManual(activity: DatabaseBackedTimeSpanUserActivity,
id: String,
kind: TimeSpanUserActivityDatabaseManualKind,
canBeStale: Boolean,
extra: Map<String, String>? = null,
moment: Instant?)
suspend fun submitPeriodicEvent(activity: WritableDatabaseBackedTimeSpanUserActivity, id: String, extra: Map<String, String>? = null)
suspend fun cancel(activity: WritableDatabaseBackedTimeSpanUserActivity, id: String)
}
internal interface IInternalTimeSpanUserActivityDatabase {
suspend fun endAllEvents(): Int
suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int
suspend fun endEventInternal(
databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>? = null,
): Int
fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit)
}
@@ -1,207 +0,0 @@
package com.intellij.ae.database.core.dbs.timespan
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.activities.toKey
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.openapi.diagnostic.logger
import kotlinx.coroutines.*
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.jetbrains.annotations.TestOnly
import java.time.Duration
import java.time.Instant
import kotlin.time.Duration.Companion.minutes
private val logger = logger<TimeSpanUserActivityDatabaseThrottler>()
internal class TimeSpanUserActivityDatabaseThrottler(cs: CoroutineScope,
private val database: IInternalTimeSpanUserActivityDatabase,
private val updatePause: kotlin.time.Duration = 4.minutes,
private val eventTtl: Duration = Duration.ofMinutes(2L),
runBackgroundUpdater: Boolean = true) {
private val events = HashMap<String, EventDescriptor>()
private val eventsLock = Mutex()
private val updaterJob: Job? = if (runBackgroundUpdater) startBackgroundUpdater(cs) else null
init {
/*cs.launch {
submitDanglingEvents()
}*/
database.executeBeforeConnectionClosed { isFinal ->
if (isFinal) {
commitChanges(true)
commitStaleEvents()
}
}
}
private fun startBackgroundUpdater(cs: CoroutineScope): Job {
return cs.launch {
while (isActive) {
delay(updatePause)
commitChanges(false)
}
}
}
@TestOnly
fun cancelBackgroundUpdate() {
updaterJob?.cancel("in test")
}
private suspend fun submitDanglingEvents() {
eventsLock.withLock {
val affected = database.endAllEvents()
logger.warn("Submitted ${affected} dangling events")
}
}
/**
* @return null if submission is incorrect, activity start Instant if kind=TimeSpanUserActivityDatabaseManualKind.Start, activity end Instant if kind=TimeSpanUserActivityDatabaseManualKind.End
*/
suspend fun submitManual(activity: DatabaseBackedTimeSpanUserActivity,
id: String,
kind: TimeSpanUserActivityDatabaseManualKind,
canBeStale: Boolean,
moment: Instant?,
extra: Map<String, String>?): Instant? {
return eventsLock.withLock {
when (kind) {
TimeSpanUserActivityDatabaseManualKind.Start -> {
if (events.containsKey(activity.toKey(id))) {
logger.warn("Already logged ${activity.toKey(id)}, it won't be overwritten")
null
}
else {
logger.info("Starting activity ${activity.id} with id $id")
val theMoment = moment ?: InstantUtils.Now
events[activity.toKey(id)] = EventDescriptor.manual(activity, id, canBeStale, theMoment, extra)
theMoment
}
}
TimeSpanUserActivityDatabaseManualKind.End -> {
val event = events[activity.toKey(id)]
if (event == null) {
logger.info("Tried to end activity ${activity.toKey(id)} that wasn't started")
null
}
else {
logger.info("Ending activity ${activity.id} with id $id")
val theMoment = moment ?: InstantUtils.Now
endEvent(event, theMoment)
theMoment
}
}
}
}
}
suspend fun cancel(activity: DatabaseBackedTimeSpanUserActivity, id: String) {
eventsLock.withLock {
val activityId = activity.toKey(id)
if (database.removeEvent(activity) == 0) {
logger.info("Attempt to remove non-existent activity $activityId")
}
if (events.remove(activityId) == null) {
logger.info("Tried to cancel activity ${activity.toKey(id)} that wasn't started")
}
}
}
suspend fun submitPeriodic(
activity: DatabaseBackedTimeSpanUserActivity,
id: String,
canBeStale: Boolean,
extra: Map<String, String>?
): Instant? {
return eventsLock.withLock {
val activityId = activity.toKey(id)
val event = events[activityId]
val now = InstantUtils.Now
if (event?.isPeriodic == false) {
logger.warn("Tried to log activity (${activityId}) as periodic, but it was already submitted as manual therefore it won't be saved")
return@withLock null
}
val value = event ?: EventDescriptor.periodic(activity, id, now, now, canBeStale, extra)
value.endAt = now
events[activityId] = value
return@withLock value.startedAt
}
}
// should run under [eventsLock]
private suspend fun endEvent(eventDescriptor: EventDescriptor, endedAtInit: Instant, isFinished: Boolean = true) {
val endedAt = if (eventDescriptor.startedAt == endedAtInit) {
// We assume that all events lasted at least for a bit
endedAtInit + Duration.ofSeconds(eventTtl.seconds / 2)
}
else {
endedAtInit
}
// Remove event from map if it's finished
if (isFinished && events.remove(eventDescriptor.toKey()) == null) {
logger.warn("Tried to endEvent ${eventDescriptor.toKey()} which was already finished")
}
if (eventDescriptor.startedAt > endedAt) {
logger.warn("Event ${eventDescriptor.toKey()} started after it ended. It would be discarded")
events.remove(eventDescriptor.toKey())
return
}
eventDescriptor.databaseId = database.endEventInternal(
eventDescriptor.databaseId,
eventDescriptor.activity,
eventDescriptor.startedAt,
endedAt,
isFinished,
eventDescriptor.extra,
)
}
private suspend fun commitStaleEvents() {
eventsLock.withLock {
val values = ArrayList(events.values)
for (event in values) {
if (!event.canBeStale && event.endAt == null) {
logger.warn("Event ${event.toKey()} wasn't marked as canBeStale, but the end wasn't reported. It will be still saved anyway")
}
else {
logger.info("Submitting stale event ${event.toKey()}")
}
endEvent(event, InstantUtils.Now)
}
}
}
internal suspend fun commitChanges(isFinal: Boolean) {
eventsLock.withLock {
val values = ArrayList(events.values)
for (descriptor in values) {
val now = InstantUtils.Now
if (descriptor.isPeriodic) {
val threshold = now - eventTtl
val endAt = descriptor.endAt
if (endAt == null) {
logger.warn("Attempt to commit periodic event (${descriptor.toKey()}) with no end time, skipping")
continue
}
endEvent(descriptor, endAt, isFinished = endAt < threshold || isFinal)
} else {
endEvent(descriptor, now, isFinished = false)
}
}
}
}
}
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.dbs.timespan;
import org.jetbrains.annotations.ApiStatus;
@@ -1,123 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core
import com.intellij.ae.database.core.activities.UserActivity
import com.intellij.idea.AppMode
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.diagnostic.logger
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import org.jetbrains.sqlite.SqliteResultSet
object AEEventUtils
/**
* Wrap the call to your update() function in user activity into this method,
* so activity won't be updated when not needed (right now – in unit test mode)
*
* @param activity user activity that will be updated
* @param action code that updates activity, is a suspend lambda. [activity] is passed into lambda
*/
inline fun <T : UserActivity> CoroutineScope.runUpdateEvent(activity: T, crossinline action: suspend (T) -> Unit) {
if (ApplicationManager.getApplication().isUnitTestMode) return
if (AppMode.isRemoteDevHost()) return // sorry, no remote dev for now
launch {
try {
action(activity)
}
catch (t: CancellationException) {
throw t
}
catch (t: Throwable) {
logger<AEEventUtils>().error(t)
}
}
}
internal fun formatString(params: Map<String, String>): String {
return buildString {
append("{")
val totalSize = params.size
for (param in params.asIterable().withIndex()) {
escape(param.value.key, this)
append(": ")
escape(param.value.value, this)
if (totalSize-1 != param.index) {
append(", ")
}
}
append("}")
}
}
fun <K0, V0, K, V> createMap(getKey: (SqliteResultSet) -> K0, getValue: (SqliteResultSet) -> V0,
validate: (K0, V0) -> Boolean,
transformKey: (K0) -> K, transformValue: (V0) -> V,
validateTransformed: (K, V) -> Boolean,
res: SqliteResultSet): Map<K, V> {
val map = mutableMapOf<K, V>()
while (res.next()) {
val k0 = getKey(res)
val v0 = getValue(res)
if (!validate(k0, v0)) continue
val k = transformKey(k0)
val v = transformValue(v0)
if (!validateTransformed(k, v)) continue
map[k] = v
}
return map
}
private val REPLACEMENT_CHARS = arrayOfNulls<String>(128).apply {
for (i in 0..31) {
this[i] = String.format("\\u%04x", i)
}
this['"'.code] = "\\\""
this['\\'.code] = "\\\\"
this['\t'.code] = "\\t"
this['\b'.code] = "\\b"
this['\n'.code] = "\\n"
this['\r'.code] = "\\r"
this['\u000c'.code] = "\\f"
}
// copy-paste of [org.jetbrains.io.JsonUtil], not to depend on platform-impl
private fun escape(value: CharSequence, sb: StringBuilder) {
val length = value.length
sb.ensureCapacity(sb.length + length + 2)
sb.append('"')
var last = 0
for (i in 0 until length) {
val c = value[i]
var replacement: String?
if (c.code < 128) {
replacement = REPLACEMENT_CHARS.get(c.code)
if (replacement == null) {
continue
}
}
else if (c == '\u2028') {
replacement = "\\u2028"
}
else if (c == '\u2029') {
replacement = "\\u2029"
}
else {
continue
}
if (last < i) {
sb.append(value, last, i)
}
sb.append(replacement)
last = i + 1
}
if (last < length) {
sb.append(value, last, length)
}
sb.append('"')
}
@@ -1,11 +0,0 @@
package com.intellij.ae.database.core.models
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import java.time.Instant
data class TimeSpan(
val activity: DatabaseBackedTimeSpanUserActivity,
val id: String,
val start: Instant,
val end: Instant
)
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.models;
import org.jetbrains.annotations.ApiStatus;
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core;
import org.jetbrains.annotations.ApiStatus;
@@ -1,7 +0,0 @@
package com.intellij.ae.database.core.utils
object BooleanUtils {
fun formatForDatabase(v: Boolean): Int {
return if (v) { 1 } else { 0 }
}
}
@@ -1,17 +0,0 @@
package com.intellij.ae.database.core.utils
import java.time.Instant
import java.time.temporal.ChronoUnit
object InstantUtils {
val SomeTimeAgo: Instant = Instant.ofEpochSecond(0)
val Now: Instant get() = Instant.now()
val NowButABitLater: Instant get() = Now.plusSeconds(20 * 60)
val StartOfDay: Instant get() = Now.truncatedTo(ChronoUnit.DAYS)
val WeekAgo: Instant get() = Now.minus(7, ChronoUnit.DAYS)
val WeekAgoStartOfDay: Instant get() = WeekAgo.truncatedTo(ChronoUnit.DAYS)
fun formatForDatabase(instant: Instant): String = instant.truncatedTo(ChronoUnit.SECONDS).toString()
fun fromString(str: String): Instant = Instant.parse(str)
}
@@ -1,5 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
@ApiStatus.Internal
package com.intellij.ae.database.core.utils;
import org.jetbrains.annotations.ApiStatus;
@@ -1,13 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.tests.dbs
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import kotlinx.coroutines.CoroutineScope
@Service
class AETestCoroutineScope(private val cs: CoroutineScope) {
companion object {
fun getScope() = service<AETestCoroutineScope>().cs
}
}
@@ -1,110 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.tests.dbs
import com.intellij.ae.database.core.IdService
import com.intellij.ae.database.core.dbs.SqliteLazyInitializedDatabase
import com.intellij.openapi.util.io.FileUtil
import com.intellij.testFramework.common.timeoutRunBlocking
import com.intellij.testFramework.fixtures.BasePlatformTestCase
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.jetbrains.sqlite.EmptyBinder
import org.jetbrains.sqlite.ObjectBinderFactory
import org.junit.Assert
class SqliteLazyInitializedDatabaseTest : BasePlatformTestCase() {
/**
* Read/write inside once sql connection
*/
fun testBasicReadWrite(): Unit = timeoutRunBlocking {
System.setProperty("ae.database.path", FileUtil.generateRandomTemporaryPath("ae", ".db").absolutePath)
val db1 = SqliteLazyInitializedDatabase(this)
withContext(Dispatchers.Default) {
db1.execute { initDb ->
val writeStatement = initDb.prepareStatement("insert into ide(ide_id, machine_id, family) values (?, ?, ?)",
ObjectBinderFactory.create3<String, String, String>()).apply {
binder.bind("test1", "test2", "test3")
}
writeStatement.executeUpdate()
val readStatement = initDb.prepareStatement("SELECT * FROM ide WHERE ide_id IS (?)", ObjectBinderFactory.create1<String>()).apply {
binder.bind("test1")
}
val resultSet = readStatement.executeQuery()
var hits = 0
while (resultSet.next()) {
++hits
// column zero is database id, not important here
val machineId = resultSet.getString(1)
val ideId = resultSet.getString(2)
val family = resultSet.getString(3)
Assert.assertArrayEquals(arrayOf("test1", "test2", "test3"), arrayOf(ideId, machineId, family))
}
Assert.assertEquals(1, hits)
}
db1.closeDatabase()
}
}
/**
* Write to DB, close it, open, read
*/
fun testReadWriteAfterClosing(): Unit = timeoutRunBlocking {
System.setProperty("ae.database.path", FileUtil.generateRandomTemporaryPath("ae", ".db").absolutePath)
withContext(Dispatchers.Default) {
val db1 = SqliteLazyInitializedDatabase(this)
db1.execute { initDb ->
val writeStatement = initDb.prepareStatement("insert into ide(ide_id, machine_id, family) values (?, ?, ?)",
ObjectBinderFactory.create3<String, String, String>()).apply {
binder.bind("test1", "test2", "test3")
}
writeStatement.executeUpdate()
}
db1.closeDatabase()
val db2 = SqliteLazyInitializedDatabase(this)
db2.execute { initDb ->
val readStatement = initDb.prepareStatement("SELECT * FROM ide WHERE ide_id IS (?)", ObjectBinderFactory.create1<String>()).apply {
binder.bind("test1")
}
val resultSet = readStatement.executeQuery()
var hits = 0
while (resultSet.next()) {
++hits
// column zero is database id, not important here
val machineId = resultSet.getString(1)
val ideId = resultSet.getString(2)
val family = resultSet.getString(3)
Assert.assertArrayEquals(arrayOf("test1", "test2", "test3"), arrayOf(ideId, machineId, family))
}
Assert.assertEquals(1, hits)
}
db2.closeDatabase()
}
}
fun testProperMetadata(): Unit = timeoutRunBlocking {
System.setProperty("ae.database.path", FileUtil.generateRandomTemporaryPath("ae", ".db").absolutePath)
withContext(Dispatchers.Default) {
val db1 = SqliteLazyInitializedDatabase(this)
db1.execute { initDb, metadata ->
Assert.assertEquals(1, metadata.ideId)
val entries = initDb.prepareStatement("SELECT * FROM ide", EmptyBinder).executeQuery()
var hits = 0
while (entries.next()) {
++hits
// column zero is database id, not important here
val machineId = entries.getString(1)
val ideId = entries.getString(2)
val family = entries.getString(3)
Assert.assertArrayEquals(arrayOf(IdService.getInstance().id, IdService.getInstance().machineId, IdService.getInstance().ideCode), arrayOf(ideId, machineId, family))
}
Assert.assertEquals(1, hits)
}
db1.closeDatabase()
}
}
}
@@ -1,34 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.tests.dbs.counter
import com.intellij.ae.database.core.activities.DatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.dbs.counter.CounterUserActivityDatabase
import com.intellij.ae.database.core.tests.dbs.runDatabaseLayerTest
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.testFramework.fixtures.BasePlatformTestCase
import kotlinx.coroutines.CoroutineScope
import org.junit.Assert
class CounterUserActivityDatabaseTest : BasePlatformTestCase() {
private val databaseFactory = { cs: CoroutineScope ->
CounterUserActivityDatabase(cs)
}
fun testGetActivitySum() = runDatabaseLayerTest(databaseFactory) { it ->
val item1 = 13
val item2 = 1989
val result = item1+item2
val activity = createActivity()
it.submitDirect(activity, item1, InstantUtils.Now, null)
it.submitDirect(activity, item2, InstantUtils.Now, null)
val fromDb = it.getActivitySum(activity, InstantUtils.SomeTimeAgo, InstantUtils.NowButABitLater)
Assert.assertEquals(result, fromDb)
}
private fun createActivity() = object : DatabaseBackedCounterUserActivity() {
override val id = "testActivity"
}
}
@@ -1,196 +0,0 @@
package com.intellij.ae.database.core.tests.dbs.counter
import com.intellij.ae.database.core.activities.DatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.dbs.counter.CounterUserActivityDatabaseThrottler
import com.intellij.ae.database.core.dbs.counter.IInternalCounterUserActivityDatabase
import com.intellij.testFramework.common.timeoutRunBlocking
import com.intellij.testFramework.fixtures.BasePlatformTestCase
import kotlinx.coroutines.coroutineScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import org.junit.Assert
import java.time.Duration
import java.time.Instant
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds
class CounterUserActivityDatabaseThrottlerTest : BasePlatformTestCase() {
private fun runCounterUserActivityTest(flushImmediately: Boolean, submit: suspend (CounterUserActivityDatabaseThrottler, DatabaseBackedCounterUserActivity) -> Unit, asserts: (diff: Int, instant: Instant, lock: Mutex) -> Unit) {
val myActivity = object : DatabaseBackedCounterUserActivity() {
override val id: String get() = "testActivity"
}
val lock = Mutex()
val onDatabaseDeath = mutableListOf<suspend (Boolean) -> Unit>()
val fakeDatabase = object : IInternalCounterUserActivityDatabase {
override suspend fun submitDirect(activity: DatabaseBackedCounterUserActivity,
diff: Int,
instant: Instant,
extra: Map<String, String>?) {
Assert.assertTrue(myActivity === activity)
asserts(diff, instant, lock)
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
onDatabaseDeath.add(action)
}
}
timeoutRunBlocking {
val throttlerCoroutine = launch {
CounterUserActivityDatabaseThrottler(this, fakeDatabase, 2.seconds).apply {
submit(this, myActivity)
/*
// TODO: if this is uncommented, tests work faster, but different from how it works in prod
if (flushImmediately) {
commitChanges()
}
*/
}
}
val submissionTime = Instant.now()
// The first lock() call locks, the second one awaits
lock.lock()
lock.lock()
println("Took ${Duration.between(submissionTime, Instant.now()).seconds} seconds to submit event")
for (task in onDatabaseDeath) {
task(true)
}
throttlerCoroutine.cancel()
}
}
fun testOneEvent() {
val newValue = 13
val submit: suspend (CounterUserActivityDatabaseThrottler, DatabaseBackedCounterUserActivity) -> Unit = { thr, act ->
thr.submit(act, newValue)
}
val verify: (Int, Instant, Mutex) -> Unit = { diff, instant, lock ->
Assert.assertEquals(newValue, diff)
lock.unlock()
}
runCounterUserActivityTest(true, submit, verify)
}
fun testSeveralEvents() {
val firstValue = 13
val secondValue = 1989
val result = firstValue+secondValue
val submit: suspend (CounterUserActivityDatabaseThrottler, DatabaseBackedCounterUserActivity) -> Unit = { thr, act ->
thr.submit(act, firstValue)
thr.submit(act, secondValue)
}
val verify: (Int, Instant, Mutex) -> Unit = { diff, instant, lock ->
Assert.assertEquals(result, diff)
lock.unlock()
}
runCounterUserActivityTest(true, submit, verify)
}
fun testSeveralEventsNegative() {
val firstValue = 13
val secondValue = -1989
val result = firstValue+secondValue
val submit: suspend (CounterUserActivityDatabaseThrottler, DatabaseBackedCounterUserActivity) -> Unit = { thr, act ->
thr.submit(act, firstValue)
thr.submit(act, secondValue)
}
val verify: (Int, Instant, Mutex) -> Unit = { diff, instant, lock ->
Assert.assertEquals(result, diff)
lock.unlock()
}
runCounterUserActivityTest(true, submit, verify)
}
fun testSeveralValuesSeparate() {
val firstValue = 13
val secondValue = -1989
val submit: suspend (CounterUserActivityDatabaseThrottler, DatabaseBackedCounterUserActivity) -> Unit = { thr, act ->
thr.submit(act, firstValue)
coroutineScope {
launch {
delay(2.seconds)
thr.submit(act, secondValue)
}
}
}
var hit = 0
val verify: (Int, Instant, Mutex) -> Unit = { diff, instant, lock ->
when (++hit) {
1 -> {
Assert.assertEquals(firstValue, diff)
}
2 -> {
Assert.assertEquals(secondValue, diff)
lock.unlock()
}
else -> {
fail("Got hit $hit, expected >=2")
}
}
}
runCounterUserActivityTest(true, submit, verify)
}
fun testShowdown() {
val myActivity1 = object : DatabaseBackedCounterUserActivity() {
override val id: String get() = "testActivity1"
}
val myActivity2 = object : DatabaseBackedCounterUserActivity() {
override val id: String get() = "testActivity2"
}
val onDatabaseDeath = mutableListOf<suspend (Boolean) -> Unit>()
val expected = mapOf(
myActivity1.id to 13,
myActivity2.id to 13*2
)
val endedEvents = mutableMapOf<String, Int>()
timeoutRunBlocking {
val fakeDatabase = object : IInternalCounterUserActivityDatabase {
override suspend fun submitDirect(activity: DatabaseBackedCounterUserActivity,
diff: Int,
instant: Instant,
extra: Map<String, String>?) {
endedEvents[activity.id] = diff
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
onDatabaseDeath.add(action)
}
}
val submissionLock = Mutex(true)
val throttlerCoroutine = launch {
CounterUserActivityDatabaseThrottler(this, fakeDatabase, 10.minutes).apply {
submit(myActivity1, expected[myActivity1.id]!!)
submit(myActivity2, (expected[myActivity2.id]!!)/2)
submit(myActivity2, (expected[myActivity2.id]!!)/2)
submissionLock.unlock()
}
}
submissionLock.lock()
for (task in onDatabaseDeath) {
task(true)
}
throttlerCoroutine.cancel()
}
for (expectedValue in expected) {
Assert.assertTrue(endedEvents.contains(expectedValue.key))
Assert.assertEquals(expectedValue.value, endedEvents[expectedValue.key])
}
}
}
@@ -1,37 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.tests.dbs
import com.intellij.ae.database.core.dbs.IUserActivityDatabaseLayer
import com.intellij.ae.database.core.dbs.SqliteLazyInitializedDatabase
import com.intellij.testFramework.common.timeoutRunBlocking
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
/**
* Runs a test for [IUserActivityDatabaseLayer]
*/
fun <T : IUserActivityDatabaseLayer> runDatabaseLayerTest(
dbFactory: (CoroutineScope) -> T,
action: suspend (T) -> Unit,
) = runInitializedDatabaseTestInternal { cs, db -> action(dbFactory(cs)) }
fun <T : IUserActivityDatabaseLayer> runDatabaseLayerTest(
dbFactory: (CoroutineScope) -> T,
action: suspend (T, SqliteLazyInitializedDatabase) -> Unit,
) = runInitializedDatabaseTestInternal { cs, db -> action(dbFactory(cs), db) }
fun <T : IUserActivityDatabaseLayer> runDatabaseLayerTest(
dbFactory: (CoroutineScope) -> T,
action: suspend (T, SqliteLazyInitializedDatabase, CoroutineScope) -> Unit,
) = runInitializedDatabaseTestInternal { cs, db -> action(dbFactory(cs), db, cs) }
private fun runInitializedDatabaseTestInternal(action: suspend (CoroutineScope, SqliteLazyInitializedDatabase) -> Unit) {
timeoutRunBlocking {
withContext(Dispatchers.IO) {
val db = SqliteLazyInitializedDatabase.getInstanceAsync()
action(this, db)
db.closeDatabase()
}
}
}
@@ -1,140 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.core.tests.dbs.timespan
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.SqliteLazyInitializedDatabase
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabase
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import com.intellij.ae.database.core.tests.dbs.runDatabaseLayerTest
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.testFramework.fixtures.BasePlatformTestCase
import kotlinx.coroutines.CoroutineScope
import org.jetbrains.sqlite.ObjectBinderFactory
import org.junit.jupiter.api.Assertions
import java.time.Instant
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toJavaDuration
class TimeSpanUserActivityDatabaseTest : BasePlatformTestCase() {
private val databaseFactory = { cs: CoroutineScope ->
TimeSpanUserActivityDatabase(cs).also { it -> it.cancelBackgroundUpdate() }
}
private data class TestEvent(
val dbId: Int,
val activityId: String,
val startedAt: String,
val endAt: String,
val isFinished: Boolean,
)
private suspend fun SqliteLazyInitializedDatabase.getActivityEvents(activityId: String): List<TestEvent> {
return execute { db ->
val stmt = db.prepareStatement(
"""
SELECT id, activity_id, started_at, ended_at, is_finished
FROM "timespanUserActivity"
WHERE activity_id = ?
""".trimIndent(),
ObjectBinderFactory.create1<String>(),
)
val result = mutableListOf<TestEvent>()
stmt.binder.bind(activityId)
stmt.executeQuery().let {
while (it.next()) {
result.add(TestEvent(
it.getInt(0),
it.getString(1)!!,
it.getString(2)!!,
it.getString(3)!!,
it.getBoolean(4),
))
}
}
result
} ?: error("Unexpected null in getActivityEvents")
}
fun testEndAllEvents() = runDatabaseLayerTest(databaseFactory) { db, initDb, cs ->
val testActivity1 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testEndAllActivity1"
}
val testActivity2 = object : WritableDatabaseBackedTimeSpanUserActivity() {
override val canBeStale: Boolean = true
override val id: String get() = "testEndAllActivity2"
}
db.submitManual(
testActivity1,
id=testActivity1.id,
canBeStale=false,
kind=TimeSpanUserActivityDatabaseManualKind.Start,
moment= Instant.now()
)
db.submitPeriodicEvent(testActivity2, id=testActivity2.id)
Assertions.assertTrue(initDb.getActivityEvents(testActivity1.id).all { !it.isFinished })
Assertions.assertTrue(initDb.getActivityEvents(testActivity2.id).all { !it.isFinished })
db.endAllEvents()
Assertions.assertTrue(initDb.getActivityEvents(testActivity1.id).all { it.isFinished })
Assertions.assertTrue(initDb.getActivityEvents(testActivity2.id).all { it.isFinished })
}
fun testRemoveEvent() = runDatabaseLayerTest(databaseFactory) { db, initDb, cs ->
val testActivity1 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testRemoveActivity1"
}
db.endEventInternal(
null,
testActivity1,
InstantUtils.Now,
InstantUtils.NowButABitLater,
false,
)
Assertions.assertEquals(1, initDb.getActivityEvents(testActivity1.id).size)
db.removeEvent(testActivity1)
Assertions.assertEquals(0, initDb.getActivityEvents(testActivity1.id).size)
}
fun testEndEventInternal() = runDatabaseLayerTest(databaseFactory) { db, initDb, cs ->
val testActivity = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testEndActivity1"
}
// Assert event stored correctly on first call
val initialStart = InstantUtils.Now
val initialEnd = InstantUtils.NowButABitLater
val insertedId = db.endEventInternal(null, testActivity, initialStart, initialEnd, isFinished = false)
// Assert event stored correctly on first call
val storedEvents1 = initDb.getActivityEvents(testActivity.id)
Assertions.assertEquals(1, storedEvents1.size)
val storedEvent1 = storedEvents1.first()
Assertions.assertEquals(insertedId, storedEvent1.dbId)
Assertions.assertEquals(testActivity.id, storedEvent1.activityId)
Assertions.assertFalse(storedEvent1.isFinished)
Assertions.assertEquals(InstantUtils.formatForDatabase(initialStart), storedEvent1.startedAt)
Assertions.assertEquals(InstantUtils.formatForDatabase(initialEnd), storedEvent1.endAt)
val updatedEnd = initialEnd + 10.seconds.toJavaDuration()
val updatedId = db.endEventInternal(insertedId, testActivity, initialEnd, updatedEnd, isFinished = true)
// Assert id didn't change
Assertions.assertEquals(insertedId, updatedId)
// Assert event updated correctly on consequent calls
val storedEvents2 = initDb.getActivityEvents(testActivity.id)
Assertions.assertEquals(1, storedEvents2.size)
val storedEvent2 = storedEvents2.first()
Assertions.assertEquals(updatedId, storedEvent2.dbId)
Assertions.assertEquals(testActivity.id, storedEvent2.activityId)
Assertions.assertTrue(storedEvent2.isFinished)
Assertions.assertEquals(InstantUtils.formatForDatabase(initialStart), storedEvent2.startedAt)
Assertions.assertEquals(InstantUtils.formatForDatabase(updatedEnd), storedEvent2.endAt)
}
}
@@ -1,417 +0,0 @@
package com.intellij.ae.database.core.tests.dbs.timespan
import com.intellij.ae.database.core.activities.DatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.timespan.IInternalTimeSpanUserActivityDatabase
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseThrottler
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.testFramework.common.timeoutRunBlocking
import com.intellij.testFramework.fixtures.BasePlatformTestCase
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.junit.Assert
import java.time.Duration
import java.time.Instant
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
import kotlin.time.Duration.Companion.seconds
import kotlin.time.toKotlinDuration
class TimeSpanUserActivityDatabaseThrottlerTest : BasePlatformTestCase() {
private val updatePause = 4.seconds
private val eventTtl = Duration.ofSeconds(2L)
private fun runCounterUserActivityTest(
submit: suspend (TimeSpanUserActivityDatabaseThrottler, DatabaseBackedTimeSpanUserActivity) -> Unit,
asserts: (
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
lock: Mutex
) -> Unit) {
val myActivity = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity"
}
val lock = Mutex()
val onDatabaseDeath = mutableListOf<suspend () -> Unit>()
val fakeDatabase = object : IInternalTimeSpanUserActivityDatabase {
override suspend fun endEventInternal(databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?): Int {
asserts(activity, startedAt, endedAt, isFinished, lock)
return 0
}
override suspend fun endAllEvents(): Int {
return 0
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
error("Called removeEvent which doesn't work in current test")
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
error("Called executeBeforeConnectionClosed which doesnt work in current test")
}
}
timeoutRunBlocking {
val throttlerCoroutine = launch {
TimeSpanUserActivityDatabaseThrottler(this, fakeDatabase, updatePause, eventTtl).apply {
submit(this, myActivity)
}
}
val submissionTime = Instant.now()
lock.withLock {
println("Took ${Duration.between(submissionTime, Instant.now()).seconds} seconds to submit event")
for (task in onDatabaseDeath) {
task()
}
throttlerCoroutine.cancel()
}
}
}
fun testOnePeriodicNoUpdate() {
var submittedAt: Instant = InstantUtils.Now
val submit: suspend (TimeSpanUserActivityDatabaseThrottler, DatabaseBackedTimeSpanUserActivity) -> Unit = { thr, activity ->
submittedAt = thr.submitPeriodic(activity, "testing", false, null)!!
}
val verify: (DatabaseBackedTimeSpanUserActivity, Instant, Instant, Boolean, Mutex) -> Unit = { activity, start, end, isFinished, lock ->
Assert.assertEquals(submittedAt, start)
Assert.assertEquals(submittedAt + Duration.ofSeconds(eventTtl.seconds / 2), end)
Assert.assertTrue(isFinished)
lock.unlock()
}
runCounterUserActivityTest(submit, verify)
}
fun testOnePeriodicWithUpdate() {
var submittedAt = InstantUtils.Now
val submit: suspend (TimeSpanUserActivityDatabaseThrottler, DatabaseBackedTimeSpanUserActivity) -> Unit = { thr, activity ->
submittedAt = thr.submitPeriodic(activity, "testing", false, null)!!
delay(1.seconds)
thr.submitPeriodic(activity, "testing", false, null)
}
val verify: (DatabaseBackedTimeSpanUserActivity, Instant, Instant, Boolean, Mutex) -> Unit = { activity, start, end, isFinished, lock ->
Assert.assertEquals(submittedAt, start)
Assert.assertNotEquals(submittedAt + Duration.ofSeconds(eventTtl.seconds / 2), end)
Assert.assertTrue(isFinished)
lock.unlock()
}
runCounterUserActivityTest(submit, verify)
}
fun testResubmitPeriodicAsManualFails() {
val submit: suspend (TimeSpanUserActivityDatabaseThrottler, DatabaseBackedTimeSpanUserActivity) -> Unit = { thr, activity ->
val periodicSubmitTime = thr.submitPeriodic(activity, "testing", false, null)
Assert.assertNotNull(periodicSubmitTime)
val manualSubmitTime = thr.submitManual(
activity,
"testing",
TimeSpanUserActivityDatabaseManualKind.Start,
canBeStale = false,
moment = InstantUtils.Now,
extra = null,
)
Assert.assertNull(manualSubmitTime)
}
val verify: (DatabaseBackedTimeSpanUserActivity, Instant, Instant, Boolean, Mutex) -> Unit = { _, _, _, _, _ -> }
runCounterUserActivityTest(submit, verify)
}
fun testResubmitManualAsPeriodicFails() {
val submit: suspend (TimeSpanUserActivityDatabaseThrottler, DatabaseBackedTimeSpanUserActivity) -> Unit = { thr, activity ->
val periodicSubmitTime = thr.submitPeriodic(activity, "testing", false, null)
Assert.assertNotNull(periodicSubmitTime)
val manualSubmitTime = thr.submitManual(
activity,
"testing",
TimeSpanUserActivityDatabaseManualKind.Start,
canBeStale = false,
moment = InstantUtils.Now,
extra = null,
)
Assert.assertNull(manualSubmitTime)
}
val verify: (DatabaseBackedTimeSpanUserActivity, Instant, Instant, Boolean, Mutex) -> Unit = { _, _, _, _, _ -> }
runCounterUserActivityTest(submit, verify)
}
fun testShowdown() {
val myActivity1 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity1"
}
val myActivity2 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity2"
}
val onDatabaseDeath = mutableListOf<suspend (Boolean) -> Unit>()
timeoutRunBlocking {
val endedEvents = mutableListOf<String>()
val fakeDatabase = object : IInternalTimeSpanUserActivityDatabase {
override suspend fun endAllEvents(): Int {
return 0
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
return 1
}
override suspend fun endEventInternal(databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?): Int {
endedEvents.add(activity.id)
return 0
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
onDatabaseDeath.add(action)
}
}
val submissionLock = Mutex(true)
val throttlerCoroutine = launch {
TimeSpanUserActivityDatabaseThrottler(this, fakeDatabase, 10.minutes, Duration.ofNanos(1)).apply {
submitPeriodic(myActivity1, myActivity1.id, false, null)
submitManual(myActivity2, myActivity2.id, TimeSpanUserActivityDatabaseManualKind.Start, true, InstantUtils.Now, null)
submitManual(myActivity2, myActivity2.id, TimeSpanUserActivityDatabaseManualKind.End, true, InstantUtils.NowButABitLater, null)
submissionLock.unlock()
}
}
submissionLock.lock()
for (task in onDatabaseDeath) {
task(true)
}
throttlerCoroutine.cancel()
Assert.assertTrue(endedEvents.contains(myActivity1.id))
Assert.assertTrue(endedEvents.contains(myActivity2.id))
}
}
fun testUnfinishedSaved() {
val myActivity1 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity1"
}
val myActivity2 = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity2"
}
val eventTtl = Duration.ofMillis(30)
val updateInterval = 10.milliseconds
val onDatabaseDeath = mutableListOf<suspend (Boolean) -> Unit>()
timeoutRunBlocking {
val endedEvents = mutableListOf<String>()
val savedEvents = mutableMapOf<String, Int>()
val fakeDatabase = object : IInternalTimeSpanUserActivityDatabase {
var idSeq = 0
private fun nextId(): Int {
idSeq += 1
return idSeq
}
override suspend fun endAllEvents(): Int {
return 0
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
error("This method shouldn't be called in this test")
}
override suspend fun endEventInternal(databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?): Int {
val itemId = databaseId ?: nextId()
if (isFinished) {
endedEvents.add(activity.id)
} else {
savedEvents[activity.id] = itemId
}
return itemId
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
onDatabaseDeath.add(action)
}
}
val firstSubmissionLock = Mutex(true)
val secondSubmissionLock = Mutex(true)
val assertionLock = Mutex(true)
val throttlerCoroutine = launch {
TimeSpanUserActivityDatabaseThrottler(
this,
fakeDatabase,
updateInterval,
eventTtl,
).apply {
submitPeriodic(myActivity1, myActivity1.id, false, null)
submitManual(myActivity2, myActivity2.id, TimeSpanUserActivityDatabaseManualKind.Start, true, InstantUtils.Now, null)
firstSubmissionLock.unlock()
assertionLock.lock()
submitManual(myActivity2, myActivity2.id, TimeSpanUserActivityDatabaseManualKind.End, true, InstantUtils.Now, null)
secondSubmissionLock.unlock()
}
}
firstSubmissionLock.lock()
// Wait to commit changes once
delay(updateInterval.times(1.1))
Assert.assertTrue(savedEvents.contains(myActivity1.id))
Assert.assertTrue(savedEvents.contains(myActivity2.id))
Assert.assertTrue(endedEvents.isEmpty())
val activity1DbId = savedEvents[myActivity1.id]
val activity2DbId = savedEvents[myActivity2.id]
assertionLock.unlock()
// Wait for second submission
secondSubmissionLock.lock()
// Wait for event ttl to expire
delay(eventTtl.toKotlinDuration())
// Assert both events ended
Assert.assertTrue(endedEvents.contains(myActivity1.id))
Assert.assertTrue(endedEvents.contains(myActivity2.id))
// Assert id wasn't changed (called with the same dbId)
Assert.assertEquals(activity1DbId, savedEvents[myActivity1.id])
Assert.assertEquals(activity2DbId, savedEvents[myActivity2.id])
throttlerCoroutine.cancel()
}
}
fun testDanglingEventsCommited() {
val onDatabaseDeath = mutableListOf<suspend (Boolean) -> Unit>()
timeoutRunBlocking {
var endAllEventsCalled = false
val fakeDatabase = object : IInternalTimeSpanUserActivityDatabase {
override suspend fun endAllEvents(): Int {
endAllEventsCalled = true
return 1
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
error("This method shouldn't be called in this test")
}
override suspend fun endEventInternal(databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?): Int {
error("This method shouldn't be called in this test")
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
onDatabaseDeath.add(action)
}
}
val initLock = Mutex(true)
val throttlerCoroutine = launch {
TimeSpanUserActivityDatabaseThrottler(this, fakeDatabase)
initLock.unlock()
}
initLock.withLock {
Assert.assertTrue(endAllEventsCalled)
throttlerCoroutine.cancel()
}
}
}
fun testCancelActivity() {
val activity = object : DatabaseBackedTimeSpanUserActivity() {
override val id: String get() = "testActivity"
}
timeoutRunBlocking {
val removedEvents = mutableListOf<String>()
val fakeDatabase = object : IInternalTimeSpanUserActivityDatabase {
override suspend fun endAllEvents(): Int {
return 1
}
override suspend fun removeEvent(activity: DatabaseBackedTimeSpanUserActivity): Int {
removedEvents.add(activity.id)
return 1
}
override suspend fun endEventInternal(databaseId: Int?,
activity: DatabaseBackedTimeSpanUserActivity,
startedAt: Instant,
endedAt: Instant,
isFinished: Boolean,
extra: Map<String, String>?): Int {
error("This method shouldn't be called in this test")
}
override fun executeBeforeConnectionClosed(action: suspend (isFinal: Boolean) -> Unit) {
}
}
val cancelLock = Mutex(true)
val throttlerCoroutine = launch {
val throttler = TimeSpanUserActivityDatabaseThrottler(this, fakeDatabase)
throttler.submitPeriodic(activity, activity.id, false, null)
throttler.cancel(activity, activity.id)
cancelLock.unlock()
}
Assert.assertFalse(removedEvents.contains(activity.id))
cancelLock.lock()
Assert.assertTrue(removedEvents.contains(activity.id))
throttlerCoroutine.cancel()
}
}
}
@@ -1,26 +0,0 @@
### auto-generated section `build intellij.ae.database.counters.community` start
load("@rules_jvm//:jvm.bzl", "jvm_library", "jvm_resources")
jvm_resources(
name = "ae-database-counters-community_resources",
files = glob(["resources/**/*"]),
strip_prefix = "resources"
)
jvm_library(
name = "ae-database-counters-community",
module_name = "intellij.ae.database.counters.community",
visibility = ["//visibility:public"],
srcs = glob(["src/**/*.kt", "src/**/*.java"], allow_empty = True),
deps = [
"@lib//:kotlin-stdlib",
"//platform/platform-impl:ide-impl",
"//platform/xdebugger-api:debugger",
"//platform/lang-impl",
"//platform/vcs-api:vcs",
"//plugins/feature-usage-database/core:ae-database-core",
"//platform/sqlite",
],
runtime_deps = [":ae-database-counters-community_resources"]
)
### auto-generated section `build intellij.ae.database.counters.community` end
@@ -1,20 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
<sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="kotlin-stdlib" level="project" />
<orderEntry type="module" module-name="intellij.platform.ide.impl" />
<orderEntry type="module" module-name="intellij.platform.debugger" />
<orderEntry type="module" module-name="intellij.platform.lang.impl" />
<orderEntry type="module" module-name="intellij.platform.vcs" />
<orderEntry type="module" module-name="intellij.ae.database.core" />
<orderEntry type="module" module-name="intellij.platform.sqlite" />
<orderEntry type="library" scope="TEST" name="assertJ" level="project" />
</component>
</module>
@@ -1,39 +0,0 @@
<idea-plugin package="com.intellij.ae.database.counters.community">
<extensions defaultExtensionNs="com.intellij.ae.database">
<fusEventCatcher implementation="com.intellij.ae.database.counters.community.AltEnterActionInvocation$Factory"/>
<fusEventCatcher implementation="com.intellij.ae.database.counters.community.PluginInstalledFusListener"/>
<fusEventCatcher implementation="com.intellij.ae.database.counters.community.PluginUninstalledFusListener"/>
</extensions>
<extensions defaultExtensionNs="com.intellij">
<vfs.asyncListener implementation="com.intellij.ae.database.counters.community.TestFileCreationListener"/>
<editorFactoryListener implementation="com.intellij.ae.database.counters.community.CodingTimeUserActivityEditorFactoryListener"/>
<postStartupActivity implementation="com.intellij.ae.database.counters.community.BuildListenerProjectActivity"/>
<checkinHandlerFactory implementation="com.intellij.ae.database.counters.community.VcsListenerFactory"/>
<postStartupActivity implementation="com.intellij.ae.database.counters.community.ProjectOpenedTimeSpanUserActivityProjectOpenListener"/>
</extensions>
<applicationListeners>
<listener class="com.intellij.ae.database.counters.community.IdeStartedUserActivityListener"
topic="com.intellij.ide.AppLifecycleListener"/>
<listener class="com.intellij.ae.database.counters.community.CompletionCharactersSparedListener"
topic="com.intellij.featureStatistics.FeatureStatisticsUpdateListener"/>
<listener class="com.intellij.ae.database.counters.community.RunConfigurationListener"
topic="com.intellij.execution.ExecutionListener"/>
<listener class="com.intellij.ae.database.counters.community.ProjectOpenedTimeSpanUserActivityProjectCloseListener"
topic="com.intellij.openapi.project.ProjectCloseListener"/>
<listener class="com.intellij.ae.database.counters.community.CodingTimeUserActivityCommandListener"
topic="com.intellij.openapi.command.CommandListener"/>
<listener class="com.intellij.ae.database.counters.community.MyDynamicPluginListener"
topic="com.intellij.ide.plugins.DynamicPluginListener"/>
</applicationListeners>
<incompatible-with>com.intellij.cwm.guest</incompatible-with>
<incompatible-with>com.intellij.jetbrains.client</incompatible-with>
<incompatible-with>com.intellij.gateway</incompatible-with>
<dependencies>
<module name="intellij.ae.database.core"/>
</dependencies>
</idea-plugin>
@@ -1,21 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.baseEvents.FusBasedCounterUserActivity
import com.intellij.ae.database.core.baseEvents.fus.FusEventCatcher
import com.intellij.ae.database.core.baseEvents.fus.FusEventDefinitionBuilder
// todo record not popup opening, but intention call
object AltEnterActionInvocation : FusBasedCounterUserActivity() {
internal class Factory : FusEventCatcher.Factory {
override fun getInstance(): FusEventCatcher = catcher
}
override fun define(): FusEventDefinitionBuilder {
return definition("alt.enter.invocation") {
event("actions", "action.finished") {
field("action_id", "ShowIntentionActions")
}
}
}
}
@@ -1,175 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.createMap
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.lang.Language
import com.intellij.openapi.application.readAction
import com.intellij.openapi.command.CommandEvent
import com.intellij.openapi.command.CommandListener
import com.intellij.openapi.components.Service
import com.intellij.openapi.editor.Document
import com.intellij.openapi.editor.event.DocumentEvent
import com.intellij.openapi.editor.event.DocumentListener
import com.intellij.openapi.editor.event.EditorFactoryEvent
import com.intellij.openapi.editor.event.EditorFactoryListener
import com.intellij.openapi.editor.ex.EditorEx
import com.intellij.openapi.editor.ex.util.EditorUtil
import com.intellij.openapi.fileEditor.FileDocumentManager
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.Disposer
import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.validOrNull
import com.intellij.psi.impl.PsiManagerEx
import com.intellij.psi.util.validOrNull
import com.intellij.util.io.DigestUtil
import org.jetbrains.sqlite.ObjectBinderFactory
import java.math.BigInteger
import java.time.Instant
import java.util.*
/**
* Coding activity. Stores language of the file and file hash
*/
object CodingTimeUserActivity : WritableDatabaseBackedTimeSpanUserActivity() {
override val canBeStale: Boolean
get() = true
override val id: String
get() = "editor.changing"
suspend fun write(editorId: String, language: Language, file: VirtualFile) {
val extra = mapOf("lang" to language.id, "fileHash" to md5(file.presentableUrl))
submitPeriodic(editorId, extra)
}
/**
* How many files were edited in the period of [from]..[until]
*/
suspend fun getFilesEdited(from: Instant, until: Instant): Int? {
return getDatabase().execute { database ->
val filesEditedStatement = database
.prepareStatement("SELECT COUNT(DISTINCT json_extract(extra, '\$.fileHash')) FROM timespanUserActivity\n" +
"WHERE activity_id = '$id'\n" +
"AND datetime(started_at) >= datetime(?)\n" +
"AND datetime(ended_at) <= datetime(?)", ObjectBinderFactory.create2<String, String>())
filesEditedStatement.binder.bind(InstantUtils.formatForDatabase(from), InstantUtils.formatForDatabase(until))
filesEditedStatement.selectInt() ?: 0
}
}
/**
* How much files per language were edited / map of (language name -> length in seconds)
*/
suspend fun getByLanguageStat(from: Instant, until: Instant): Map<LanguageWrapper, Int>? {
return getDatabase().execute { database ->
val byLanguageStatStatement = database
.prepareStatement("SELECT json_extract(extra, '\$.lang') as lang, SUM(strftime('%s', ended_at) - strftime('%s', started_at)) FROM timespanUserActivity\n" +
"WHERE activity_id = '$id'\n" +
"AND datetime(started_at) >= datetime(?)\n" +
"AND datetime(ended_at) <= datetime(?)\n" +
"GROUP BY lang", ObjectBinderFactory.create2<String, String>())
byLanguageStatStatement.binder.bind(InstantUtils.formatForDatabase(from), InstantUtils.formatForDatabase(until))
val res = byLanguageStatStatement.executeQuery()
createMap(
{ it.getString(0) }, { it.getInt(1) },
{ k0, v0 -> k0 != null && v0 != 0 },
{ LanguageWrapper(Language.findLanguageByID(it!!), it) }, { it },
{ _, _ -> true },
res
)
}
}
private fun md5(buffer: String): String {
val md5 = DigestUtil.md5()
md5.update(buffer.toByteArray(Charsets.UTF_8))
return BigInteger(md5.digest()).abs().toString(16)
}
}
/**
* Keeps language ID in case Language is not found
*/
data class LanguageWrapper(
val language: Language?,
val id: String
)
@Service(Service.Level.PROJECT)
private class DocumentHolder {
companion object {
fun getInstance(project: Project) = project.getService(DocumentHolder::class.java)
}
private val urls = mutableSetOf<String>()
fun submit(vf: VirtualFile) {
urls.add(vf.url)
}
fun remove(vf: VirtualFile) {
urls.remove(vf.url)
}
fun isAllowed(vf: VirtualFile): Boolean {
return urls.contains(vf.url)
}
}
internal class CodingTimeUserActivityEditorFactoryListener : EditorFactoryListener {
override fun editorCreated(event: EditorFactoryEvent) {
val editor = event.editor as EditorEx
val project = editor.project ?: return
val editorId = UUID.randomUUID().toString()
val disposable = Disposer.newDisposable()
EditorUtil.disposeWithEditor(editor, disposable)
editor.document.addDocumentListener(object : DocumentListener {
override fun documentChanged(event: DocumentEvent) {
val vf = editor.virtualFile?.validOrNull() ?: return
if (DocumentHolder.getInstance(project).isAllowed(vf)) {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(CodingTimeUserActivity) {
val psiFile = readAction {
if (vf.isValid) {
PsiManagerEx.getInstance(project).findFile(vf)?.validOrNull()
}
else {
null
}
} ?: return@runUpdateEvent
it.write(editorId, psiFile.language, vf)
}
}
}
}, disposable)
}
}
internal class CodingTimeUserActivityCommandListener : CommandListener {
override fun commandStarted(event: CommandEvent) {
val doc = event.commandGroupId as? Document
if (doc != null) {
val vf = FileDocumentManager.getInstance().getFile(doc)?.validOrNull()
val project = event.project
if (vf != null && project != null) {
DocumentHolder.getInstance(project).submit(vf)
}
}
}
override fun commandFinished(event: CommandEvent) {
val doc = event.commandGroupId as? Document
if (doc != null) {
val vf = FileDocumentManager.getInstance().getFile(doc)?.validOrNull()
val project = event.project
if (vf != null && project != null) {
DocumentHolder.getInstance(project).remove(vf)
}
}
}
}
@@ -1,30 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.ReadableUserActivity
import com.intellij.ae.database.core.activities.WritableDatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.featureStatistics.FeatureStatisticsUpdateListener
/**
* Stat for 'Code completion has saved you from typing at least N characters'
*/
object CompletionCharactersSpared : ReadableUserActivity<Int>, WritableDatabaseBackedCounterUserActivity() {
override val id = "completion.spared"
override suspend fun getActivityValue(): Int {
return getDatabase().getActivitySum(this, null, null)
}
internal suspend fun write(newSpared: Int) {
submit(newSpared)
}
}
internal class CompletionCharactersSparedListener : FeatureStatisticsUpdateListener {
override fun completionStatUpdated(spared: Int) {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(CompletionCharactersSpared) {
it.write(spared)
}
}
}
@@ -1,25 +0,0 @@
package com.intellij.ae.database.counters.community
import com.intellij.openapi.Disposable
import com.intellij.openapi.components.Service
import com.intellij.openapi.components.service
import com.intellij.openapi.project.Project
import kotlinx.coroutines.CoroutineScope
@Service
class FeatureUsageDatabaseCountersScopeProvider(private val cs: CoroutineScope) : Disposable {
companion object {
fun getDisposable(): Disposable = service<FeatureUsageDatabaseCountersScopeProvider>()
fun getScope() = service<FeatureUsageDatabaseCountersScopeProvider>().cs
fun getDisposable(project: Project): Disposable = project.service<FeatureUsageDatabaseCountersProjectScopeProvider>()
fun getScope(project: Project) = project.service<FeatureUsageDatabaseCountersProjectScopeProvider>().cs
}
override fun dispose() {}
}
@Service(Service.Level.PROJECT)
private class FeatureUsageDatabaseCountersProjectScopeProvider(val cs: CoroutineScope) : Disposable {
override fun dispose() {}
}
@@ -1,35 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.ide.AppLifecycleListener
import kotlinx.coroutines.delay
object IdeRunningUserActivity : WritableDatabaseBackedTimeSpanUserActivity() {
override val canBeStale = true
override val id = "ide.running"
private const val eventId = "ideRun"
internal suspend fun writeStart() {
val moment = InstantUtils.Now
// submit event later, because submitting it now would init all the db stuff
delay(5000)
submitManual(eventId, TimeSpanUserActivityDatabaseManualKind.Start, null, moment)
}
}
internal class IdeStartedUserActivityListener : AppLifecycleListener {
/**
* Enough to write down when app was started, because event marked as canBeStale
*/
override fun appStarted() {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(IdeRunningUserActivity) {
it.writeStart()
}
}
}
@@ -1,141 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.baseEvents.fus.FusEventCatcher
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.ide.plugins.DynamicPluginListener
import com.intellij.ide.plugins.IdeaPluginDescriptor
import com.intellij.ide.plugins.InstalledPluginsState
import com.intellij.openapi.components.*
import com.intellij.openapi.diagnostic.logger
import com.intellij.openapi.extensions.PluginId
import java.time.Instant
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicLong
/**
* Plugin is considered to be installed if it wasn't uninstalled in 10 minutes after installation.
* Plugin updates are ignored
*/
object PluginsInstalledCounterUserActivity : WritableDatabaseBackedCounterUserActivity() {
override val id: String
get() = "plugin.installed"
internal suspend fun writeInstallation(pluginId: String, time: Instant) {
val installedPlugins = InstalledPluginsPersistentState.getInstanceAsync()
if (!InstalledPluginsState.getInstance().wasUpdated(PluginId.getId(pluginId)) && installedPlugins.add(pluginId)) {
submit(1, time)
}
else {
installedPlugins.removeUpdated(pluginId)
}
}
internal suspend fun writeUninstallation(pluginId: String) {
val installedPlugins = InstalledPluginsPersistentState.getInstanceAsync()
if (installedPlugins.remove(pluginId)) {
submit(-1)
}
}
}
@Service
@State(name = "aeInstalledPlugins", storages = [(Storage("aeInstalledPlugins.xml", roamingType = RoamingType.DISABLED))])
class InstalledPluginsPersistentState : PersistentStateComponentWithModificationTracker<InstalledPluginsPersistentState.State> {
companion object {
fun getInstance() = service<InstalledPluginsPersistentState>()
suspend fun getInstanceAsync() = serviceAsync<InstalledPluginsPersistentState>()
}
data class State(var myPlugins: Map<String, Long> = HashMap())
private val PLUGIN_INSTALL_TTL_MS = 10 * 60 * 1000
private val installedPlugins = ConcurrentHashMap<String, Long>()
private val updatedPlugins = ConcurrentHashMap.newKeySet<String>()
private val modTracker = AtomicLong(0)
override fun getState() = State(installedPlugins.filterValues { System.currentTimeMillis() - it < PLUGIN_INSTALL_TTL_MS })
override fun getStateModificationCount() = modTracker.get()
override fun loadState(state: State) {
for ((k, v) in state.myPlugins) {
installedPlugins[k] = v
}
}
fun add(s: String): Boolean {
val installedPluginContains = installedPlugins[s]?.let { System.currentTimeMillis() - it < PLUGIN_INSTALL_TTL_MS } == true
if (installedPluginContains || updatedPlugins.contains(s)) {
return false
}
modTracker.incrementAndGet()
installedPlugins[s] = System.currentTimeMillis()
return true
}
fun addUpdated(s: String) {
updatedPlugins.add(s)
}
fun removeUpdated(s: String) {
updatedPlugins.remove(s)
}
// plugin will be recorded as removed if it was removed within [HALF_HOUR_MS] milliseconds
fun remove(s: String): Boolean {
modTracker.incrementAndGet()
val currTime = System.currentTimeMillis()
updatedPlugins.remove(s)
return installedPlugins.remove(s)?.let { currTime - it < PLUGIN_INSTALL_TTL_MS } == true
}
}
internal class MyDynamicPluginListener : DynamicPluginListener {
// we don't want to treat theme updates as new installations, workaround for IDEA-342821
override fun beforePluginUnload(pluginDescriptor: IdeaPluginDescriptor, isUpdate: Boolean) {
if (isUpdate) {
InstalledPluginsPersistentState.getInstance().addUpdated(pluginDescriptor.pluginId.idString)
}
}
}
internal class PluginInstalledFusListener : FusEventCatcher(), FusEventCatcher.Factory {
override fun getInstance() = this
override fun define() = definition("plugin.installed") {
event("plugin.manager", "plugin.installation.finished")
}
override suspend fun onEvent(fields: Map<String, Any>, eventTime: Instant) {
val pluginId = fields["plugin"] as? String ?: return
logger.info("Plugin installed: ${pluginId}.")
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(PluginsInstalledCounterUserActivity) {
it.writeInstallation(pluginId, eventTime)
}
}
}
internal class PluginUninstalledFusListener : FusEventCatcher(), FusEventCatcher.Factory {
override fun getInstance() = this
override fun define() = definition("plugin.uninstalled") {
event("plugin.manager", "plugin.was.removed")
}
override suspend fun onEvent(fields: Map<String, Any>, eventTime: Instant) {
val pluginId = fields["plugin"] as? String ?: return
logger.info("Plugin uninstalled: ${pluginId}.")
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(PluginsInstalledCounterUserActivity) {
it.writeUninstallation(pluginId)
}
}
}
private val logger = logger<PluginsInstalledCounterUserActivity>()
@@ -1,39 +0,0 @@
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.openapi.project.Project
import com.intellij.openapi.project.ProjectCloseListener
import com.intellij.openapi.startup.ProjectActivity
object ProjectOpenedTimeSpanUserActivity : WritableDatabaseBackedTimeSpanUserActivity() {
override val canBeStale: Boolean
get() = true
override val id: String
get() = "project.opened"
suspend fun projectOpened(id: String, name: String) {
submitManual(id, TimeSpanUserActivityDatabaseManualKind.Start, mapOf("name" to name))
}
suspend fun projectClosed(id: String, name: String) {
submitManual(id, TimeSpanUserActivityDatabaseManualKind.End, mapOf("name" to name))
}
}
internal class ProjectOpenedTimeSpanUserActivityProjectOpenListener : ProjectActivity {
override suspend fun execute(project: Project) {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(ProjectOpenedTimeSpanUserActivity) {
it.projectOpened(project.locationHash, project.name)
}
}
}
internal class ProjectOpenedTimeSpanUserActivityProjectCloseListener : ProjectCloseListener {
override fun projectClosing(project: Project) {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(ProjectOpenedTimeSpanUserActivity) {
it.projectClosed(project.locationHash, project.name)
}
}
}
@@ -1,196 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedTimeSpanUserActivity
import com.intellij.ae.database.core.dbs.timespan.TimeSpanUserActivityDatabaseManualKind
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.ae.database.core.utils.InstantUtils
import com.intellij.build.BuildProgressListener
import com.intellij.build.BuildViewManager
import com.intellij.build.events.BuildEvent
import com.intellij.build.events.FinishBuildEvent
import com.intellij.build.events.StartBuildEvent
import com.intellij.execution.ExecutionListener
import com.intellij.execution.executors.DefaultDebugExecutor
import com.intellij.execution.process.ProcessHandler
import com.intellij.execution.runners.ExecutionEnvironment
import com.intellij.ide.plugins.cl.PluginAwareClassLoader
import com.intellij.openapi.components.serviceAsync
import com.intellij.openapi.project.Project
import com.intellij.openapi.startup.ProjectActivity
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.jetbrains.sqlite.ObjectBinderFactory
import java.time.Instant
// todo check all run configurations (run with profile, run coverage)
/**
* Writes events related to run configurations.
*/
object RunConfigurationTimeSpanUserActivity : WritableDatabaseBackedTimeSpanUserActivity() {
override val canBeStale = true
// TODO add flag 'errorProne' that fill turn off logs about bad end?
override val id = "runconfig.running"
private val lastBuildLock = Mutex()
private var lastBuildAt = 0L
private var lastAttemptFromBuildListener: Boolean? = null
suspend fun writeBuildStart(id: Int, timeStart: Long, attemptFromBuildListener: Boolean) {
/**
* Sometimes [BuildProgressListener] reports events, sometimes [ExecutionListener].
* Sometimes they both report build events. So if a build event happened within 300ms and other event is from different place, we will skip it
*/
lastBuildLock.withLock {
val diff = timeStart - lastBuildAt
if (diff <= 300 && lastAttemptFromBuildListener != attemptFromBuildListener) {
return
}
else {
lastBuildAt = timeStart
lastAttemptFromBuildListener = attemptFromBuildListener
}
}
val data = mapOf("act" to RunConfigurationEventKind.Build.eventName)
submitManual(id.toString(), TimeSpanUserActivityDatabaseManualKind.Start, data)
}
suspend fun writeRunConfigurationStart(kind: RunConfigurationEventKind, id: Int, timeStart: Long) {
val data = mapOf("act" to kind.eventName)
submitManual(id.toString(), TimeSpanUserActivityDatabaseManualKind.Start, data)
}
suspend fun writeEnd(id: Int) {
submitManual(id.toString(), TimeSpanUserActivityDatabaseManualKind.End, null)
}
/**
* @return sum of sessions lengths of specific [kind] in the period [from]..[until] in seconds
*/
suspend fun getSessionsLength(kind: RunConfigurationEventKind, from: Instant, until: Instant): Int? {
return getDatabase().execute { database ->
val sessionsLengthStatement = database
.prepareStatement("SELECT SUM(strftime('%s', ended_at) - strftime('%s', started_at)) FROM timespanUserActivity\n" +
"WHERE activity_id = '$id' " +
"AND json_extract(extra, '\$.act') = ? " +
"AND datetime(started_at) >= datetime(?) " +
"AND datetime(ended_at) <= datetime(?)", ObjectBinderFactory.create3<String, String, String>())
sessionsLengthStatement.binder.bind(kind.eventName, InstantUtils.formatForDatabase(from), InstantUtils.formatForDatabase(until))
sessionsLengthStatement.selectInt() ?: 0
}
}
/**
* @return sum of sessions lengths grouped by kind in the period [from]..[until] in seconds
*/
suspend fun getAllSessionsLength(from: Instant, until: Instant): Map<RunConfigurationEventKind, Int>? {
return getDatabase().execute { database ->
val allSessionsLengthStatement = database
.prepareStatement("SELECT json_extract(extra, '\$.act') as field, SUM(strftime('%s', ended_at) - strftime('%s', started_at)) FROM timespanUserActivity\n" +
"WHERE activity_id = '$id' " +
"AND datetime(started_at) >= datetime(?) " +
"AND datetime(ended_at) <= datetime(?) " +
"GROUP BY field", ObjectBinderFactory.create2<String, String>())
allSessionsLengthStatement.binder.bind(InstantUtils.formatForDatabase(from), InstantUtils.formatForDatabase(until))
val res = allSessionsLengthStatement.executeQuery()
val map = mutableMapOf<RunConfigurationEventKind, Int>()
while (res.next()) {
val keyString = res.getString(0)
val valueInt = res.getInt(1)
if (keyString == null || valueInt == 0) continue
val k = RunConfigurationEventKind.fromString(keyString) ?: continue
map[k] = valueInt
}
map
}
}
}
enum class RunConfigurationEventKind(val eventName: String) {
Run("run"), Debug("debug"), Build("build");
companion object {
fun fromString(eventName: String) = entries.find { it.eventName == eventName }
}
}
internal class RunConfigurationListener : ExecutionListener {
override fun processStarted(executorId: String, env: ExecutionEnvironment, handler: ProcessHandler) {
if (!isAllowed(env)) return
val id = System.identityHashCode(handler) // not the best ID out there, but it works
val timeStart = System.currentTimeMillis()
when { // should be similar to [isAllowed]
env.runProfile.name.let { it.contains("[build", true) || it.startsWith("build", true) } -> {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeBuildStart(id, timeStart, false)
}
}
env.executor is DefaultDebugExecutor || env.executor.id.contains("debug", true) -> {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeRunConfigurationStart(RunConfigurationEventKind.Debug, id, timeStart)
}
}
else -> {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeRunConfigurationStart(RunConfigurationEventKind.Run, id, timeStart)
}
}
}
}
override fun processTerminated(executorId: String, env: ExecutionEnvironment, handler: ProcessHandler, exitCode: Int) {
if (!isAllowed(env)) return
val id = System.identityHashCode(handler)
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeEnd(id)
}
}
private fun isAllowed(env: ExecutionEnvironment): Boolean {
// todo: extension point
if ((env.runProfile.javaClass.classLoader as? PluginAwareClassLoader)?.pluginId?.idString == "com.intellij.database") {
return false
}
return true
}
}
internal class BuildListenerProjectActivity : ProjectActivity {
override suspend fun execute(project: Project) {
project.serviceAsync<BuildViewManager>().addListener(
BuildListener(),
FeatureUsageDatabaseCountersScopeProvider.getDisposable(project)
)
}
}
internal class BuildListener : BuildProgressListener {
override fun onEvent(buildId: Any, event: BuildEvent) {
val id = System.identityHashCode(buildId)
val timeStart = System.currentTimeMillis()
when (event) {
is StartBuildEvent -> {
// Skip Gradle initial loading
if (event.buildDescriptor.title == "Classes up-to-date check") return
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeBuildStart(id, timeStart, true)
}
}
is FinishBuildEvent -> {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(RunConfigurationTimeSpanUserActivity) {
it.writeEnd(id)
}
}
}
}
}
@@ -1,56 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.openapi.application.readAction
import com.intellij.openapi.fileTypes.PlainTextLanguage
import com.intellij.openapi.project.ProjectLocator
import com.intellij.openapi.roots.TestSourcesFilter
import com.intellij.openapi.vfs.AsyncFileListener
import com.intellij.openapi.vfs.AsyncFileListener.ChangeApplier
import com.intellij.openapi.vfs.newvfs.events.VFileCopyEvent
import com.intellij.openapi.vfs.newvfs.events.VFileCreateEvent
import com.intellij.openapi.vfs.newvfs.events.VFileEvent
import com.intellij.psi.impl.PsiManagerEx
import com.intellij.psi.util.validOrNull
object TestFileCreatedUserActivity : WritableDatabaseBackedCounterUserActivity() {
override val id = "test.file.created"
suspend fun write() {
submit(1)
}
}
internal class TestFileCreationListener : AsyncFileListener {
override fun prepareChange(events: MutableList<out VFileEvent>): ChangeApplier {
val filteredEvents = events.filter { it is VFileCreateEvent || it is VFileCopyEvent }
return object : ChangeApplier {
override fun afterVfsChange() {
for (event in filteredEvents) {
val file = event.file ?: return
val project = ProjectLocator.getInstance().guessProjectForFile(file) ?: return
val isTest = TestSourcesFilter.isTestSources(file, project)
if (isTest) {
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(TestFileCreatedUserActivity) {
val psiFile = readAction {
PsiManagerEx.getInstance(project).findFile(file)?.validOrNull()
}
if (psiFile == null) {
return@runUpdateEvent
}
// try to write only actual tests, not test data and etc
if (psiFile.language != PlainTextLanguage.INSTANCE && !file.fileType.isBinary) {
it.write()
}
}
}
}
}
}
}
}
@@ -1,35 +0,0 @@
// Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.ae.database.counters.community
import com.intellij.ae.database.core.activities.WritableDatabaseBackedCounterUserActivity
import com.intellij.ae.database.core.runUpdateEvent
import com.intellij.openapi.vcs.CheckinProjectPanel
import com.intellij.openapi.vcs.changes.CommitContext
import com.intellij.openapi.vcs.checkin.CheckinHandler
import com.intellij.openapi.vcs.checkin.CheckinHandlerFactory
import java.time.Instant
object VcsFileCommittedUserActivity : WritableDatabaseBackedCounterUserActivity() {
override val id = "vcs.changes"
suspend fun getFilesCommitted(from: Instant, until: Instant): Int {
return getDatabase().getActivitySum(this, from, until)
}
suspend fun write(filesChanged: Int) {
submit(filesChanged)
}
}
internal class VcsListenerFactory : CheckinHandlerFactory() {
override fun createHandler(panel: CheckinProjectPanel, commitContext: CommitContext): CheckinHandler {
return object : CheckinHandler() {
override fun checkinSuccessful() {
val count = panel.files.size
FeatureUsageDatabaseCountersScopeProvider.getScope().runUpdateEvent(VcsFileCommittedUserActivity) {
it.write(count)
}
}
}
}
}
@@ -1,20 +0,0 @@
### auto-generated section `build intellij.ae.database.community` start
load("@rules_java//java:defs.bzl", "java_library")
load("@rules_jvm//:jvm.bzl", "jvm_resources")
jvm_resources(
name = "ae-database-community_resources",
files = glob(["resources/**/*"]),
strip_prefix = "resources"
)
java_library(
name = "ae-database-community",
visibility = ["//visibility:public"],
runtime_deps = [
":ae-database-community_resources",
"//plugins/feature-usage-database/core:ae-database-core",
"//plugins/feature-usage-database/counters:ae-database-counters-community",
]
)
### auto-generated section `build intellij.ae.database.community` end
@@ -1,13 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/resources" type="java-resource" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module" module-name="intellij.ae.database.core" scope="RUNTIME" />
<orderEntry type="module" module-name="intellij.ae.database.counters.community" scope="RUNTIME" />
</component>
</module>
@@ -1,26 +0,0 @@
<idea-plugin require-restart="true" allow-bundled-update="true">
<id>com.jetbrains.ae.database</id>
<name>Local Productivity Statistics</name>
<description><![CDATA[
<p>The <i>Local Productivity Statistics</i> plugin gathers local statistics on your machine about how the IDE is used.
You will be provided with an overview of your usage at the end of the year.</p>
<p>For the purpose of Local Productivity Statistics plugin, no data are transferred from your computer,
and the database is not shared or synchronized with any remote storage or third-party service.
JetBrains does not have access to your personal data created in the IDEs.
For more information on data protection, please click here:
<a href="https://www.jetbrains.com/legal/docs/privacy/privacy/">https://www.jetbrains.com/legal/docs/privacy/privacy/</a></p>
]]></description>
<vendor>JetBrains</vendor>
<incompatible-with>com.intellij.cwm.guest</incompatible-with>
<incompatible-with>com.intellij.jetbrains.client</incompatible-with>
<incompatible-with>com.intellij.gateway</incompatible-with>
<incompatible-with>com.intellij.modules.ultimate</incompatible-with>
<content>
<module name="intellij.ae.database.core"/>
<module name="intellij.ae.database.counters.community"/>
</content>
<depends>com.intellij.modules.platform</depends>
</idea-plugin>