IDEA-228935 Submit FUS logs on IDE exit

GitOrigin-RevId: d797219d592987e0c28e65dae3723accab75d911

platform/platform-impl/src/com/intellij/internal/statistic/EventLogApplicationLifecycleListener.java

@@ -0,0 +1,29 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic;
+
+import com.intellij.ide.AppLifecycleListener;
+import com.intellij.ide.plugins.PluginManagerCore;
+import com.intellij.ide.util.PropertiesComponent;
+import com.intellij.internal.statistic.eventLog.StatisticsEventLoggerProvider;
+import com.intellij.internal.statistic.eventLog.fus.FeatureUsageLogger;
+import com.intellij.internal.statistic.eventLog.uploader.EventLogExternalUploader;
+import com.intellij.openapi.application.ApplicationInfo;
+
+public class EventLogApplicationLifecycleListener implements AppLifecycleListener {
+
+  @Override
+  public void appWillBeClosed(boolean isRestart) {
+    if (!isRestart && !PluginManagerCore.isRunningFromSources()) {
+      StatisticsEventLoggerProvider config = FeatureUsageLogger.INSTANCE.getConfig();
+      if (config.isSendEnabled()) {
+        boolean isUpdateInProgress = isUpdateInProgress();
+        EventLogExternalUploader.INSTANCE.startExternalUpload(config.getRecorderId(), false, isUpdateInProgress);
+      }
+    }
+  }
+
+  private static boolean isUpdateInProgress() {
+    return ApplicationInfo.getInstance().getBuild().asString().
+      equals(PropertiesComponent.getInstance().getValue("ide.self.update.started.for.build"));
+  }
+}

platform/platform-resources/src/componentSets/PlatformLangComponents.xml

@@ -21,7 +21,7 @@
     <listener class="com.intellij.notification.impl.NotificationsConfigurationImpl$MyNotificationListener" topic="com.intellij.notification.Notifications"/>
 
     <listener class="com.intellij.internal.statistic.collectors.fus.fileTypes.FileTypeUsageCounterCollector$MyAnActionListener" topic="com.intellij.openapi.actionSystem.ex.AnActionListener"/>
-
+    <listener class="com.intellij.internal.statistic.EventLogApplicationLifecycleListener" topic="com.intellij.ide.AppLifecycleListener" activeInHeadlessMode="false"/>
     <listener class="com.intellij.internal.statistic.local.ActionsLocalSummaryListener" topic="com.intellij.openapi.actionSystem.ex.AnActionListener"/>
 
     <listener class="com.intellij.ide.GeneratedSourceFileChangeTrackerImpl$MyProjectManagerListener" topic="com.intellij.openapi.project.ProjectManagerListener"/>

platform/platform-tests/testSrc/com/intellij/internal/statistics/uploader/EventLogUploaderArgumentParserTest.kt

@@ -0,0 +1,69 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistics.uploader
+
+import com.intellij.internal.statistic.uploader.EventLogUploaderCliParser
+import com.intellij.testFramework.UsefulTestCase
+import junit.framework.TestCase
+
+class EventLogUploaderArgumentParserTest : UsefulTestCase() {
+
+  private fun doTest(arguments: Array<String>, vararg expected: Pair<String, String?>) {
+    val actual = EventLogUploaderCliParser.parseOptions(arguments)
+    TestCase.assertEquals(hashMapOf(*expected), actual)
+  }
+
+  fun test_parsing_all_arguments() {
+    doTest(
+      arrayOf("--bucket", "123", "--recorder", "FUS", "--test", "--device", "12345", "--url", "http://some.url", "--internal"),
+      "--bucket" to "123",
+      "--recorder" to "FUS",
+      "--test" to null,
+      "--device" to "12345",
+      "--url" to "http://some.url",
+      "--internal" to null
+    )
+  }
+
+  fun test_next_option_instead_of_value() {
+    doTest(
+      arrayOf("--bucket", "123", "--recorder", "FUS", "--test", "--device", "12345", "--url", "--internal"),
+      "--bucket" to "123",
+      "--recorder" to "FUS",
+      "--test" to null,
+      "--device" to "12345",
+      "--url" to null,
+      "--internal" to null
+    )
+  }
+
+  fun test_end_instead_of_value() {
+    doTest(
+      arrayOf("--bucket", "123", "--recorder", "FUS", "--test", "--device", "12345", "--url"),
+      "--bucket" to "123",
+      "--recorder" to "FUS",
+      "--test" to null,
+      "--device" to "12345",
+      "--url" to null
+    )
+  }
+
+  fun test_two_values() {
+    doTest(
+      arrayOf("--bucket", "123", "--recorder", "FUS", "--test", "--device", "12345", "6789"),
+      "--bucket" to "123",
+      "--recorder" to "FUS",
+      "--test" to null,
+      "--device" to "12345"
+    )
+  }
+
+  fun test_first_value() {
+    doTest(
+      arrayOf("12345", "--bucket", "123", "--recorder", "FUS", "--test", "--device", "12345"),
+      "--bucket" to "123",
+      "--recorder" to "FUS",
+      "--test" to null,
+      "--device" to "12345"
+    )
+  }
+}

platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogExternalUploader.kt

@@ -0,0 +1,150 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog.uploader
+
+import com.intellij.internal.statistic.eventLog.*
+import com.intellij.internal.statistic.uploader.EventLogUploaderOptions.*
+import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.diagnostic.Logger
+import com.intellij.openapi.util.SystemInfo
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.util.ArrayUtil
+import com.intellij.util.io.exists
+import java.io.File
+import java.nio.file.Paths
+
+object EventLogExternalUploader {
+  private val LOG = Logger.getInstance(EventLogExternalUploader.javaClass)
+  private const val UPLOADER_MAIN_CLASS = "com.intellij.internal.statistic.uploader.EventLogUploader"
+
+  fun startExternalUpload(recorderId: String, isTest: Boolean, shouldCopy: Boolean) {
+    val recorder = EventLogInternalRecorderConfig(recorderId)
+    if (!recorder.isSendEnabled()) {
+      LOG.info("Don't start external process because sending logs is disabled")
+      return
+    }
+
+    val device = DeviceConfiguration(EventLogConfiguration.deviceId, EventLogConfiguration.bucket)
+    val application = EventLogInternalApplicationInfo(isTest)
+    try {
+      val command = prepareUploadCommand(device, recorder, application, shouldCopy)
+      Runtime.getRuntime().exec(command)
+      LOG.info("Started external process for uploading event log")
+    }
+    catch (e: EventLogUploadException) {
+      LOG.info(e)
+    }
+  }
+
+  private fun prepareUploadCommand(device: DeviceConfiguration,
+                                   recorder: EventLogRecorderConfig,
+                                   applicationInfo: EventLogApplicationInfo,
+                                   shouldCopy: Boolean): Array<out String> {
+    val root = logsRoot(recorder)
+    if (root == null || !root.exists()) {
+      throw EventLogUploadException("Event logs root directory is not specified")
+    }
+
+    val tempDir = getTempDir()
+    if (shouldCopy && FileUtil.isAncestor(PathManager.getHomePath(), tempDir.path, true)) {
+      throw EventLogUploadException("Temp directory inside installation: $tempDir")
+    }
+    val uploader = findUploader()
+    val libs = findLibsByPrefixes(
+      "kotlin-stdlib", "gson", "commons-logging", "log4j.jar", "httpclient", "httpcore", "httpmime", "jdom.jar", "annotations.jar"
+    )
+
+    val uploaderCopy = if (shouldCopy) uploader.copyTo(File(tempDir, uploader.name), true) else uploader
+    val libCopies = if (shouldCopy) libs.map {it.copyTo(File(tempDir, it.name), true)}.map { it.path } else libs.map { it.path }
+    val classpath = joinAsClasspath(libCopies, uploaderCopy)
+
+    val args = arrayListOf<String>()
+    val java = findOrCopyJava(tempDir, shouldCopy)
+    args += File(java, if (SystemInfo.isWindows) "bin\\java.exe" else "bin/java").path
+    addArgument(args, "-cp", classpath)
+
+    args += "-Djava.io.tmpdir=${tempDir.path}"
+    args += UPLOADER_MAIN_CLASS
+
+    addArgument(args, RECORDER_OPTION, recorder.getRecorderId())
+    addArgument(args, DIRECTORY_OPTION, root.toString())
+    addArgument(args, DEVICE_OPTION, device.deviceId)
+    addArgument(args, BUCKET_OPTION, device.bucket.toString())
+    addArgument(args, URL_OPTION, applicationInfo.templateUrl)
+    addArgument(args, PRODUCT_OPTION, applicationInfo.productCode)
+
+    if (applicationInfo.isInternal) {
+      args += INTERNAL_OPTION
+    }
+
+    if (applicationInfo.isTest) {
+      args += TEST_OPTION
+    }
+    return ArrayUtil.toStringArray(args)
+  }
+
+  private fun addArgument(args: ArrayList<String>, name: String, value: String) {
+    args += name
+    args += value
+  }
+
+  private fun logsRoot(recorder: EventLogRecorderConfig) = recorder.getLogFilesProvider().getLogFilesDir()?.toAbsolutePath()
+
+  private fun joinAsClasspath(libCopies: List<String>, uploaderCopy: File): String {
+    if (libCopies.isEmpty()) {
+      return uploaderCopy.path
+    }
+    val libClassPath = libCopies.joinToString(separator = File.pathSeparator)
+    return "$libClassPath${File.pathSeparator}${uploaderCopy.path}"
+  }
+
+  private fun findUploader(): File {
+    val uploader = File(PathManager.getLibPath(), "platform-statistics-uploader.jar")
+    if (uploader.exists() && !uploader.isDirectory) {
+      return uploader
+    }
+
+    //consider local debug IDE case
+    val localBuild = File(PathManager.getHomePath(), "out/artifacts/statistics-uploader.jar")
+    if (localBuild.exists() && !localBuild.isDirectory) {
+      return localBuild
+    }
+    throw EventLogUploadException("Cannot find uploader jar")
+  }
+
+  private fun findOrCopyJava(tempDir: File, shouldCopy: Boolean): String {
+    var java = System.getProperty("java.home")
+    val jrePath = Paths.get(java)
+    val idePath = Paths.get(PathManager.getHomePath()).toRealPath()
+    if (jrePath.startsWith(idePath) && shouldCopy) {
+      val javaCopy = File(tempDir, "jre")
+      if (javaCopy.exists()) FileUtil.delete(javaCopy)
+      FileUtil.copyDir(File(java), javaCopy)
+      java = javaCopy.path
+    }
+    return java
+  }
+
+  private fun findLibsByPrefixes(vararg prefixes: String): Array<File> {
+    val lib = PathManager.getLibPath()
+    val libFiles = File(lib).listFiles { file -> startsWithAny(file.name, prefixes) }
+    if (libFiles == null || libFiles.isEmpty()) {
+      throw EventLogUploadException("Cannot find libraries from dependency for event log uploader")
+    }
+    return libFiles
+  }
+
+  private fun startsWithAny(str: String, prefixes: Array<out String>): Boolean {
+    for (prefix in prefixes) {
+      if (str.startsWith(prefix)) return true
+    }
+    return false
+  }
+
+  private fun getTempDir(): File {
+    val tempDir = File(PathManager.getTempPath(), "statistics-uploader")
+    if (!(tempDir.exists() || tempDir.mkdirs())) {
+      throw EventLogUploadException("Cannot create temp directory: $tempDir")
+    }
+    return tempDir
+  }
+}

platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogUploadException.java

@@ -0,0 +1,8 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog.uploader;
+
+public class EventLogUploadException extends Exception {
+  public EventLogUploadException(String s) {
+    super(s);
+  }
+}

platform/statistics/uploader/src/META-INF/MANIFEST.MF

@@ -0,0 +1,2 @@
+Manifest-Version: 1.0
+Main-Class: com.intellij.internal.statistic.uploader.EventLogUploader

platform/statistics/uploader/src/com/intellij/internal/statistic/eventLog/EmptyDataCollectorDebugLogger.java

@@ -0,0 +1,24 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog;
+
+public class EmptyDataCollectorDebugLogger implements DataCollectorDebugLogger {
+  @Override
+  public void info(String message) {}
+
+  @Override
+  public void info(String message, Throwable t) {}
+
+  @Override
+  public void warn(String message) {}
+
+  @Override
+  public void warn(String message, Throwable t) {}
+
+  @Override
+  public void trace(String message) {}
+
+  @Override
+  public boolean isTraceEnabled() {
+    return false;
+  }
+}

platform/statistics/uploader/src/com/intellij/internal/statistic/eventLog/config/EventLogExternalApplicationInfo.java

@@ -0,0 +1,53 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog.config;
+
+import com.intellij.internal.statistic.eventLog.DataCollectorDebugLogger;
+import com.intellij.internal.statistic.eventLog.EventLogApplicationInfo;
+import org.jetbrains.annotations.NotNull;
+
+public class EventLogExternalApplicationInfo implements EventLogApplicationInfo {
+  private final DataCollectorDebugLogger myLogger;
+
+  private final String myTemplateUrl;
+  private final String myProductCode;
+  private final boolean myIsInternal;
+  private final boolean myIsTest;
+
+  public EventLogExternalApplicationInfo(@NotNull String templateUrl, @NotNull String productCode,
+                                         boolean isInternal, boolean isTest,
+                                         @NotNull DataCollectorDebugLogger logger) {
+    myTemplateUrl = templateUrl;
+    myProductCode = productCode;
+    myIsInternal = isInternal;
+    myIsTest = isTest;
+    myLogger = logger;
+  }
+
+  @NotNull
+  @Override
+  public String getTemplateUrl() {
+    return myTemplateUrl;
+  }
+
+  @NotNull
+  @Override
+  public String getProductCode() {
+    return myProductCode;
+  }
+
+  @Override
+  public boolean isInternal() {
+    return myIsInternal;
+  }
+
+  @Override
+  public boolean isTest() {
+    return myIsTest;
+  }
+
+  @NotNull
+  @Override
+  public DataCollectorDebugLogger getLogger() {
+    return myLogger;
+  }
+}

platform/statistics/uploader/src/com/intellij/internal/statistic/eventLog/config/EventLogExternalRecorderConfig.java

@@ -0,0 +1,36 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog.config;
+
+import com.intellij.internal.statistic.eventLog.DefaultEventLogFilesProvider;
+import com.intellij.internal.statistic.eventLog.EventLogFilesProvider;
+import com.intellij.internal.statistic.eventLog.EventLogRecorderConfig;
+import org.jetbrains.annotations.NotNull;
+
+import java.nio.file.Paths;
+
+public class EventLogExternalRecorderConfig implements EventLogRecorderConfig {
+  private final String myRecorderId;
+  private final EventLogFilesProvider myFilesProvider;
+
+  public EventLogExternalRecorderConfig(@NotNull String recorderId, @NotNull String logRoot) {
+    myRecorderId = recorderId;
+    myFilesProvider = new DefaultEventLogFilesProvider(Paths.get(logRoot), () -> null);
+  }
+
+  @NotNull
+  @Override
+  public String getRecorderId() {
+    return myRecorderId;
+  }
+
+  @Override
+  public boolean isSendEnabled() {
+    return true;
+  }
+
+  @NotNull
+  @Override
+  public EventLogFilesProvider getLogFilesProvider() {
+    return myFilesProvider;
+  }
+}

platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploader.java

@@ -0,0 +1,105 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.uploader;
+
+import com.intellij.internal.statistic.connect.StatisticsResult;
+import com.intellij.internal.statistic.eventLog.*;
+import com.intellij.internal.statistic.eventLog.config.EventLogExternalApplicationInfo;
+import com.intellij.internal.statistic.eventLog.config.EventLogExternalRecorderConfig;
+import org.jetbrains.annotations.Nullable;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Map;
+
+public class EventLogUploader {
+  public static void main(String[] args) {
+    execute(args);
+  }
+
+  private static void execute(String[] args) {
+    DataCollectorDebugLogger logger = new EmptyDataCollectorDebugLogger();
+    logger.info("Process started with '" + String.join(" ", args) + "'");
+
+    if (args.length == 0) {
+      logger.warn("No arguments were found");
+      return;
+    }
+
+    Map<String, String> options = EventLogUploaderCliParser.parseOptions(args);
+    DeviceConfiguration device = newDeviceConfig(options);
+    if (device == null) {
+      logger.warn("Failed creating device config from arguments");
+      return;
+    }
+
+    EventLogRecorderConfig recorder = newRecorderConfig(options);
+    if (recorder == null) {
+      logger.warn("Failed creating recorder config from arguments");
+      return;
+    }
+
+    EventLogApplicationInfo appInfo = newApplicationInfo(options, logger);
+    if (appInfo == null) {
+      logger.warn("Failed creating application info from arguments");
+      return;
+    }
+
+    logger.info("Start uploading...");
+    logger.info("{url:" + appInfo.getTemplateUrl() + ", product:" + appInfo.getProductCode() + ", internal:" + appInfo.isInternal() + ", isTest:" + appInfo.isTest() + "}");
+    logger.info("{recorder:" + recorder.getRecorderId() + ", root:" + recorder.getLogFilesProvider().getLogFilesDir() + "}");
+    logger.info("{device:" + device.getDeviceId() + ", bucket:" + device.getBucket() + "}");
+    try {
+      //TODO: save the number of uploaded files and log it during the next IDE session
+      EventLogStatisticsService service = new EventLogStatisticsService(device, recorder, appInfo, null);
+      StatisticsResult result = service.send();
+      if (logger.isTraceEnabled()) {
+        logger.trace("Uploading finished with " + result.getCode().name());
+        logger.trace(result.getDescription());
+      }
+    }
+    catch (Exception e) {
+      logger.warn("Failed sending files: " + e.getMessage());
+    }
+  }
+
+  @Nullable
+  private static DeviceConfiguration newDeviceConfig(Map<String, String> options) {
+    try {
+      String bucketOption = options.get(EventLogUploaderOptions.BUCKET_OPTION);
+      String deviceOption = options.get(EventLogUploaderOptions.DEVICE_OPTION);
+      int bucketInt = bucketOption != null ? Integer.parseInt(bucketOption) : -1;
+      if (deviceOption != null && bucketInt >= 0 && bucketInt < 256) {
+        return new DeviceConfiguration(deviceOption, bucketInt);
+      }
+    }
+    catch (NumberFormatException e) {
+      // ignore
+    }
+    return null;
+  }
+
+  @Nullable
+  private static EventLogRecorderConfig newRecorderConfig(Map<String, String> options) {
+    String recorder = options.get(EventLogUploaderOptions.RECORDER_OPTION);
+    if (recorder != null) {
+      String dir = options.get(EventLogUploaderOptions.DIRECTORY_OPTION);
+      Path path = dir != null ? Paths.get(dir) : null;
+      if (path != null && path.toFile().exists()) {
+        return new EventLogExternalRecorderConfig(recorder, path.toString());
+      }
+    }
+    return null;
+  }
+
+  @Nullable
+  private static EventLogApplicationInfo newApplicationInfo(Map<String, String> options, DataCollectorDebugLogger logger) {
+    String url = options.get(EventLogUploaderOptions.URL_OPTION);
+    String productCode = options.get(EventLogUploaderOptions.PRODUCT_OPTION);
+    if (url != null && productCode != null) {
+      boolean isInternal = options.containsKey(EventLogUploaderOptions.INTERNAL_OPTION);
+      boolean isTest = options.containsKey(EventLogUploaderOptions.TEST_OPTION);
+      return new EventLogExternalApplicationInfo(url, productCode, isInternal, isTest, logger);
+    }
+    return null;
+  }
+}

platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploaderCliParser.java

@@ -0,0 +1,35 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.uploader;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class EventLogUploaderCliParser {
+
+  @NotNull
+  public static Map<String, String> parseOptions(String[] args) {
+    Map<String, String> options = new HashMap<>();
+    for (int i = 0, length = args.length; i < length; i++) {
+      String arg = args[i];
+      if (!isOptionName(arg)) continue;
+
+      if (requireValue(arg) && (i + 1) < length && !isOptionName(args[i + 1])) {
+        options.put(arg, args[i + 1]);
+      }
+      else {
+        options.put(arg, null);
+      }
+    }
+    return options;
+  }
+
+  private static boolean isOptionName(String arg) {
+    return arg.startsWith("--");
+  }
+
+  private static boolean requireValue(String arg) {
+    return !EventLogUploaderOptions.INTERNAL_OPTION.equals(arg) && !EventLogUploaderOptions.TEST_OPTION.equals(arg);
+  }
+}

platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploaderOptions.java

@@ -0,0 +1,15 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.uploader;
+
+public interface EventLogUploaderOptions {
+  String RECORDER_OPTION = "--recorder";
+  String DIRECTORY_OPTION = "--dir";
+
+  String DEVICE_OPTION = "--device";
+  String BUCKET_OPTION = "--bucket";
+
+  String URL_OPTION = "--url";
+  String PRODUCT_OPTION = "--product";
+  String INTERNAL_OPTION = "--internal";
+  String TEST_OPTION = "--test";
+}