IDEA-CR-61417: IDEA-227831 Implement environment tests to check sending logs

GitOrigin-RevId: c1e6f0c0c2999914cbf82d5e7226b17f5b884cbd

.idea/libraries/testcontainers.xml

@@ -0,0 +1,44 @@
+<component name="libraryTable">
+  <library name="testcontainers" type="repository">
+    <properties maven-id="org.testcontainers:testcontainers:1.10.6" />
+    <CLASSES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/testcontainers/testcontainers/1.10.6/testcontainers-1.10.6.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/junit/junit/4.12/junit-4.12.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/annotations/16.0.3/annotations-16.0.3.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-compress/1.18/commons-compress-1.18.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/xml/bind/jaxb-api/2.3.1/jaxb-api-2.3.1.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/duct-tape/duct-tape/1.0.7/duct-tape-1.0.7.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/visible-assertions/visible-assertions/2.1.2/visible-assertions-2.1.2.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/net/java/dev/jna/jna/5.2.0/jna-5.2.0.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/tcp-unix-socket-proxy/1.0.2/tcp-unix-socket-proxy-1.0.2.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/com/kohlschutter/junixsocket/junixsocket-native-common/2.0.4/junixsocket-native-common-2.0.4.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/scijava/native-lib-loader/2.0.2/native-lib-loader-2.0.2.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/com/kohlschutter/junixsocket/junixsocket-common/2.0.4/junixsocket-common-2.0.4.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/net/java/dev/jna/jna-platform/5.2.0/jna-platform-5.2.0.jar!/" />
+    </CLASSES>
+    <JAVADOC />
+    <SOURCES>
+      <root url="jar://$MAVEN_REPOSITORY$/org/testcontainers/testcontainers/1.10.6/testcontainers-1.10.6-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/junit/junit/4.12/junit-4.12-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/jetbrains/annotations/16.0.3/annotations-16.0.3-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/annotation/javax.annotation-api/1.3.2/javax.annotation-api-1.3.2-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/apache/commons/commons-compress/1.18/commons-compress-1.18-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/xml/bind/jaxb-api/2.3.1/jaxb-api-2.3.1-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/javax/activation/javax.activation-api/1.2.0/javax.activation-api-1.2.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/duct-tape/duct-tape/1.0.7/duct-tape-1.0.7-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/visible-assertions/visible-assertions/2.1.2/visible-assertions-2.1.2-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/net/java/dev/jna/jna/5.2.0/jna-5.2.0-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/rnorth/tcp-unix-socket-proxy/1.0.2/tcp-unix-socket-proxy-1.0.2-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/com/kohlschutter/junixsocket/junixsocket-native-common/2.0.4/junixsocket-native-common-2.0.4-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/org/scijava/native-lib-loader/2.0.2/native-lib-loader-2.0.2-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/com/kohlschutter/junixsocket/junixsocket-common/2.0.4/junixsocket-common-2.0.4-sources.jar!/" />
+      <root url="jar://$MAVEN_REPOSITORY$/net/java/dev/jna/jna-platform/5.2.0/jna-platform-5.2.0-sources.jar!/" />
+    </SOURCES>
+  </library>
+</component>

.idea/modules.xml

@@ -687,6 +687,7 @@
       <module fileurl="file://$PROJECT_DIR$/platform/smRunner/intellij.platform.smRunner.iml" filepath="$PROJECT_DIR$/platform/smRunner/intellij.platform.smRunner.iml" />
       <module fileurl="file://$PROJECT_DIR$/platform/statistics/intellij.platform.statistics.iml" filepath="$PROJECT_DIR$/platform/statistics/intellij.platform.statistics.iml" />
       <module fileurl="file://$PROJECT_DIR$/platform/statistics/devkit/intellij.platform.statistics.devkit.iml" filepath="$PROJECT_DIR$/platform/statistics/devkit/intellij.platform.statistics.devkit.iml" />
+      <module fileurl="file://$PROJECT_DIR$/platform/statistics/envTests/intellij.platform.statistics.envTests.iml" filepath="$PROJECT_DIR$/platform/statistics/envTests/intellij.platform.statistics.envTests.iml" />
       <module fileurl="file://$PROJECT_DIR$/platform/statistics/uploader/intellij.platform.statistics.uploader.iml" filepath="$PROJECT_DIR$/platform/statistics/uploader/intellij.platform.statistics.uploader.iml" />
       <module fileurl="file://$PROJECT_DIR$/platform/structuralsearch/intellij.platform.structuralSearch.iml" filepath="$PROJECT_DIR$/platform/structuralsearch/intellij.platform.structuralSearch.iml" />
       <module fileurl="file://$PROJECT_DIR$/platform/structuralsearch/intellij.platform.structuralSearch.tests.iml" filepath="$PROJECT_DIR$/platform/structuralsearch/intellij.platform.structuralSearch.tests.iml" />

platform/statistics/devkit/src/com/intellij/internal/statistic/actions/SendEventLogAction.java

@@ -108,12 +108,12 @@ public class SendEventLogAction extends AnAction {
     private final List<LogEventRecordRequest> myFailed = new ArrayList<>();
 
     @Override
-    public void onSucceed(@NotNull LogEventRecordRequest request) {
+    public void onSucceed(@NotNull LogEventRecordRequest request, @NotNull String content) {
       mySucceed.add(request);
     }
 
     @Override
-    public void onFailed(@Nullable LogEventRecordRequest request) {
+    public void onFailed(@Nullable LogEventRecordRequest request, @Nullable String content) {
       if (request != null) {
         myFailed.add(request);
       }

platform/statistics/envTests/intellij.platform.statistics.envTests.iml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="TEST" name="kotlin-stdlib-jdk8" level="project" />
+    <orderEntry type="library" scope="TEST" name="JUnit4" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.testFramework" scope="TEST" />
+    <orderEntry type="library" scope="TEST" name="testcontainers" level="project" />
+    <orderEntry type="module" module-name="intellij.platform.statistics" />
+    <orderEntry type="library" scope="TEST" name="gson" level="project" />
+  </component>
+</module>

platform/statistics/envTests/test/com/intellij/internal/statistic/envTest/ApacheContainer.java

@@ -0,0 +1,35 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.envTest;
+
+import com.intellij.openapi.util.text.StringUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+final class ApacheContainer extends GenericContainer<ApacheContainer> {
+  private static final int DEFAULT_PORT = 80;
+  private static final String REMOTE_ROOT = "/var/www/html";
+
+  ApacheContainer(@NotNull String localPath) {
+    super("php:7.1-apache");
+    addFileSystemBind(localPath, REMOTE_ROOT, BindMode.READ_ONLY);
+  }
+
+  @Override
+  protected void configure() {
+    addExposedPort(DEFAULT_PORT);
+  }
+
+  @NotNull
+  public URL getBaseUrl(@Nullable String path) throws MalformedURLException {
+    String base = "http://" + getContainerIpAddress() + ":" + getMappedPort(DEFAULT_PORT) + "/";
+    if (StringUtil.isNotEmpty(path)) {
+      base += path;
+    }
+    return new URL(base);
+  }
+}

platform/statistics/envTests/test/com/intellij/internal/statistic/envTest/EventLogStatisticsServiceTest.kt

@@ -0,0 +1,174 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.envTest
+
+import com.google.gson.GsonBuilder
+import com.google.gson.JsonObject
+import com.intellij.internal.statistic.connect.StatisticsResult
+import com.intellij.internal.statistic.eventLog.*
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.util.io.readText
+import junit.framework.TestCase
+import java.io.File
+import java.nio.file.Path
+import java.nio.file.Paths
+
+internal class EventLogStatisticsServiceTest : StatisticsUploaderBaseTest() {
+  private val recorderId = "FUS"
+  private val productCode = "IC"
+  private val gson = GsonBuilder().registerTypeAdapter(LogEvent::class.java, LogEventJsonDeserializer()).create()
+
+  fun testSend() {
+    val logText = Paths.get(getTestDataPath()).resolve("log1.txt").readText()
+    val requests = doTest(listOf(createTempFile("testLogFile.log", logText)))
+
+    TestCase.assertEquals(1, requests.size)
+    val logEventRecordRequest = requests.first()
+    TestCase.assertEquals(recorderId, logEventRecordRequest.recorder)
+    TestCase.assertEquals(productCode, logEventRecordRequest.product)
+    TestCase.assertEquals(1, logEventRecordRequest.records.size)
+    val events = logEventRecordRequest.records.first().events
+    TestCase.assertEquals(1, events.size)
+    val expected = gson.fromJson(logText, LogEvent::class.java)
+    TestCase.assertEquals(expected, events.first())
+  }
+
+  fun testSendMultipleLogFiles() {
+    val logFiles = listOf(
+      createLogFile("testLogFile1.log", "log1.txt"),
+      createLogFile("testLogFile2.log", "log2.txt")
+    )
+    val requests = doTest(logFiles)
+    TestCase.assertEquals(logFiles.size, requests.size)
+  }
+
+  fun testSendIfSettingsUnreachable() {
+    val logFile = createLogFile("testLogFile.log", "log1.txt")
+    val service = createEventLogStatisticsService(listOf(logFile), settingsResponseFile = "non-existent.php")
+    val resultHolder = ResultHolder()
+    val result = service.send(resultHolder)
+    TestCase.assertEquals(result.description, StatisticsResult.ResultCode.ERROR_IN_CONFIG, result.code)
+    TestCase.assertEquals(0, resultHolder.succeed)
+    TestCase.assertTrue(logFile.exists())
+  }
+
+  fun testSendIfServerUnreachable() {
+    createSettingsResponseScript(sendResponseFile = "non-existent.php")
+    val logFile = createLogFile("testLogFile.log", "log1.txt")
+    val service = createEventLogStatisticsService(listOf(logFile))
+
+    val resultHolder = ResultHolder()
+    val result = service.send(resultHolder)
+    TestCase.assertEquals(result.description, StatisticsResult.ResultCode.SENT_WITH_ERRORS, result.code)
+    TestCase.assertEquals(0, resultHolder.succeed)
+    TestCase.assertTrue(logFile.exists())
+  }
+
+  fun testSendIfWhitelistUnreachable() {
+    createSettingsResponseScript(whitelistResponseFile = "non-existent.php")
+    val logFile = createLogFile("testLogFile.log", "log1.txt")
+    val service = createEventLogStatisticsService(listOf(logFile))
+    val resultHolder = ResultHolder()
+    val result = service.send(resultHolder)
+    TestCase.assertEquals(result.description, StatisticsResult.ResultCode.SENT_WITH_ERRORS, result.code)
+    TestCase.assertEquals(0, resultHolder.succeed)
+    TestCase.assertTrue(!logFile.exists())
+  }
+
+  fun testNotSendIfSendDisabled() {
+    val logFile = createLogFile("testLogFile.log", "log1.txt")
+    val service = createEventLogStatisticsService(listOf(logFile), sendEnabled = false)
+    val resultHolder = ResultHolder()
+    val result = service.send(resultHolder)
+    TestCase.assertEquals(result.description, StatisticsResult.ResultCode.NOTHING_TO_SEND, result.code)
+    TestCase.assertEquals(0, resultHolder.succeed)
+    TestCase.assertTrue(!logFile.exists())
+  }
+
+  private fun createLogFile(targetName: String, sourceName: String): File {
+    return createTempFile(targetName, Paths.get(getTestDataPath()).resolve(sourceName).readText())
+  }
+
+  private fun doTest(logFiles: List<File>): List<LogEventRecordRequest> {
+    createSettingsResponseScript()
+    val service = createEventLogStatisticsService(logFiles)
+    val resultHolder = ResultHolder()
+
+    val result = service.send(resultHolder)
+    TestCase.assertEquals(result.description, StatisticsResult.ResultCode.SEND, result.code)
+    for (logFile in logFiles) {
+      TestCase.assertTrue(!logFile.exists())
+    }
+
+    return resultHolder.successContents.map {
+      val jsonObject = gson.fromJson(it, JsonObject::class.java)
+      TestCase.assertEquals(jsonObject["method"].asString, "POST")
+      gson.fromJson(jsonObject["body"].asString, LogEventRecordRequest::class.java)
+    }
+  }
+
+  private fun createSettingsResponseScript(sendResponseFile: String = "dump-request.php", whitelistResponseFile: String = "") {
+    val sendUrl = container.getBaseUrl(sendResponseFile).toString()
+    val whitelistUrl = container.getBaseUrl(whitelistResponseFile).toString()
+    val settings = """<service url="$sendUrl" percent-traffic="100" white-list-service="$whitelistUrl"/>"""
+    FileUtil.writeToFile(Paths.get(tmpLocalRoot).resolve("settings.php").toFile(), settings)
+  }
+
+  private fun createEventLogStatisticsService(logFiles: List<File>,
+                                              settingsResponseFile: String = "settings.php",
+                                              sendEnabled: Boolean = true): EventLogStatisticsService {
+    val applicationInfo = TestApplicationInfo(recorderId, productCode, container.getBaseUrl(settingsResponseFile).toString())
+    return EventLogStatisticsService(
+      DeviceConfiguration(EventLogConfiguration.deviceId, EventLogConfiguration.bucket),
+      TestEventLogRecorderConfig(recorderId, logFiles, sendEnabled),
+      EventLogSendListener { _, _, _ -> Unit },
+      EventLogUploadSettingsService(recorderId, applicationInfo)
+    )
+  }
+
+  private class TestApplicationInfo(recorderId: String,
+                                    val product: String,
+                                    val settingsUrl: String) : EventLogInternalApplicationInfo(recorderId, true) {
+    override fun getProductCode(): String = product
+    override fun getTemplateUrl(): String = settingsUrl
+  }
+
+  private class TestEventLogRecorderConfig(recorderId: String,
+                                           logFiles: List<File>,
+                                           val sendEnabled: Boolean = true) : EventLogInternalRecorderConfig(recorderId) {
+    private val evenLogFilesProvider = object : EventLogFilesProvider {
+      override fun getLogFilesDir(): Path? = null
+
+      override fun getLogFiles(): List<EventLogFile> = logFiles.map { EventLogFile(it) }
+    }
+
+    override fun isSendEnabled(): Boolean = sendEnabled
+
+    override fun getLogFilesProvider(): EventLogFilesProvider = evenLogFilesProvider
+  }
+
+  private class ResultHolder : EventLogResultDecorator {
+    var failed = 0
+    var succeed = 0
+    val successContents = mutableListOf<String>()
+
+    override fun onSucceed(request: LogEventRecordRequest, content: String) {
+      succeed++
+      successContents.add(content)
+    }
+
+    override fun onFailed(request: LogEventRecordRequest?, content: String?) {
+      failed++
+    }
+
+    override fun onFinished(): StatisticsResult {
+      val total = succeed + failed
+      if (total == 0) {
+        return StatisticsResult(StatisticsResult.ResultCode.NOTHING_TO_SEND, "No files to upload.")
+      }
+      else if (failed > 0) {
+        return StatisticsResult(StatisticsResult.ResultCode.SENT_WITH_ERRORS, "Uploaded $succeed out of $total files.")
+      }
+      return StatisticsResult(StatisticsResult.ResultCode.SEND, "Uploaded $succeed files.")
+    }
+  }
+}

platform/statistics/envTests/test/com/intellij/internal/statistic/envTest/StatisticsUploaderBaseTest.kt

@@ -0,0 +1,49 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.envTest
+
+import com.intellij.openapi.Disposable
+import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.util.Disposer
+import com.intellij.openapi.util.io.FileUtil
+import com.intellij.openapi.vfs.LocalFileSystem
+import com.intellij.testFramework.HeavyPlatformTestCase
+import com.intellij.testFramework.ThreadTracker
+import junit.framework.TestCase
+import java.io.File
+
+internal abstract class StatisticsUploaderBaseTest : HeavyPlatformTestCase() {
+  private val longRunningThreadDisposable: Disposable = Disposer.newDisposable()
+  protected lateinit var container: ApacheContainer
+  protected lateinit var tmpLocalRoot: String
+
+  override fun setUp() {
+    super.setUp()
+    val tempDir = LocalFileSystem.getInstance().findFileByPath(FileUtil.getTempDirectory())
+    TestCase.assertNotNull(tempDir)
+    tmpLocalRoot = createChildDirectory(tempDir!!, "docker_server_root").path
+    FileUtil.copyDir(File("${getTestDataPath()}/default_root"), File(tmpLocalRoot))
+
+    ThreadTracker.longRunningThreadCreated(longRunningThreadDisposable, "Okio Watchdog")
+    ThreadTracker.longRunningThreadCreated(longRunningThreadDisposable, "testcontainers-ryuk")
+
+    container = ApacheContainer(tmpLocalRoot)
+    container.start()
+  }
+
+  protected fun getTestDataPath(): String {
+    return FileUtil.toSystemIndependentName(PathManager.getCommunityHomePath()) + "/platform/statistics/envTests/testData/"
+  }
+
+  override fun tearDown() {
+    try {
+      container.stop()
+    }
+    catch (e: Throwable) {
+      addSuppressedException(e)
+    }
+    finally {
+      super.tearDown()
+      Disposer.dispose(longRunningThreadDisposable)
+    }
+  }
+}

platform/statistics/envTests/testData/default_root/IC.json

@@ -0,0 +1,153 @@
+{
+  "groups" : [ {
+    "id" : "actions",
+    "builds" : [ ],
+    "versions" : [ {
+      "from" : "9"
+    } ],
+    "rules" : {
+      "event_id" : [ "{util#action}", "{enum:action.invoked|custom.action.invoked}" ],
+      "event_data" : {
+        "action_id" : [ "{util#action}" ],
+        "class" : [ "{util#class_name}" ],
+        "context_menu" : [ "{enum#boolean}" ],
+        "current_file" : [ "{util#current_file}" ],
+        "input_event" : [ "{util#shortcut}" ],
+        "parent" : [ "{enum#parent}", "{util#class_name}", "{enum:LineMarkerActionWrapper|TreeActionWrapper|MyTreeActionWrapper}" ],
+        "place" : [ "{util#place}" ],
+        "plugin" : [ "{util#plugin}" ],
+        "plugin_type" : [ "{util#plugin_type}" ],
+        "plugin_version" : [ "{util#plugin_version}" ],
+        "toolwindow" : [ "{util#toolwindow}" ]
+      },
+      "enums" : {
+        "parent" : [ "LineMarkerActionWrapper", "TreeActionWrapper", "MyTreeActionWrapper" ]
+      }
+    }
+  }, {
+    "id" : "lifecycle",
+    "builds" : [ {
+      "from" : "191.4738"
+    } ],
+    "rules" : {
+      "event_id" : [ "{enum#__event_id}" ],
+      "event_data" : {
+        "command_line" : [ "{enum#boolean}" ],
+        "debug_agent" : [ "{enum#boolean}" ],
+        "duration_grouped" : [ "{regexp#integer}s", "-{regexp#integer}s", "{regexp#integer}s+" ],
+        "duration_ms" : [ "{regexp#integer}" ],
+        "duration_s" : [ "{regexp#integer}" ],
+        "eap" : [ "{enum#boolean}" ],
+        "error" : [ "{util#class_name}" ],
+        "error_frames" : [ "{util#method_name}" ],
+        "error_hash" : [ "{regexp#integer}" ],
+        "error_size" : [ "{regexp#integer}" ],
+        "headless" : [ "{enum#boolean}" ],
+        "internal" : [ "{enum#boolean}" ],
+        "mapping_failed" : [ "{enum#boolean}" ],
+        "memory_error_kind" : [ "{enum:heap|perm_gen|metaspace|code_cache}" ],
+        "oom" : [ "{enum#boolean}" ],
+        "plugin" : [ "{util#plugin}" ],
+        "plugin_type" : [ "{util#plugin_type}" ],
+        "plugin_version" : [ "{util#plugin_version}" ],
+        "restart" : [ "{enum#boolean}" ],
+        "test" : [ "{enum#boolean}" ],
+        "time_ms" : [ "{regexp#integer}" ],
+        "too_many_errors" : [ "{enum#boolean}" ]
+      },
+      "enums" : {
+        "__event_id" : [ "ide.error", "ide.freeze", "ide.start", "ide.close", "project.opening.finished", "project.opened", "project.closed", "frame.activated", "frame.deactivated", "project.module.attached" ]
+      }
+    }
+  }, {
+    "id" : "productivity",
+    "builds" : [ ],
+    "versions" : [ {
+      "from" : "11"
+    } ],
+    "rules" : {
+      "event_id" : [ "{util#productivity}", "{enum:feature.used}" ],
+      "event_data" : {
+        "group" : [ "{util#productivity_group}" ],
+        "id" : [ "{util#productivity}" ],
+        "plugin" : [ "{util#plugin}" ],
+        "plugin_type" : [ "{util#plugin_type}" ],
+        "plugin_version" : [ "{util#plugin_version}" ]
+      }
+    }
+  },{
+    "id" : "refactoring",
+    "builds" : [ ],
+    "versions" : [ {
+      "from" : "1"
+    } ],
+    "rules" : {
+      "event_id" : [ "{enum:handler.invoked}" ],
+      "event_data" : {
+        "element" : [ "{util#class_name}" ],
+        "handler" : [ "{util#class_name}" ],
+        "lang" : [ "{util#lang}" ]
+      }
+    }
+  }, {
+    "id" : "toolwindow",
+    "builds" : [ ],
+    "versions" : [ {
+      "from" : "6"
+    } ],
+    "rules" : {
+      "event_id" : [ "{util#toolwindow}", "{enum:activated}", "{enum:activated|shown|hidden}" ],
+      "event_data" : {
+        "Location" : [ "{enum#__Location}" ],
+        "ViewMode" : [ "{enum#__ViewMode}" ],
+        "id" : [ "{util#toolwindow}" ],
+        "plugin" : [ "{util#plugin}" ],
+        "plugin_type" : [ "{util#plugin_type}" ],
+        "plugin_version" : [ "{util#plugin_version}" ]
+      },
+      "enums" : {
+        "__Location" : [ "Left_Top", "Left_Bottom", "Bottom_Left", "Bottom_Right", "Right_Bottom", "Right_Top", "Top_Right", "Top_Left" ],
+        "__ViewMode" : [ "Dock_Pinned", "DockPinned", "Dock_Unpinned", "DockUnpinned", "Undock", "Float", "Window" ]
+      }
+    }
+  }, {
+    "id" : "ui.dialogs",
+    "builds" : [ ],
+    "versions" : [ {
+      "from" : "6"
+    } ],
+    "rules" : {
+      "event_id" : [ "{util#dialog_class}", "{enum:show|close}", "{enum:show|close|help.clicked}" ],
+      "event_data" : {
+        "code" : [ "{enum:0|1|2}" ],
+        "dialog_class" : [ "{util#dialog_class}" ],
+        "plugin" : [ "{util#plugin}" ],
+        "plugin_type" : [ "{util#plugin_type}" ],
+        "plugin_version" : [ "{util#plugin_version}" ],
+        "type" : [ "{enum:show|close}" ]
+      }
+    }
+  }],
+  "rules" : {
+    "enums" : {
+      "boolean" : [ "true", "false" ],
+      "build_tools" : [ "Maven", "Gradle", "gradle", "sbt", "Clojure_Deps", "clojure_deps", "Pants", "pants", "bsp", "Kobalt", "kobalt", "undefined.system", "third.party", "SPM", "CompDB", "Compilation_Database", "Makefile" ],
+      "os" : [ "Windows", "Mac", "Linux", "FreeBSD", "Other" ],
+      "plugin_type" : [ "PLATFORM", "JB_BUNDLED", "JB_NOT_BUNDLED", "LISTED", "NOT_LISTED", "UNKNOWN" ],
+      "state" : [ "enabled", "disabled" ],
+      "vcs" : [ "Git", "git", "SVN", "svn", "hg4idea", "Perforce", "perforce", "TFS", "tfs", "SourceSafe", "sourcesafe", "ClearCase", "clearcase", "CVS", "cvs", "TFVS", "tfvs", "VSS", "vss" ]
+    },
+    "regexps" : {
+      "count" : "\\d+K?M?\\+?",
+      "float" : "-?\\d+\\.\\d+",
+      "hash" : "[0-9A-Fa-f]{40,64}",
+      "integer" : "-?\\d+(\\+)?",
+      "kotlin_version" : "\\d\\.\\d\\.\\d{1,3}-(dev|eap|release)-(\\d+-)?(AppCode|CLion|IJ|Studio)[0-9\\-\\.]+",
+      "license_metadata" : "[0-9]{10}[A-Z]{4}[-0-9X]{6}",
+      "mcu_name" : "UNKNOWN|UPDATE_FAILED|(STM32[A-Z]{1,2}\\w+)",
+      "series" : "^(?:AreaRange|Area|Bubble|Heatmap|Pie|Stock|Scatter|Line|Bar)+(?:_(?:AreaRange|Area|Bubble|Heatmap|Pie|Stock|Scatter|Line|Bar)+)*$",
+      "version" : "(\\d+\\.?)*\\d+"
+    }
+  },
+  "version" : "1237"
+}

platform/statistics/envTests/testData/default_root/dump-request.php

@@ -0,0 +1,11 @@
+<?php
+
+$info = array(
+    "method" => $_SERVER["REQUEST_METHOD"],
+    "params" => $_GET,
+    "headers" => getallheaders(),
+    "cookie" => $_COOKIE,
+    "body" => gzdecode(file_get_contents('php://input'))
+);
+
+echo json_encode($info);

platform/statistics/envTests/testData/log1.txt

@@ -0,0 +1 @@
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586765061695,"group":{"id":"productivity","version":"40"},"event":{"count":1,"data":{"created":1586765061695,"plugin_type":"PLATFORM","id":"editing.select.word","group":"editing"},"id":"feature.used"}}

platform/statistics/envTests/testData/log2.txt

@@ -0,0 +1,10 @@
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833613859,"group":{"id":"ui.dialogs","version":"40"},"event":{"count":1,"data":{"code":1,"created":1586833613859,"plugin_type":"PLATFORM","dialog_class":"com.intellij.openapi.progress.util.ProgressDialog$MyDialogWrapper"},"id":"close"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833622940,"group":{"id":"toolwindow","version":"40"},"event":{"count":1,"data":{"ViewMode":"DockPinned","created":1586833622940,"plugin_type":"PLATFORM","id":"Project","Location":"Left_Top"},"id":"activated"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833625621,"group":{"id":"actions","version":"40"},"event":{"count":1,"data":{"context_menu":false,"current_file":"TEXT","action_id":"RenameElement","created":1586833625621,"plugin_type":"PLATFORM","project":"testProject","place":"keyboard_shortcut","class":"com.intellij.refactoring.actions.RenameElementAction","input_event":"Shift+F6"},"id":"action.invoked"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833625626,"group":{"id":"refactoring","version":"1"},"event":{"count":1,"data":{"handler":"com.intellij.refactoring.rename.PsiElementRenameHandler","created":1586833625626,"plugin_type":"PLATFORM","project":"testProject","lang":"TEXT","element":"com.intellij.psi.impl.source.PsiPlainTextFileImpl"},"id":"handler.invoked"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833625627,"group":{"id":"productivity","version":"40"},"event":{"count":1,"data":{"created":1586833625627,"plugin_type":"PLATFORM","id":"refactoring.rename","group":"refactoring"},"id":"feature.used"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833625667,"group":{"id":"ui.dialogs","version":"40"},"event":{"count":1,"data":{"created":1586833625667,"plugin_type":"PLATFORM","dialog_class":"com.intellij.refactoring.rename.RenamePsiFileProcessor$PsiFileRenameDialog"},"id":"show"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833627262,"group":{"id":"actions","version":"40"},"event":{"count":1,"data":{"context_menu":false,"action_id":"EditorRight","created":1586833627262,"plugin_type":"PLATFORM","project":"testProject","place":"keyboard_shortcut","class":"com.intellij.openapi.editor.actions.MoveCaretRightAction","input_event":"Right"},"id":"action.invoked"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833627422,"group":{"id":"ui.dialogs","version":"40"},"event":{"count":1,"data":{"created":1586833627423,"plugin_type":"PLATFORM","dialog_class":"com.intellij.refactoring.rename.RenamePsiFileProcessor$PsiFileRenameDialog"},"id":"help.clicked"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833627858,"group":{"id":"lifecycle","version":"40"},"event":{"count":1,"data":{"created":1586833627858,"project":"testProject"},"id":"frame.deactivated"}}
+{"recorder_version":"40","session":"12345","build":"202.2171","bucket":"175","time":1586833630531,"group":{"id":"lifecycle","version":"40"},"event":{"count":1,"data":{"created":1586833630531,"project":"testProject"},"id":"frame.activated"}}

platform/statistics/uploader/src/com/intellij/internal/statistic/connect/SettingsConnectionService.java

@@ -5,6 +5,8 @@ import com.intellij.internal.statistic.StatisticsEventLogUtil;
 import com.intellij.internal.statistic.eventLog.DataCollectorDebugLogger;
 import com.intellij.internal.statistic.eventLog.DataCollectorSystemEventLogger;
 import org.apache.http.HttpEntity;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpGet;
 import org.jdom.Element;
 import org.jdom.JDOMException;
@@ -68,9 +70,10 @@ public abstract class SettingsConnectionService {
     if (mySettingsUrl == null) return Collections.emptyMap();
 
     try {
-      HttpEntity entity = StatisticsEventLogUtil.create(myUserAgent).execute(new HttpGet(mySettingsUrl)).getEntity();
+      CloseableHttpResponse response = StatisticsEventLogUtil.create(myUserAgent).execute(new HttpGet(mySettingsUrl));
+      HttpEntity entity = response.getEntity();
       InputStream content = entity != null ? entity.getContent() : null;
-      if (content != null) {
+      if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && content != null) {
         Map<String, String> settings = new LinkedHashMap<>();
         try {
           Element root = StatisticsEventLogUtil.parseXml(content);

platform/statistics/uploader/src/com/intellij/internal/statistic/eventLog/EventLogResultDecorator.java

@@ -8,9 +8,9 @@ import org.jetbrains.annotations.Nullable;
 public interface EventLogResultDecorator {
   default void onLogsLoaded(int localFiles) {}
 
-  void onSucceed(@NotNull LogEventRecordRequest request);
+  void onSucceed(@NotNull LogEventRecordRequest request, @NotNull String content);
 
-  void onFailed(@Nullable LogEventRecordRequest request);
+  void onFailed(@Nullable LogEventRecordRequest request, @Nullable String content);
 
   @NotNull
   StatisticsResult onFinished();

platform/statistics/uploader/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsService.java

@@ -17,6 +17,7 @@ import org.apache.http.entity.StringEntity;
 import org.apache.http.util.EntityUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
+import org.jetbrains.annotations.TestOnly;
 
 import java.io.File;
 import java.io.IOException;
@@ -46,6 +47,17 @@ public class EventLogStatisticsService implements StatisticsService {
     mySendListener = listener;
   }
 
+  @TestOnly
+  public EventLogStatisticsService(@NotNull DeviceConfiguration device,
+                                   @NotNull EventLogRecorderConfig config,
+                                   @Nullable EventLogSendListener listener,
+                                   @Nullable EventLogUploadSettingsService settingsService) {
+    myDeviceConfiguration = device;
+    myRecorderConfiguration = config;
+    mySettingsService = settingsService;
+    mySendListener = listener;
+  }
+
   @Override
   public StatisticsResult send() {
     return send(myDeviceConfiguration, myRecorderConfiguration, mySettingsService, new EventLogCounterResultDecorator(mySendListener));
@@ -99,7 +111,7 @@ public class EventLogStatisticsService implements StatisticsService {
           if (logger.isTraceEnabled()) {
             logger.trace(file.getName() + "-> " + error);
           }
-          decorator.onFailed(recordRequest);
+          decorator.onFailed(recordRequest, null);
           toRemove.add(file);
           continue;
         }
@@ -107,19 +119,20 @@ public class EventLogStatisticsService implements StatisticsService {
         try {
           HttpResponse response = execute(info.getUserAgent(), serviceUrl, recordRequest);
           int code = response.getStatusLine().getStatusCode();
+          String content = getResponseMessage(response);
           if (code == HttpStatus.SC_OK) {
-            decorator.onSucceed(recordRequest);
+            decorator.onSucceed(recordRequest, content);
             toRemove.add(file);
           }
           else {
-            decorator.onFailed(recordRequest);
+            decorator.onFailed(recordRequest, content);
             if (code == HttpURLConnection.HTTP_BAD_REQUEST) {
               toRemove.add(file);
             }
           }
 
           if (logger.isTraceEnabled()) {
-            logger.trace(file.getName() + " -> " + getResponseMessage(response));
+            logger.trace(file.getName() + " -> " + content);
           }
         }
         catch (Exception e) {
@@ -238,12 +251,12 @@ public class EventLogStatisticsService implements StatisticsService {
     }
 
     @Override
-    public void onSucceed(@NotNull LogEventRecordRequest request) {
+    public void onSucceed(@NotNull LogEventRecordRequest request, @NotNull String content) {
       mySucceed++;
     }
 
     @Override
-    public void onFailed(@Nullable LogEventRecordRequest request) {
+    public void onFailed(@Nullable LogEventRecordRequest request, @Nullable String content) {
       myFailed++;
     }