Feature usage log: upload event log to statistic server

2026-09-27 10:03:11 +07:00 · 2018-02-14 18:26:11 +01:00
parent 910cbb04ba
commit 57a1ca5bd9
10 changed files with 235 additions and 5 deletions
@@ -99,6 +99,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
  private String myFUStatisticsSettingsUrl;
  private String myStatisticsServiceUrl;
  private String myStatisticsServiceKey;
+  private String myEventLogSettingsUrl;
  private String myThirdPartySoftwareUrl;
  private String myJetbrainsTvUrl;
  private String myEvalLicenseUrl = "https://www.jetbrains.com/store/license.html";
@@ -184,6 +185,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
  private static final String ATTRIBUTE_FU_STATISTICS_SETTINGS = "fus-settings";
  private static final String ATTRIBUTE_STATISTICS_SERVICE = "service";
  private static final String ATTRIBUTE_STATISTICS_SERVICE_KEY = "service-key";
+  private static final String ATTRIBUTE_EVENT_LOG_STATISTICS_SETTINGS = "event-log-settings";
  private static final String ELEMENT_THIRD_PARTY = "third-party";
  private static final String ELEMENT_JB_TV = "jetbrains-tv";
  private static final String CUSTOMIZE_IDE_WIZARD_STEPS = "customize-ide-wizard";
@@ -545,6 +547,10 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
    return myStatisticsServiceKey;
  }

+  public String getEventLogSettingsUrl() {
+    return myEventLogSettingsUrl;
+  }
+
  @Override
  public String getThirdPartySoftwareURL() {
    return myThirdPartySoftwareUrl;
@@ -880,12 +886,14 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
      myFUStatisticsSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_FU_STATISTICS_SETTINGS);
      myStatisticsServiceUrl  = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE);
      myStatisticsServiceKey  = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE_KEY);
+      myEventLogSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_EVENT_LOG_STATISTICS_SETTINGS);
    }
    else {
      myStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/stat-assistant.xml";
      myFUStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/fus-assistant.xml";
      myStatisticsServiceUrl  = "https://www.jetbrains.com/idea/statistics/index.jsp";
      myStatisticsServiceKey  = null;
+      myEventLogSettingsUrl = "https://www.jetbrains.com/idea/statistics/fus-lion-assistant.xml";
    }

    Element thirdPartyElement = getChild(parentNode, ELEMENT_THIRD_PARTY);
@@ -0,0 +1,41 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.actions;
+
+import com.intellij.internal.statistic.connect.StatisticsResult;
+import com.intellij.internal.statistic.connect.StatisticsService;
+import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
+import com.intellij.openapi.actionSystem.AnAction;
+import com.intellij.openapi.actionSystem.AnActionEvent;
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ModalityState;
+import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.progress.Task;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.ui.Messages;
+import com.intellij.openapi.util.text.StringUtil;
+import org.jetbrains.annotations.NotNull;
+
+public class SendEventLogAction extends AnAction {
+
+  @Override
+  public void actionPerformed(@NotNull AnActionEvent e) {
+    final Project project = e.getProject();
+    if (project == null) {
+      return;
+    }
+
+    ProgressManager.getInstance().run(new Task.Backgroundable(project, "Send Feature Usage Event Log", false) {
+      @Override
+      public void run(@NotNull ProgressIndicator indicator) {
+        StatisticsService service = StatisticsUploadAssistant.getEventLogStatisticsService();
+        final StatisticsResult result = service.send();
+
+        ApplicationManager.getApplication().invokeLater(
+          () -> Messages.showMultilineInputDialog(project, "Result: " + result.getCode(), "Statistics Result",
+                                                  StringUtil.replace(result.getDescription(), ";", "\n"),
+                                                  null, null), ModalityState.NON_MODAL, project.getDisposed());
+      }
+    });
+  }
+}
@@ -0,0 +1,131 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog;
+
+import com.intellij.internal.statistic.connect.StatServiceException;
+import com.intellij.internal.statistic.connect.StatisticsResult;
+import com.intellij.internal.statistic.connect.StatisticsResult.ResultCode;
+import com.intellij.internal.statistic.connect.StatisticsService;
+import com.intellij.notification.Notification;
+import com.intellij.notification.NotificationListener;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.io.StreamUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class EventLogStatisticsService implements StatisticsService {
+  private static final Logger LOG = Logger.getInstance("com.intellij.internal.statistic.eventLog.EventLogStatisticsService");
+
+  private static final EventLogStatisticsSettingsService mySettingsService = EventLogStatisticsSettingsService.getInstance();
+
+  @Override
+  public StatisticsResult send() {
+    if (!FeatureUsageLogger.INSTANCE.isEnabled()) {
+      throw new StatServiceException("Event Log collector is not enabled");
+    }
+
+    final String serviceUrl = mySettingsService.getServiceUrl();
+    if (serviceUrl == null) {
+      return new StatisticsResult(StatisticsResult.ResultCode.ERROR_IN_CONFIG, "ERROR: unknown Statistics Service URL.");
+    }
+
+    if (!mySettingsService.isTransmissionPermitted()) {
+      return new StatisticsResult(StatisticsResult.ResultCode.NOT_PERMITTED_SERVER, "NOT_PERMITTED");
+    }
+
+    try {
+      int succeed = 0;
+      final List<File> logs = FeatureUsageLogger.INSTANCE.getLogFiles();
+      final List<File> toRemove = new ArrayList<>(logs.size());
+      for (File file : logs) {
+        final LogEventContent session = LogEventContent.Companion.create(file);
+        if (session != null) {
+          final HttpClient httpClient = HttpClientBuilder.create().build();
+          final HttpPost post = createPostRequest(serviceUrl, LogEventSerializer.INSTANCE.toString(session));
+          final HttpResponse response = httpClient.execute(post);
+
+          final int code = response.getStatusLine().getStatusCode();
+          if (code == HttpStatus.SC_OK) {
+            toRemove.add(file);
+            succeed++;
+          }
+          else if (code == HttpStatus.SC_BAD_REQUEST) {
+            toRemove.add(file);
+          }
+
+          if (LOG.isTraceEnabled()) {
+            LOG.trace(getResponseMessage(response));
+          }
+        }
+        else {
+          if (LOG.isTraceEnabled()) {
+            LOG.trace("File is empty or has invalid format: " + file.getName());
+          }
+          toRemove.add(file);
+        }
+      }
+
+      cleanupSentFiles(toRemove);
+
+      if (logs.isEmpty()) {
+        return new StatisticsResult(ResultCode.NOTHING_TO_SEND, "No files to upload.");
+      }
+      else if (succeed != logs.size()) {
+        return new StatisticsResult(ResultCode.SENT_WITH_ERRORS, "Uploaded " + succeed + " out of " + logs.size() + " files.");
+      }
+      return new StatisticsResult(ResultCode.SEND, "Uploaded " + succeed + " files.");
+    }
+    catch (Exception e) {
+      LOG.info(e);
+      throw new StatServiceException("Error during data sending.", e);
+    }
+  }
+
+  public void cleanupSentFiles(@NotNull List<File> toRemove) {
+    for (File file : toRemove) {
+      if (!file.delete()) {
+        LOG.warn("Failed deleting event log: " + file.getName());
+      }
+
+      if (LOG.isTraceEnabled()) {
+        LOG.trace("Removed sent log: " + file.getName());
+      }
+    }
+  }
+
+  @NotNull
+  public static HttpPost createPostRequest(@NotNull String serviceUrl, @NotNull String content) throws UnsupportedEncodingException {
+    final HttpPost post = new HttpPost(serviceUrl);
+    final StringEntity postingString = new StringEntity(content);
+    post.setEntity(postingString);
+    post.setHeader("Content-type", "application/json");
+    return post;
+  }
+
+  @Override
+  public Notification createNotification(@NotNull String groupDisplayId, @Nullable NotificationListener listener) {
+    return null;
+  }
+
+  @NotNull
+  private static String getResponseMessage(HttpResponse response) throws IOException {
+    HttpEntity entity = response.getEntity();
+    if (entity != null) {
+      return StreamUtil.readText(entity.getContent(), CharsetToolkit.UTF8);
+    }
+    return Integer.toString(response.getStatusLine().getStatusCode());
+  }
+}
@@ -0,0 +1,15 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.eventLog;
+
+import com.intellij.internal.statistic.connect.StatisticsConnectionService;
+import com.intellij.openapi.application.impl.ApplicationInfoImpl;
+
+public class EventLogStatisticsSettingsService extends StatisticsConnectionService {
+  public static EventLogStatisticsSettingsService getInstance() {
+    return new EventLogStatisticsSettingsService();
+  }
+
+  private EventLogStatisticsSettingsService() {
+    super(((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getEventLogSettingsUrl(), null);
+  }
+}
@@ -1,6 +1,8 @@
 // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.internal.statistic.eventLog;

+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.util.PathUtil;
 import org.apache.log4j.FileAppender;
 import org.apache.log4j.Layout;
 import org.apache.log4j.helpers.CountingQuietWriter;
@@ -32,6 +34,11 @@ public class FeatureUsageEventFileAppender extends FileAppender {
    return new FeatureUsageEventFileAppender(layout, dir, file.getPath());
  }

+  @NotNull
+  public String getActiveLogName() {
+    return StringUtil.isNotEmpty(fileName) ? PathUtil.getFileName(fileName) : "";
+  }
+
  public void setMaxFileSize(String value) {
    maxFileSize = OptionConverter.toFileSize(value, maxFileSize + 1);
  }
@@ -7,9 +7,11 @@ package com.intellij.internal.statistic.eventLog
 import com.intellij.openapi.application.ApplicationAdapter
 import com.intellij.openapi.application.ApplicationManager
 import com.intellij.openapi.application.PathManager
+import com.intellij.openapi.util.text.StringUtil
 import org.apache.log4j.Level
 import org.apache.log4j.Logger
 import org.apache.log4j.PatternLayout
+import java.io.File
 import java.io.IOException
 import java.nio.file.Paths
 import java.util.*
@@ -32,7 +34,7 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger {

    val pattern = PatternLayout("%m\n")
    try {
-      val dir = Paths.get(PathManager.getSystemPath()).resolve("event-log")
+      val dir = getEventLogDir()
      fileAppender = FeatureUsageEventFileAppender.create(pattern, dir)
      fileAppender?.let { appender ->
        appender.setMaxFileSize("200KB")
@@ -50,6 +52,8 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger {
    })
  }

+  private fun getEventLogDir() = Paths.get(PathManager.getSystemPath()).resolve("event-log")
+
  private fun String.shortedUUID(): String {
    val start = this.lastIndexOf('-')
    if (start > 0 && start + 1 < this.length) {
@@ -89,4 +93,10 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger {
    lastEvent = null
    count = 1
  }
+
+  override fun getLogFiles() : List<File> {
+    val activeLog = fileAppender?.activeLogName
+    val files = File(getEventLogDir().toUri()).listFiles({ f: File-> !StringUtil.equals(f.name, activeLog) })
+    return files?.toList() ?: emptyList()
+  }
 }
@@ -2,6 +2,7 @@
 package com.intellij.internal.statistic.eventLog

 import com.intellij.openapi.util.registry.Registry
+import java.io.File

 object FeatureUsageLogger {
  private val ourLogger = if (isEnabled()) FeatureUsageFileEventLogger() else FeatureUsageEmptyEventLogger()
@@ -10,6 +11,10 @@ object FeatureUsageLogger {
    return ourLogger.log(recorderId, action)
  }

+  fun getLogFiles() : List<File> {
+    return ourLogger.getLogFiles()
+  }
+
  fun isEnabled() : Boolean {
    return Registry.`is`("feature.usage.event.log.collect.and.upload")
  }
@@ -17,9 +22,15 @@ object FeatureUsageLogger {

 interface FeatureUsageEventLogger {
  fun log(recorderId: String, action: String)
+
+  fun getLogFiles(): List<File>
 }

 class FeatureUsageEmptyEventLogger : FeatureUsageEventLogger {
  override fun log(recorderId: String, action: String) {
  }
+
+  override fun getLogFiles() : List<File> {
+    return emptyList()
+  }
 }
@@ -1,15 +1,15 @@
 // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.internal.statistic.eventLog

+import com.google.gson.JsonSyntaxException
 import com.intellij.openapi.application.ApplicationInfo
 import com.intellij.openapi.application.PermanentInstallationID
 import com.intellij.openapi.diagnostic.Logger
-
 import java.io.BufferedReader
 import java.io.File
 import java.io.FileReader
 import java.io.IOException
-import java.util.ArrayList
+import java.util.*

 class LogEventContent(val events: List<LogEvent>) {
  val product = ApplicationInfo.getInstance().build.productCode
@@ -33,6 +33,9 @@ class LogEventContent(val events: List<LogEvent>) {
          return LogEventContent(events)
        }
      }
+      catch (e: JsonSyntaxException) {
+        LOG.warn(e)
+      }
      catch (e: IOException) {
        LOG.warn(e)
      }
@@ -17,12 +17,12 @@ package com.intellij.internal.statistic.utils;

 import com.intellij.featureStatistics.FeatureUsageTracker;
 import com.intellij.featureStatistics.FeatureUsageTrackerImpl;
-import com.intellij.internal.statistic.service.old.OldConfigurableStatisticsService;
 import com.intellij.internal.statistic.connect.StatisticsService;
+import com.intellij.internal.statistic.eventLog.EventLogStatisticsService;
 import com.intellij.internal.statistic.persistence.SentUsagesPersistence;
 import com.intellij.internal.statistic.persistence.UsageStatisticsPersistenceComponent;
 import com.intellij.internal.statistic.service.fus.FUStatisticsService;
-import com.intellij.openapi.util.KeyedExtensionCollector;
+import com.intellij.internal.statistic.service.old.OldConfigurableStatisticsService;
 import com.intellij.util.Time;

 public class StatisticsUploadAssistant {
@@ -66,4 +66,7 @@ public class StatisticsUploadAssistant {
    return new FUStatisticsService();
  }

+  public static StatisticsService getEventLogStatisticsService() {
+    return new EventLogStatisticsService();
+  }
 }
@@ -827,6 +827,7 @@

      <action internal="true" id="SendStatistics" class="com.intellij.internal.statistic.actions.SendStatisticsAction" text="OLD: Collect And Send Statistics"/>
      <action internal="true" id="SendFUStatistics" class="com.intellij.internal.statistic.actions.SendFUStatisticsAction" text="NEW: Collect And Send Statistics"/>
+      <action internal="true" id="SendEventLogStatistics" class="com.intellij.internal.statistic.actions.SendEventLogAction" text="Send Feature Usage Event Log"/>
      <action internal="true" id="PersistFUStatisticsSessionAction" class="com.intellij.internal.statistic.actions.PersistFUStatisticsSessionAction" text="Persist statistics session (for QA)"/>

      <add-to-group group-id="ToolsMenu" anchor="last"/>