From 57a1ca5bd9f9c67689870bdb24129d32304995a2 Mon Sep 17 00:00:00 2001 From: "Svetlana.Zemlyanskaya" Date: Wed, 14 Feb 2018 16:38:38 +0100 Subject: [PATCH] Feature usage log: upload event log to statistic server --- .../application/impl/ApplicationInfoImpl.java | 8 ++ .../statistic/actions/SendEventLogAction.java | 41 ++++++ .../eventLog/EventLogStatisticsService.java | 131 ++++++++++++++++++ .../EventLogStatisticsSettingsService.java | 15 ++ .../FeatureUsageEventFileAppender.java | 7 + .../eventLog/FeatureUsageFileEventLogger.kt | 12 +- .../statistic/eventLog/FeatureUsageLogger.kt | 11 ++ .../statistic/eventLog/LogEventContent.kt | 7 +- .../utils/StatisticsUploadAssistant.java | 7 +- .../src/idea/PlatformActions.xml | 1 + 10 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 platform/platform-impl/src/com/intellij/internal/statistic/actions/SendEventLogAction.java create mode 100644 platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsService.java create mode 100644 platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsSettingsService.java diff --git a/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java b/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java index b54678ebbd01..2463ddf03cf7 100644 --- a/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java +++ b/platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java @@ -99,6 +99,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx { private String myFUStatisticsSettingsUrl; private String myStatisticsServiceUrl; private String myStatisticsServiceKey; + private String myEventLogSettingsUrl; private String myThirdPartySoftwareUrl; private String myJetbrainsTvUrl; private String myEvalLicenseUrl = "https://www.jetbrains.com/store/license.html"; @@ -184,6 +185,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx { private static final String ATTRIBUTE_FU_STATISTICS_SETTINGS = "fus-settings"; private static final String ATTRIBUTE_STATISTICS_SERVICE = "service"; private static final String ATTRIBUTE_STATISTICS_SERVICE_KEY = "service-key"; + private static final String ATTRIBUTE_EVENT_LOG_STATISTICS_SETTINGS = "event-log-settings"; private static final String ELEMENT_THIRD_PARTY = "third-party"; private static final String ELEMENT_JB_TV = "jetbrains-tv"; private static final String CUSTOMIZE_IDE_WIZARD_STEPS = "customize-ide-wizard"; @@ -545,6 +547,10 @@ public class ApplicationInfoImpl extends ApplicationInfoEx { return myStatisticsServiceKey; } + public String getEventLogSettingsUrl() { + return myEventLogSettingsUrl; + } + @Override public String getThirdPartySoftwareURL() { return myThirdPartySoftwareUrl; @@ -880,12 +886,14 @@ public class ApplicationInfoImpl extends ApplicationInfoEx { myFUStatisticsSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_FU_STATISTICS_SETTINGS); myStatisticsServiceUrl = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE); myStatisticsServiceKey = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE_KEY); + myEventLogSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_EVENT_LOG_STATISTICS_SETTINGS); } else { myStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/stat-assistant.xml"; myFUStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/fus-assistant.xml"; myStatisticsServiceUrl = "https://www.jetbrains.com/idea/statistics/index.jsp"; myStatisticsServiceKey = null; + myEventLogSettingsUrl = "https://www.jetbrains.com/idea/statistics/fus-lion-assistant.xml"; } Element thirdPartyElement = getChild(parentNode, ELEMENT_THIRD_PARTY); diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/actions/SendEventLogAction.java b/platform/platform-impl/src/com/intellij/internal/statistic/actions/SendEventLogAction.java new file mode 100644 index 000000000000..2d7499c365d4 --- /dev/null +++ b/platform/platform-impl/src/com/intellij/internal/statistic/actions/SendEventLogAction.java @@ -0,0 +1,41 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.actions; + +import com.intellij.internal.statistic.connect.StatisticsResult; +import com.intellij.internal.statistic.connect.StatisticsService; +import com.intellij.internal.statistic.utils.StatisticsUploadAssistant; +import com.intellij.openapi.actionSystem.AnAction; +import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.application.ApplicationManager; +import com.intellij.openapi.application.ModalityState; +import com.intellij.openapi.progress.ProgressIndicator; +import com.intellij.openapi.progress.ProgressManager; +import com.intellij.openapi.progress.Task; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.ui.Messages; +import com.intellij.openapi.util.text.StringUtil; +import org.jetbrains.annotations.NotNull; + +public class SendEventLogAction extends AnAction { + + @Override + public void actionPerformed(@NotNull AnActionEvent e) { + final Project project = e.getProject(); + if (project == null) { + return; + } + + ProgressManager.getInstance().run(new Task.Backgroundable(project, "Send Feature Usage Event Log", false) { + @Override + public void run(@NotNull ProgressIndicator indicator) { + StatisticsService service = StatisticsUploadAssistant.getEventLogStatisticsService(); + final StatisticsResult result = service.send(); + + ApplicationManager.getApplication().invokeLater( + () -> Messages.showMultilineInputDialog(project, "Result: " + result.getCode(), "Statistics Result", + StringUtil.replace(result.getDescription(), ";", "\n"), + null, null), ModalityState.NON_MODAL, project.getDisposed()); + } + }); + } +} diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsService.java b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsService.java new file mode 100644 index 000000000000..47289d1a1c3e --- /dev/null +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsService.java @@ -0,0 +1,131 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.eventLog; + +import com.intellij.internal.statistic.connect.StatServiceException; +import com.intellij.internal.statistic.connect.StatisticsResult; +import com.intellij.internal.statistic.connect.StatisticsResult.ResultCode; +import com.intellij.internal.statistic.connect.StatisticsService; +import com.intellij.notification.Notification; +import com.intellij.notification.NotificationListener; +import com.intellij.openapi.diagnostic.Logger; +import com.intellij.openapi.util.io.StreamUtil; +import com.intellij.openapi.vfs.CharsetToolkit; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +public class EventLogStatisticsService implements StatisticsService { + private static final Logger LOG = Logger.getInstance("com.intellij.internal.statistic.eventLog.EventLogStatisticsService"); + + private static final EventLogStatisticsSettingsService mySettingsService = EventLogStatisticsSettingsService.getInstance(); + + @Override + public StatisticsResult send() { + if (!FeatureUsageLogger.INSTANCE.isEnabled()) { + throw new StatServiceException("Event Log collector is not enabled"); + } + + final String serviceUrl = mySettingsService.getServiceUrl(); + if (serviceUrl == null) { + return new StatisticsResult(StatisticsResult.ResultCode.ERROR_IN_CONFIG, "ERROR: unknown Statistics Service URL."); + } + + if (!mySettingsService.isTransmissionPermitted()) { + return new StatisticsResult(StatisticsResult.ResultCode.NOT_PERMITTED_SERVER, "NOT_PERMITTED"); + } + + try { + int succeed = 0; + final List logs = FeatureUsageLogger.INSTANCE.getLogFiles(); + final List toRemove = new ArrayList<>(logs.size()); + for (File file : logs) { + final LogEventContent session = LogEventContent.Companion.create(file); + if (session != null) { + final HttpClient httpClient = HttpClientBuilder.create().build(); + final HttpPost post = createPostRequest(serviceUrl, LogEventSerializer.INSTANCE.toString(session)); + final HttpResponse response = httpClient.execute(post); + + final int code = response.getStatusLine().getStatusCode(); + if (code == HttpStatus.SC_OK) { + toRemove.add(file); + succeed++; + } + else if (code == HttpStatus.SC_BAD_REQUEST) { + toRemove.add(file); + } + + if (LOG.isTraceEnabled()) { + LOG.trace(getResponseMessage(response)); + } + } + else { + if (LOG.isTraceEnabled()) { + LOG.trace("File is empty or has invalid format: " + file.getName()); + } + toRemove.add(file); + } + } + + cleanupSentFiles(toRemove); + + if (logs.isEmpty()) { + return new StatisticsResult(ResultCode.NOTHING_TO_SEND, "No files to upload."); + } + else if (succeed != logs.size()) { + return new StatisticsResult(ResultCode.SENT_WITH_ERRORS, "Uploaded " + succeed + " out of " + logs.size() + " files."); + } + return new StatisticsResult(ResultCode.SEND, "Uploaded " + succeed + " files."); + } + catch (Exception e) { + LOG.info(e); + throw new StatServiceException("Error during data sending.", e); + } + } + + public void cleanupSentFiles(@NotNull List toRemove) { + for (File file : toRemove) { + if (!file.delete()) { + LOG.warn("Failed deleting event log: " + file.getName()); + } + + if (LOG.isTraceEnabled()) { + LOG.trace("Removed sent log: " + file.getName()); + } + } + } + + @NotNull + public static HttpPost createPostRequest(@NotNull String serviceUrl, @NotNull String content) throws UnsupportedEncodingException { + final HttpPost post = new HttpPost(serviceUrl); + final StringEntity postingString = new StringEntity(content); + post.setEntity(postingString); + post.setHeader("Content-type", "application/json"); + return post; + } + + @Override + public Notification createNotification(@NotNull String groupDisplayId, @Nullable NotificationListener listener) { + return null; + } + + @NotNull + private static String getResponseMessage(HttpResponse response) throws IOException { + HttpEntity entity = response.getEntity(); + if (entity != null) { + return StreamUtil.readText(entity.getContent(), CharsetToolkit.UTF8); + } + return Integer.toString(response.getStatusLine().getStatusCode()); + } +} diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsSettingsService.java b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsSettingsService.java new file mode 100644 index 000000000000..ecc71b337474 --- /dev/null +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/EventLogStatisticsSettingsService.java @@ -0,0 +1,15 @@ +// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.eventLog; + +import com.intellij.internal.statistic.connect.StatisticsConnectionService; +import com.intellij.openapi.application.impl.ApplicationInfoImpl; + +public class EventLogStatisticsSettingsService extends StatisticsConnectionService { + public static EventLogStatisticsSettingsService getInstance() { + return new EventLogStatisticsSettingsService(); + } + + private EventLogStatisticsSettingsService() { + super(((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getEventLogSettingsUrl(), null); + } +} diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageEventFileAppender.java b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageEventFileAppender.java index 9ae96161cbca..78f6b7cf4240 100644 --- a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageEventFileAppender.java +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageEventFileAppender.java @@ -1,6 +1,8 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.internal.statistic.eventLog; +import com.intellij.openapi.util.text.StringUtil; +import com.intellij.util.PathUtil; import org.apache.log4j.FileAppender; import org.apache.log4j.Layout; import org.apache.log4j.helpers.CountingQuietWriter; @@ -32,6 +34,11 @@ public class FeatureUsageEventFileAppender extends FileAppender { return new FeatureUsageEventFileAppender(layout, dir, file.getPath()); } + @NotNull + public String getActiveLogName() { + return StringUtil.isNotEmpty(fileName) ? PathUtil.getFileName(fileName) : ""; + } + public void setMaxFileSize(String value) { maxFileSize = OptionConverter.toFileSize(value, maxFileSize + 1); } diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageFileEventLogger.kt b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageFileEventLogger.kt index dc17bcdef25d..f7ba6f5c0dda 100644 --- a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageFileEventLogger.kt +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageFileEventLogger.kt @@ -7,9 +7,11 @@ package com.intellij.internal.statistic.eventLog import com.intellij.openapi.application.ApplicationAdapter import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.application.PathManager +import com.intellij.openapi.util.text.StringUtil import org.apache.log4j.Level import org.apache.log4j.Logger import org.apache.log4j.PatternLayout +import java.io.File import java.io.IOException import java.nio.file.Paths import java.util.* @@ -32,7 +34,7 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger { val pattern = PatternLayout("%m\n") try { - val dir = Paths.get(PathManager.getSystemPath()).resolve("event-log") + val dir = getEventLogDir() fileAppender = FeatureUsageEventFileAppender.create(pattern, dir) fileAppender?.let { appender -> appender.setMaxFileSize("200KB") @@ -50,6 +52,8 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger { }) } + private fun getEventLogDir() = Paths.get(PathManager.getSystemPath()).resolve("event-log") + private fun String.shortedUUID(): String { val start = this.lastIndexOf('-') if (start > 0 && start + 1 < this.length) { @@ -89,4 +93,10 @@ class FeatureUsageFileEventLogger : FeatureUsageEventLogger { lastEvent = null count = 1 } + + override fun getLogFiles() : List { + val activeLog = fileAppender?.activeLogName + val files = File(getEventLogDir().toUri()).listFiles({ f: File-> !StringUtil.equals(f.name, activeLog) }) + return files?.toList() ?: emptyList() + } } \ No newline at end of file diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageLogger.kt b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageLogger.kt index ae70038f97be..c3e1daca443d 100644 --- a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageLogger.kt +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/FeatureUsageLogger.kt @@ -2,6 +2,7 @@ package com.intellij.internal.statistic.eventLog import com.intellij.openapi.util.registry.Registry +import java.io.File object FeatureUsageLogger { private val ourLogger = if (isEnabled()) FeatureUsageFileEventLogger() else FeatureUsageEmptyEventLogger() @@ -10,6 +11,10 @@ object FeatureUsageLogger { return ourLogger.log(recorderId, action) } + fun getLogFiles() : List { + return ourLogger.getLogFiles() + } + fun isEnabled() : Boolean { return Registry.`is`("feature.usage.event.log.collect.and.upload") } @@ -17,9 +22,15 @@ object FeatureUsageLogger { interface FeatureUsageEventLogger { fun log(recorderId: String, action: String) + + fun getLogFiles(): List } class FeatureUsageEmptyEventLogger : FeatureUsageEventLogger { override fun log(recorderId: String, action: String) { } + + override fun getLogFiles() : List { + return emptyList() + } } \ No newline at end of file diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/LogEventContent.kt b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/LogEventContent.kt index ea0d1f56099a..649bd2eb843e 100644 --- a/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/LogEventContent.kt +++ b/platform/platform-impl/src/com/intellij/internal/statistic/eventLog/LogEventContent.kt @@ -1,15 +1,15 @@ // Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.internal.statistic.eventLog +import com.google.gson.JsonSyntaxException import com.intellij.openapi.application.ApplicationInfo import com.intellij.openapi.application.PermanentInstallationID import com.intellij.openapi.diagnostic.Logger - import java.io.BufferedReader import java.io.File import java.io.FileReader import java.io.IOException -import java.util.ArrayList +import java.util.* class LogEventContent(val events: List) { val product = ApplicationInfo.getInstance().build.productCode @@ -33,6 +33,9 @@ class LogEventContent(val events: List) { return LogEventContent(events) } } + catch (e: JsonSyntaxException) { + LOG.warn(e) + } catch (e: IOException) { LOG.warn(e) } diff --git a/platform/platform-impl/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java b/platform/platform-impl/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java index 1c3bc52c5efb..58cf05ed0cc1 100755 --- a/platform/platform-impl/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java +++ b/platform/platform-impl/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java @@ -17,12 +17,12 @@ package com.intellij.internal.statistic.utils; import com.intellij.featureStatistics.FeatureUsageTracker; import com.intellij.featureStatistics.FeatureUsageTrackerImpl; -import com.intellij.internal.statistic.service.old.OldConfigurableStatisticsService; import com.intellij.internal.statistic.connect.StatisticsService; +import com.intellij.internal.statistic.eventLog.EventLogStatisticsService; import com.intellij.internal.statistic.persistence.SentUsagesPersistence; import com.intellij.internal.statistic.persistence.UsageStatisticsPersistenceComponent; import com.intellij.internal.statistic.service.fus.FUStatisticsService; -import com.intellij.openapi.util.KeyedExtensionCollector; +import com.intellij.internal.statistic.service.old.OldConfigurableStatisticsService; import com.intellij.util.Time; public class StatisticsUploadAssistant { @@ -66,4 +66,7 @@ public class StatisticsUploadAssistant { return new FUStatisticsService(); } + public static StatisticsService getEventLogStatisticsService() { + return new EventLogStatisticsService(); + } } diff --git a/platform/platform-resources/src/idea/PlatformActions.xml b/platform/platform-resources/src/idea/PlatformActions.xml index 6c546656cac1..a3fe763586ce 100644 --- a/platform/platform-resources/src/idea/PlatformActions.xml +++ b/platform/platform-resources/src/idea/PlatformActions.xml @@ -827,6 +827,7 @@ +