diff --git a/platform/platform-tests/testSrc/com/intellij/internal/statistics/uploader/EventLogResultParserTest.kt b/platform/platform-tests/testSrc/com/intellij/internal/statistics/uploader/EventLogResultParserTest.kt new file mode 100644 index 000000000000..34d9298141fe --- /dev/null +++ b/platform/platform-tests/testSrc/com/intellij/internal/statistics/uploader/EventLogResultParserTest.kt @@ -0,0 +1,163 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistics.uploader + +import com.intellij.internal.statistic.uploader.events.* +import com.intellij.openapi.util.io.FileUtil +import com.intellij.testFramework.UsefulTestCase +import junit.framework.TestCase +import java.io.File + +class EventLogResultParserTest : UsefulTestCase() { + + fun test_parse_start_event() { + assertEquals(ExternalUploadEventSerializer.deserialize("1583419425124 STARTED"), ExternalUploadStartedEvent(1583419425124)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583317425114 STARTED"), ExternalUploadStartedEvent(1583317425114)) + } + + fun test_parse_send_event() { + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2 1 10"), ExternalUploadSendEvent(1583419435214, 2, 1, 10)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 6 0 15"), ExternalUploadSendEvent(1583419435214, 6, 0, 15)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2 a 5"), ExternalUploadSendEvent(1583419435214, 2, -1, 5)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 SEND a b b"), ExternalUploadSendEvent(1583419435214, -1, -1, -1)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 12a 34 50"), ExternalUploadSendEvent(1583419435214, -1, 34, 50)) + } + + fun test_parse_finished_event() { + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 FINISHED no-arguments"), ExternalUploadFinishedEvent(1583419435214, "no-arguments")) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 FINISHED no-files"), ExternalUploadFinishedEvent(1583419435214, "no-files")) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 FINISHED"), ExternalUploadFinishedEvent(1583419435214, null)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 FINISHED "), ExternalUploadFinishedEvent(1583419435214, null)) + assertEquals(ExternalUploadEventSerializer.deserialize("1583419435214 FINISHED multi word message"), ExternalUploadFinishedEvent(1583419435214, "multi")) + } + + fun test_parse_finished_event_failed() { + assertNull(ExternalUploadEventSerializer.deserialize("FINISHED")) + assertNull(ExternalUploadEventSerializer.deserialize("FINISHED multi word message")) + assertNull(ExternalUploadEventSerializer.deserialize("FINISHED 1583419435214 message")) + assertNull(ExternalUploadEventSerializer.deserialize("FINISHED message")) + } + + fun test_parse_send_event_failed() { + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2 3")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2 3 4 5")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND 2,3,5")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEND,2,3,5")) + assertNull(ExternalUploadEventSerializer.deserialize("SEND 2 3 4 5")) + assertNull(ExternalUploadEventSerializer.deserialize("SEND 2 3 4")) + } + + fun test_parse_unknown_event_failed() { + assertNull(ExternalUploadEventSerializer.deserialize("SEN")) + assertNull(ExternalUploadEventSerializer.deserialize("STARTEDD")) + assertNull(ExternalUploadEventSerializer.deserialize("FINISH")) + assertNull(ExternalUploadEventSerializer.deserialize("FAIL")) + assertNull(ExternalUploadEventSerializer.deserialize("UNKNOWN")) + assertNull(ExternalUploadEventSerializer.deserialize("UNKNOWN with-message")) + assertNull(ExternalUploadEventSerializer.deserialize("UNKNOWN 1 23 4")) + + assertNull(ExternalUploadEventSerializer.deserialize("STARTED")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 SEN")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 STARTEDD")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 FINISH")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 FAIL")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 UNKNOWN")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 UNKNOWN with-message")) + assertNull(ExternalUploadEventSerializer.deserialize("1583419435214 UNKNOWN 1 23 4")) + } + + fun test_serialize_start_event() { + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadStartedEvent(1583419435214)), "1583419435214 STARTED") + } + + fun test_serialize_send_event() { + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadSendEvent(1583419435214, 1, 2, 3)), "1583419435214 SEND 1 2 3") + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadSendEvent(1583419435214, 5, -1, 12)), "1583419435214 SEND 5 -1 12") + } + + fun test_serialize_failed_event() { + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadFinishedEvent(1583419435214, "error-on-send")), "1583419435214 FINISHED error-on-send") + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadFinishedEvent(1583419435214, "message")), "1583419435214 FINISHED message") + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadFinishedEvent(1583419435214, "multi word message")), "1583419435214 FINISHED multi_word_message") + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadFinishedEvent(1583419435214, "")), "1583419435214 FINISHED") + TestCase.assertEquals(ExternalUploadEventSerializer.serialize(ExternalUploadFinishedEvent(1583419435214, null)), "1583419435214 FINISHED") + } + + private fun doTestParseFile(fileText: String, vararg expectedEvents: ExternalUploadEvent) { + val directory = FileUtil.createTempDirectory("idea-external-send-result", null) + try { + FileUtil.writeToFile(File(directory, "idea_statistics_uploader_events.log"), fileText) + val actual = ExternalEventsLogger.parseEvents(directory) + assertEquals(actual.size, expectedEvents.size) + for ((index, expectedEvent) in expectedEvents.withIndex()) { + assertEquals(actual[index], expectedEvent) + } + } + finally { + directory.deleteRecursively() + } + } + + fun test_parse_external_send_started_result_file() { + doTestParseFile( + "1583419435214 STARTED", + ExternalUploadStartedEvent(1583419435214) + ) + } + + fun test_parse_external_send_succeed_result_file() { + doTestParseFile( + "1583419435214 STARTED\n1583419435234 SEND 1 2 3", + ExternalUploadStartedEvent(1583419435214), ExternalUploadSendEvent(1583419435234, 1, 2, 3) + ) + } + + fun test_parse_external_send_failed_result_file() { + doTestParseFile( + "1583419435214 STARTED\n1583419435218 FINISHED error-on-send", + ExternalUploadStartedEvent(1583419435214), ExternalUploadFinishedEvent(1583419435218, "error-on-send") + ) + } + + fun test_parse_external_send_with_ending_lineseparator() { + doTestParseFile( + "1583419435218 STARTED\n", + ExternalUploadStartedEvent(1583419435218) + ) + } + + fun test_parse_external_send_with_multiple_lineseparators() { + doTestParseFile( + "1583419435218 STARTED\n\n\n", + ExternalUploadStartedEvent(1583419435218) + ) + } + + fun test_parse_external_send_with_multiple_events_including_invalid() { + doTestParseFile( + "1583419435218 STARTED\n1583419435219 UNKNOWN\n1583419435220 FINISHED error", + ExternalUploadStartedEvent(1583419435218), ExternalUploadFinishedEvent(1583419435220, "error") + ) + } + + fun test_parse_external_send_with_invalid_event() { + doTestParseFile("UNKNOWN") + } + + fun test_parse_external_send_started_without_timestamp() { + doTestParseFile( + "STARTED" + ) + } + + fun test_parse_external_send_multiple_events_without_timestamp() { + doTestParseFile( + "STARTED\nSEND 1 2 3" + ) + } + + fun test_parse_external_send_with_empty_file() { + doTestParseFile("") + } +} \ No newline at end of file diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/EventLogSystemLogger.java b/platform/statistics/src/com/intellij/internal/statistic/eventLog/EventLogSystemLogger.java index 8d8af9a13914..f07ae012d3dc 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/EventLogSystemLogger.java +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/EventLogSystemLogger.java @@ -2,6 +2,7 @@ package com.intellij.internal.statistic.eventLog; import com.intellij.internal.statistic.eventLog.uploader.EventLogUploadException.EventLogUploadErrorType; +import com.intellij.openapi.util.text.StringUtil; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -18,14 +19,29 @@ public class EventLogSystemLogger { logEvent(recorderId, "whitelist.updated", data); } - public static void logFilesSend(@NotNull String recorderId, int total, int succeed, int failed) { + public static void logFilesSend(@NotNull String recorderId, int total, int succeed, int failed, boolean external) { final FeatureUsageData data = new FeatureUsageData(). addData("total", total). addData("send", succeed + failed). - addData("failed", failed); + addData("failed", failed). + addData("external", external); logEvent(recorderId, "logs.send", data); } + public static void logStartingExternalSend(@NotNull String recorderId, long time) { + FeatureUsageData data = new FeatureUsageData().addData("send_ts", time); + logEvent(recorderId, "external.send.started", data); + } + + public static void logFinishedExternalSend(@NotNull String recorderId, @Nullable String error, long time) { + boolean succeed = StringUtil.isEmpty(error); + FeatureUsageData data = new FeatureUsageData().addData("succeed", succeed).addData("send_ts", time); + if (!succeed) { + data.addData("error", error); + } + logEvent(recorderId, "external.send.finished", data); + } + public static void logCreatingExternalSendCommand(@NotNull String recorderId) { logEvent(recorderId, "external.send.command.creation.started"); } diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/fus/FeatureUsageEventLoggerProvider.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/fus/FeatureUsageEventLoggerProvider.kt index 1db1ec5c5071..de76a8985cd5 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/fus/FeatureUsageEventLoggerProvider.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/fus/FeatureUsageEventLoggerProvider.kt @@ -6,7 +6,7 @@ import com.intellij.internal.statistic.utils.StatisticsUploadAssistant import com.intellij.openapi.application.ApplicationManager import java.util.concurrent.TimeUnit -internal class FeatureUsageEventLoggerProvider : StatisticsEventLoggerProvider("FUS", 35, sendFrequencyMs = TimeUnit.MINUTES.toMillis(15)) { +internal class FeatureUsageEventLoggerProvider : StatisticsEventLoggerProvider("FUS", 36, sendFrequencyMs = TimeUnit.MINUTES.toMillis(15)) { override fun isRecordEnabled(): Boolean { return !ApplicationManager.getApplication().isHeadlessEnvironment && StatisticsUploadAssistant.isCollectAllowed() diff --git a/platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogExternalUploader.kt b/platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogExternalUploader.kt index ff3547c34b61..c2903f9e6b32 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogExternalUploader.kt +++ b/platform/statistics/src/com/intellij/internal/statistic/eventLog/uploader/EventLogExternalUploader.kt @@ -4,18 +4,51 @@ package com.intellij.internal.statistic.eventLog.uploader import com.intellij.internal.statistic.eventLog.* import com.intellij.internal.statistic.eventLog.uploader.EventLogUploadException.EventLogUploadErrorType.* import com.intellij.internal.statistic.uploader.EventLogUploaderOptions.* +import com.intellij.internal.statistic.uploader.events.* import com.intellij.openapi.application.PathManager import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.util.SystemInfo import com.intellij.util.ArrayUtil import com.intellij.util.io.exists import java.io.File +import java.lang.Exception import java.nio.file.Paths object EventLogExternalUploader { private val LOG = Logger.getInstance(EventLogExternalUploader.javaClass) private const val UPLOADER_MAIN_CLASS = "com.intellij.internal.statistic.uploader.EventLogUploader" + fun logPreviousExternalUploadResult(recorderId: String) { + val recorder = EventLogInternalRecorderConfig(recorderId) + if (!recorder.isSendEnabled()) { + return + } + + try { + val tempDir = getTempFile() + if (tempDir.exists()) { + val events = ExternalEventsLogger.parseEvents(tempDir) + for (event in events) { + when (event) { + is ExternalUploadStartedEvent -> { + EventLogSystemLogger.logStartingExternalSend(recorderId, event.timestamp) + } + is ExternalUploadSendEvent -> { + EventLogSystemLogger.logFilesSend(recorderId, event.total, event.succeed, event.failed, true) + } + is ExternalUploadFinishedEvent -> { + EventLogSystemLogger.logFinishedExternalSend(recorderId, event.error, event.timestamp) + } + } + } + } + tempDir.deleteRecursively() + } + catch (e: Exception) { + LOG.warn("Failed reading previous upload result: " + e.message) + } + } + fun startExternalUpload(recorderId: String, isTest: Boolean) { val recorder = EventLogInternalRecorderConfig(recorderId) if (!recorder.isSendEnabled()) { @@ -46,7 +79,7 @@ object EventLogExternalUploader { throw EventLogUploadException("No available logs to send", NO_LOGS) } - val tempDir = getTempDir() + val tempDir = getOrCreateTempDir() val uploader = findUploader() val libs = findLibsByPrefixes( "kotlin-stdlib", "gson", "commons-logging", "log4j.jar", "httpclient", "httpcore", "httpmime", "jdom.jar", "annotations.jar" @@ -138,11 +171,15 @@ object EventLogExternalUploader { return false } - private fun getTempDir(): File { - val tempDir = File(PathManager.getTempPath(), "statistics-uploader") + private fun getOrCreateTempDir(): File { + val tempDir = getTempFile() if (!(tempDir.exists() || tempDir.mkdirs())) { throw EventLogUploadException("Cannot create temp directory: $tempDir", NO_TEMP_FOLDER) } return tempDir } + + private fun getTempFile(): File { + return File(PathManager.getTempPath(), "statistics-uploader") + } } diff --git a/platform/statistics/src/com/intellij/internal/statistic/updater/StatisticsJobsScheduler.java b/platform/statistics/src/com/intellij/internal/statistic/updater/StatisticsJobsScheduler.java index f70c19f2adba..6ecb525015da 100644 --- a/platform/statistics/src/com/intellij/internal/statistic/updater/StatisticsJobsScheduler.java +++ b/platform/statistics/src/com/intellij/internal/statistic/updater/StatisticsJobsScheduler.java @@ -7,6 +7,8 @@ import com.intellij.ide.StatisticsNotificationManager; import com.intellij.internal.statistic.connect.StatisticsService; import com.intellij.internal.statistic.eventLog.StatisticsEventLoggerKt; import com.intellij.internal.statistic.eventLog.StatisticsEventLoggerProvider; +import com.intellij.internal.statistic.eventLog.fus.FeatureUsageLogger; +import com.intellij.internal.statistic.eventLog.uploader.EventLogExternalUploader; import com.intellij.internal.statistic.eventLog.validator.SensitiveDataValidator; import com.intellij.internal.statistic.service.fus.collectors.FUStateUsagesLogger; import com.intellij.internal.statistic.service.fus.collectors.FUStatisticsPersistence; @@ -33,6 +35,7 @@ import java.util.concurrent.TimeUnit; final class StatisticsJobsScheduler implements ApplicationInitializedListener { private static final int SEND_STATISTICS_INITIAL_DELAY_IN_MILLIS = 5 * 60 * 1000; private static final int CHECK_STATISTICS_PROVIDERS_DELAY_IN_MIN = 1; + private static final int CHECK_EXTERNAL_UPLOADER_DELAY_IN_MIN = 3; public static final int LOG_APPLICATION_STATES_INITIAL_DELAY_IN_MIN = 15; public static final int LOG_APPLICATION_STATES_DELAY_IN_MIN = 24 * 60; @@ -54,6 +57,7 @@ final class StatisticsJobsScheduler implements ApplicationInitializedListener { notificationManager.showNotificationIfNeeded(); } + checkPreviousExternalUploadResult(); runEventLogStatisticsService(); runStatesLogging(); runLegacyDataCleanupService(); @@ -72,6 +76,16 @@ final class StatisticsJobsScheduler implements ApplicationInitializedListener { }, 3, 180, TimeUnit.MINUTES); } + private static void checkPreviousExternalUploadResult() { + JobScheduler.getScheduler().schedule(() -> { + StatisticsEventLoggerProvider config = FeatureUsageLogger.INSTANCE.getConfig(); + if (config.isRecordEnabled()) { + EventLogExternalUploader.INSTANCE.logPreviousExternalUploadResult(config.getRecorderId()); + } + }, CHECK_EXTERNAL_UPLOADER_DELAY_IN_MIN, TimeUnit.MINUTES); + } + + private static void runEventLogStatisticsService() { JobScheduler.getScheduler().schedule(() -> { final List providers = StatisticsEventLoggerKt.getEventLogProviders(); diff --git a/platform/statistics/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java b/platform/statistics/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java index 507f154872ab..02646ff1c47a 100755 --- a/platform/statistics/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java +++ b/platform/statistics/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java @@ -57,7 +57,7 @@ public class StatisticsUploadAssistant { EventLogSendListener listener = new EventLogSendListener() { @Override public void onLogsSend(int succeed, int failed, int totalLocalFiles) { - EventLogSystemLogger.logFilesSend(recorderId, totalLocalFiles, succeed, failed); + EventLogSystemLogger.logFilesSend(recorderId, totalLocalFiles, succeed, failed, false); } }; diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploader.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploader.java index 708ee24491ad..ee90286a4846 100644 --- a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploader.java +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/EventLogUploader.java @@ -6,6 +6,7 @@ import com.intellij.internal.statistic.connect.StatisticsResult; import com.intellij.internal.statistic.eventLog.*; import com.intellij.internal.statistic.eventLog.config.EventLogExternalApplicationInfo; import com.intellij.internal.statistic.eventLog.config.EventLogExternalRecorderConfig; +import com.intellij.internal.statistic.uploader.events.ExternalEventsLogger; import org.jetbrains.annotations.Nullable; import java.io.File; @@ -21,11 +22,14 @@ public class EventLogUploader { } private static void execute(String[] args) { + ExternalEventsLogger eventsLogger = new ExternalEventsLogger(); DataCollectorDebugLogger logger = new ExternalDataCollectorLogger(); logger.info("Process started with '" + String.join(" ", args) + "'"); + eventsLogger.logSendingLogsStarted(); if (args.length == 0) { logger.warn("No arguments were found"); + eventsLogger.logSendingLogsFinished("NO_ARGUMENTS"); return; } @@ -33,23 +37,27 @@ public class EventLogUploader { DeviceConfiguration device = newDeviceConfig(options); if (device == null) { logger.warn("Failed creating device config from arguments"); + eventsLogger.logSendingLogsFinished("NO_DEVICE_CONFIG"); return; } EventLogRecorderConfig recorder = newRecorderConfig(options); if (recorder == null) { logger.warn("Failed creating recorder config from arguments"); + eventsLogger.logSendingLogsFinished("NO_RECORDER_CONFIG"); return; } EventLogApplicationInfo appInfo = newApplicationInfo(options, logger); if (appInfo == null) { logger.warn("Failed creating application info from arguments"); + eventsLogger.logSendingLogsFinished("NO_APPLICATION_CONFIG"); return; } if (!waitForIde(logger, options, 20)) { logger.warn("Cannot send logs because IDE didn't close during " + (10 * WAIT_FOR_IDE_MS) + "ms"); + eventsLogger.logSendingLogsFinished("IDE_NOT_CLOSING"); return; } @@ -62,9 +70,15 @@ public class EventLogUploader { logger.info("{recorder:" + recorder.getRecorderId() + ", files:" + logs + "}"); logger.info("{device:" + device.getDeviceId() + ", bucket:" + device.getBucket() + "}"); try { - //TODO: save the number of uploaded files and log it during the next IDE session - EventLogStatisticsService service = new EventLogStatisticsService(device, recorder, appInfo, null); + EventLogStatisticsService service = new EventLogStatisticsService(device, recorder, appInfo, new EventLogSendListener() { + @Override + public void onLogsSend(int succeed, int failed, int totalLocalFiles) { + eventsLogger.logSendingLogsSucceed(succeed, failed, totalLocalFiles); + } + }); + StatisticsResult result = service.send(); + eventsLogger.logSendingLogsFinished(result.getCode()); if (logger.isTraceEnabled()) { logger.trace("Uploading finished with " + result.getCode().name()); logger.trace(result.getDescription()); @@ -72,6 +86,7 @@ public class EventLogUploader { } catch (Exception e) { logger.warn("Failed sending files: " + e.getMessage()); + eventsLogger.logSendingLogsFinished("ERROR_ON_SEND"); } } diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalEventsLogger.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalEventsLogger.java new file mode 100644 index 000000000000..d562c778c24d --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalEventsLogger.java @@ -0,0 +1,86 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import com.intellij.internal.statistic.connect.StatisticsResult; +import org.apache.log4j.FileAppender; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.jetbrains.annotations.NonNls; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.intellij.internal.statistic.uploader.ExternalDataCollectorLogger.findDirectory; + +public class ExternalEventsLogger { + @NonNls private final Logger myLogger; + + public ExternalEventsLogger() { + myLogger = Logger.getLogger("com.intellij.internal.statistic.uploader.events"); + String logDirectory = findDirectory(1_000_000L); + if (logDirectory != null) { + myLogger.addAppender(newAppender(getEventLogFile(logDirectory).getAbsolutePath())); + myLogger.setLevel(Level.ALL); + } + } + + @NotNull + public static FileAppender newAppender(@NotNull String directory) { + @NonNls FileAppender appender = new FileAppender(); + appender.setFile(directory); + appender.setLayout(new PatternLayout("%m\n")); + appender.setThreshold(Level.ALL); + appender.setAppend(false); + appender.activateOptions(); + return appender; + } + + @NotNull + private static File getEventLogFile(@NotNull String logDirectory) { + return new File(logDirectory, "idea_statistics_uploader_events.log"); + } + + public void logSendingLogsStarted() { + logEvent(new ExternalUploadStartedEvent(System.currentTimeMillis())); + } + + public void logSendingLogsFinished(@NotNull String error) { + logEvent(new ExternalUploadFinishedEvent(System.currentTimeMillis(), error)); + } + + public void logSendingLogsFinished(@NotNull StatisticsResult.ResultCode code) { + String error = code == StatisticsResult.ResultCode.SEND ? null : code.name(); + logEvent(new ExternalUploadFinishedEvent(System.currentTimeMillis(), error)); + } + + public void logSendingLogsSucceed(int succeed, int failed, int total) { + logEvent(new ExternalUploadSendEvent(System.currentTimeMillis(), succeed, failed, total)); + } + + private void logEvent(@NotNull ExternalUploadEvent event) { + myLogger.info(ExternalUploadEventSerializer.serialize(event)); + } + + @NotNull + public static List parseEvents(@NotNull File directory) throws IOException { + File file = getEventLogFile(directory.getAbsolutePath()); + List lines = file.exists() ? Files.readAllLines(file.toPath()) : Collections.emptyList(); + if (!lines.isEmpty()) { + List events = new ArrayList<>(); + for (String line : lines) { + ExternalUploadEvent event = ExternalUploadEventSerializer.deserialize(line); + if (event != null) { + events.add(event); + } + } + return events; + } + return Collections.emptyList(); + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEvent.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEvent.java new file mode 100644 index 000000000000..509f5a9aff80 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEvent.java @@ -0,0 +1,38 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import org.jetbrains.annotations.NotNull; + +import java.util.Objects; + +public abstract class ExternalUploadEvent { + private final long myTimestamp; + private final ExternalUploadEventType myEventType; + + public ExternalUploadEvent(@NotNull ExternalUploadEventType eventType, long timestamp) { + myTimestamp = timestamp; + myEventType = eventType; + } + + public long getTimestamp() { + return myTimestamp; + } + + public ExternalUploadEventType getEventType() { + return myEventType; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExternalUploadEvent event = (ExternalUploadEvent)o; + return myTimestamp == event.myTimestamp && + myEventType == event.myEventType; + } + + @Override + public int hashCode() { + return Objects.hash(myTimestamp, myEventType); + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventSerializer.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventSerializer.java new file mode 100644 index 000000000000..751ec95ee801 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventSerializer.java @@ -0,0 +1,71 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import com.intellij.internal.statistic.StatisticsEventLogUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class ExternalUploadEventSerializer { + + @NotNull + public static String serialize(@NotNull ExternalUploadEvent event) { + String prefix = event.getTimestamp() + " " + event.getEventType().name(); + if (event instanceof ExternalUploadFinishedEvent) { + ExternalUploadFinishedEvent failed = (ExternalUploadFinishedEvent)event; + String error = failed.getError(); + if (StatisticsEventLogUtil.isNotEmpty(error)) { + return prefix + " " + error.replace(" ", "_"); + } + return prefix; + } + else if (event instanceof ExternalUploadSendEvent) { + ExternalUploadSendEvent finished = (ExternalUploadSendEvent)event; + return prefix + " " + finished.getSucceed() + " " + finished.getFailed() + " " + finished.getTotal(); + } + return prefix; + } + + @Nullable + public static ExternalUploadEvent deserialize(@NotNull String line) { + String[] parts = line.split(" "); + int length = parts.length; + ExternalUploadEventType type = length > 1 ? ExternalUploadEventType.parse(parts[1]) : null; + if (type == null) { + return null; + } + + long timestamp = parseLong(parts[0]); + if (type == ExternalUploadEventType.FINISHED) { + String error = parts.length >= 3 ? parts[2].trim() : null; + return new ExternalUploadFinishedEvent(timestamp, error); + } + else if (type == ExternalUploadEventType.SEND && length == 5) { + int succeed = parseInt(parts[2]); + int failed = parseInt(parts[3]); + int total = parseInt(parts[4]); + return new ExternalUploadSendEvent(timestamp, succeed, failed, total); + } + else if (type == ExternalUploadEventType.STARTED && length == 2) { + return new ExternalUploadStartedEvent(timestamp); + } + return null; + } + + private static int parseInt(String value) { + try { + return Integer.parseInt(value); + } + catch (NumberFormatException e) { + return -1; + } + } + + private static long parseLong(String value) { + try { + return Long.parseLong(value); + } + catch (NumberFormatException e) { + return -1; + } + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventType.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventType.java new file mode 100644 index 000000000000..f1bf0bab5de8 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadEventType.java @@ -0,0 +1,20 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import com.intellij.internal.statistic.StatisticsEventLogUtil; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public enum ExternalUploadEventType { + STARTED, FINISHED, SEND; + + @Nullable + static ExternalUploadEventType parse(@NotNull String event) { + for (ExternalUploadEventType type : values()) { + if (StatisticsEventLogUtil.equals(type.name(), event)) { + return type; + } + } + return null; + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadFinishedEvent.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadFinishedEvent.java new file mode 100644 index 000000000000..65327a6b9695 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadFinishedEvent.java @@ -0,0 +1,34 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import org.jetbrains.annotations.Nullable; + +import java.util.Objects; + +public class ExternalUploadFinishedEvent extends ExternalUploadEvent { + private final String myError; + + public ExternalUploadFinishedEvent(long timestamp, @Nullable String error) { + super(ExternalUploadEventType.FINISHED, timestamp); + myError = error; + } + + @Nullable + public String getError() { + return myError; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + ExternalUploadFinishedEvent event = (ExternalUploadFinishedEvent)o; + return Objects.equals(myError, event.myError); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), myError); + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadSendEvent.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadSendEvent.java new file mode 100644 index 000000000000..96662e385871 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadSendEvent.java @@ -0,0 +1,45 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +import java.util.Objects; + +public class ExternalUploadSendEvent extends ExternalUploadEvent { + private final int mySucceed; + private final int myFailed; + private final int myTotal; + + public ExternalUploadSendEvent(long timestamp, int succeed, int failed, int total) { + super(ExternalUploadEventType.SEND, timestamp); + mySucceed = succeed; + myFailed = failed; + myTotal = total; + } + + public int getSucceed() { + return mySucceed; + } + + public int getFailed() { + return myFailed; + } + + public int getTotal() { + return myTotal; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + ExternalUploadSendEvent event = (ExternalUploadSendEvent)o; + return mySucceed == event.mySucceed && + myFailed == event.myFailed && + myTotal == event.myTotal; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), mySucceed, myFailed, myTotal); + } +} diff --git a/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadStartedEvent.java b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadStartedEvent.java new file mode 100644 index 000000000000..20799e53fcd7 --- /dev/null +++ b/platform/statistics/uploader/src/com/intellij/internal/statistic/uploader/events/ExternalUploadStartedEvent.java @@ -0,0 +1,9 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal.statistic.uploader.events; + +public class ExternalUploadStartedEvent extends ExternalUploadEvent { + + public ExternalUploadStartedEvent(long timestamp) { + super(ExternalUploadEventType.STARTED, timestamp); + } +}