statistics 2018.1: supported old-statistics service and new feature-usages-statistics(fus) service

(cherry picked from commit ddf5607)

platform/core-impl/src/com/intellij/openapi/application/impl/ApplicationInfoImpl.java

@@ -97,6 +97,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
   private List<PluginChooserPage> myPluginChooserPages = new ArrayList<>();
   private String[] myEssentialPluginsIds;
   private String myStatisticsSettingsUrl;
+  private String myFUStatisticsSettingsUrl;
   private String myStatisticsServiceUrl;
   private String myStatisticsServiceKey;
   private String myThirdPartySoftwareUrl;
@@ -181,6 +182,7 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
   private static final String ATTRIBUTE_MAC_URL = "mac";
   private static final String ELEMENT_STATISTICS = "statistics";
   private static final String ATTRIBUTE_STATISTICS_SETTINGS = "settings";
+  private static final String ATTRIBUTE_FU_STATISTICS_SETTINGS = "fus-settings";
   private static final String ATTRIBUTE_STATISTICS_SERVICE = "service";
   private static final String ATTRIBUTE_STATISTICS_SERVICE_KEY = "service-key";
   private static final String ELEMENT_THIRD_PARTY = "third-party";
@@ -532,6 +534,10 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
     return myStatisticsSettingsUrl;
   }
 
+  public String getFUStatisticsSettingsUrl() {
+    return myFUStatisticsSettingsUrl;
+  }
+
   public String getStatisticsServiceUrl() {
     return myStatisticsServiceUrl;
   }
@@ -875,11 +881,13 @@ public class ApplicationInfoImpl extends ApplicationInfoEx {
     Element statisticsElement = getChild(parentNode, ELEMENT_STATISTICS);
     if (statisticsElement != null) {
       myStatisticsSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SETTINGS);
+      myFUStatisticsSettingsUrl = statisticsElement.getAttributeValue(ATTRIBUTE_FU_STATISTICS_SETTINGS);
       myStatisticsServiceUrl  = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE);
       myStatisticsServiceKey  = statisticsElement.getAttributeValue(ATTRIBUTE_STATISTICS_SERVICE_KEY);
     }
     else {
       myStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/stat-assistant.xml";
+      myFUStatisticsSettingsUrl = "https://www.jetbrains.com/idea/statistics/fus-assistant.xml";
       myStatisticsServiceUrl  = "https://www.jetbrains.com/idea/statistics/index.jsp";
       myStatisticsServiceKey  = null;
     }

platform/platform-impl/src/com/intellij/featureStatistics/fusCollectors/ProductivityUsageCollector.java

@@ -0,0 +1,40 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.featureStatistics.fusCollectors;
+
+import com.intellij.featureStatistics.FeatureDescriptor;
+import com.intellij.featureStatistics.ProductivityFeaturesRegistry;
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.internal.statistic.service.fus.collectors.ApplicationUsagesCollector;
+import com.intellij.util.containers.HashSet;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Set;
+
+public class ProductivityUsageCollector extends ApplicationUsagesCollector {
+
+  private static String GROUP_ID = "statistics.productivity";
+  @NotNull
+  @Override
+  public String getGroupId() {
+    return GROUP_ID;
+  }
+
+  @NotNull
+  @Override
+  public Set<UsageDescriptor> getUsages() {
+    Set<UsageDescriptor> usages = new HashSet<>();
+
+    final ProductivityFeaturesRegistry registry = ProductivityFeaturesRegistry.getInstance();
+    for (String featureId : registry.getFeatureIds()) {
+      final FeatureDescriptor featureDescriptor = registry.getFeatureDescriptor(featureId);
+      if (featureDescriptor != null) {
+        int count = featureDescriptor.getUsageCount();
+        if (count > 0) {
+          usages.add(new UsageDescriptor(featureId, count));
+        }
+      }
+    }
+
+    return usages;
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/actions/SendFUStatisticsAction.java

@@ -0,0 +1,41 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.actions;
+
+import com.intellij.internal.statistic.connect.StatisticsResult;
+import com.intellij.internal.statistic.connect.StatisticsService;
+import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
+import com.intellij.openapi.actionSystem.AnAction;
+import com.intellij.openapi.actionSystem.AnActionEvent;
+import com.intellij.openapi.application.ApplicationManager;
+import com.intellij.openapi.application.ModalityState;
+import com.intellij.openapi.progress.ProgressIndicator;
+import com.intellij.openapi.progress.ProgressManager;
+import com.intellij.openapi.progress.Task;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.ui.Messages;
+import com.intellij.openapi.util.text.StringUtil;
+import org.jetbrains.annotations.NotNull;
+
+public class SendFUStatisticsAction extends AnAction {
+
+  @Override
+  public void actionPerformed(@NotNull AnActionEvent e) {
+    final Project project = e.getProject();
+    if (project == null) {
+      return;
+    }
+
+    ProgressManager.getInstance().run(new Task.Backgroundable(project, "Collecting And Sending _NEW_ Statistics", false) {
+      @Override
+      public void run(@NotNull ProgressIndicator indicator) {
+        StatisticsService service = StatisticsUploadAssistant.getApprovedGroupsStatisticsService();
+        final StatisticsResult result = service.send();
+
+        ApplicationManager.getApplication().invokeLater(
+          () -> Messages.showMultilineInputDialog(project, "Result: " + result.getCode(), "Statistics Result",
+                                                StringUtil.replace(result.getDescription(), ";", "\n"),
+                                                null, null), ModalityState.NON_MODAL, project.getDisposed());
+      }
+    });
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/actions/SendStatisticsAction.java

@@ -41,19 +41,19 @@ public class SendStatisticsAction extends AnAction {
       return;
     }
 
-    ProgressManager.getInstance().run(new Task.Backgroundable(project, "Collecting And Sending Statistics", false) {
+    ProgressManager.getInstance().run(new Task.Backgroundable(project, "Collecting And Sending _OLD_ Statistics", false) {
       @Override
       public void run(@NotNull ProgressIndicator indicator) {
         UsageStatisticsPersistenceComponent statisticsPersistenceComponent = UsageStatisticsPersistenceComponent.getInstance();
         final boolean sendAllowed = statisticsPersistenceComponent.isAllowed();
         try {
           statisticsPersistenceComponent.setAllowed(true);
-          ApplicationStatisticsPersistenceComponent.persistOpenedProjects();
+          ApplicationStatisticsPersistenceComponent.getInstance().persistOpenedProjects();
         }
         finally {
           statisticsPersistenceComponent.setAllowed(sendAllowed);
         }
-        StatisticsService service = StatisticsUploadAssistant.getStatisticsService();
+        StatisticsService service = StatisticsUploadAssistant.getOldStatisticsService();
         final StatisticsResult result = service.send();
 
         ApplicationManager.getApplication().invokeLater(

platform/platform-impl/src/com/intellij/internal/statistic/beans/ConvertUsagesUtil.java

@@ -35,15 +35,13 @@ public class ConvertUsagesUtil {
 
 
   // @NotNull
-  public static <T extends UsageDescriptor> String convertUsages(Map<GroupDescriptor, Set<T>> map) {
+  public static <T extends UsageDescriptor> String convertUsages(Map<String, Set<T>> map) {
     assert map != null;
-    final Map<GroupDescriptor, Set<T>> sortedMap = sortDescriptorsByPriority(map);
-
     StringBuffer buffer = new StringBuffer();
-    for (Map.Entry<GroupDescriptor, Set<T>> entry : sortedMap.entrySet()) {
+    for (Map.Entry<String, Set<T>> entry : map.entrySet()) {
       String value = convertValueMap(entry.getValue());
       if (!StringUtil.isEmptyOrSpaces(value)) {
-        buffer.append(entry.getKey().getId());
+        buffer.append(entry.getKey());
         buffer.append(GROUP_SEPARATOR);
         buffer.append(value);
         buffer.append(GROUPS_SEPARATOR);

platform/platform-impl/src/com/intellij/internal/statistic/connect/RemotelyConfigurableStatisticsService.java

@@ -1,67 +0,0 @@
-package com.intellij.internal.statistic.connect;
-
-import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
-import com.intellij.notification.Notification;
-import com.intellij.notification.NotificationListener;
-import com.intellij.openapi.util.text.StringUtil;
-import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
-
-import java.util.Map;
-
-public class RemotelyConfigurableStatisticsService implements StatisticsService {
-
-  private StatisticsConnectionService myConnectionService;
-  private StatisticsDataSender sender;
-  private StatisticsUploadAssistant myAssistant;
-
-  public RemotelyConfigurableStatisticsService(@NotNull StatisticsConnectionService connectionService,
-                                               @NotNull StatisticsDataSender sender,
-                                               @NotNull StatisticsUploadAssistant assistant) {
-    myConnectionService = connectionService;
-    this.sender = sender;
-    myAssistant = assistant;
-  }
-
-  @Override
-  public StatisticsResult send() {
-    final String serviceUrl = myConnectionService.getServiceUrl();
-
-    if (serviceUrl == null) {
-      return new StatisticsResult(StatisticsResult.ResultCode.ERROR_IN_CONFIG, "ERROR");
-    }
-
-    if (!myConnectionService.isTransmissionPermitted()) {
-      return new StatisticsResult(StatisticsResult.ResultCode.NOT_PERMITTED_SERVER, "NOT_PERMITTED");
-    }
-
-    String content = myAssistant.getData(myConnectionService.getDisabledGroups());
-
-    if (StringUtil.isEmptyOrSpaces(content)) {
-      return new StatisticsResult(StatisticsResult.ResultCode.NOTHING_TO_SEND, "NOTHING_TO_SEND");
-    }
-
-    try {
-      sender.send(serviceUrl, content);
-      StatisticsUploadAssistant.updateSentTime();
-
-      return new StatisticsResult(StatisticsResult.ResultCode.SEND, content);
-    }
-    catch (Exception e) {
-      return new StatisticsResult(StatisticsResult.ResultCode.SENT_WITH_ERRORS, e.getMessage() != null ? e.getMessage() : "NPE");
-    }
-  }
-
-
-  @Override
-  public Notification createNotification(@NotNull final String groupDisplayId, @Nullable NotificationListener listener) {
-    return new StatisticsNotification(groupDisplayId, listener);
-  }
-
-  @Nullable
-  @Override
-  public Map<String, String> getStatisticsConfigurationLabels() {
-    // Use defaults
-    return null;
-  }
-}

platform/platform-impl/src/com/intellij/internal/statistic/connect/StatisticsConnectionService.java

@@ -17,23 +17,20 @@ package com.intellij.internal.statistic.connect;
 
 import com.intellij.facet.frameworks.SettingsConnectionService;
 import com.intellij.openapi.application.impl.ApplicationInfoImpl;
-import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.util.ArrayUtil;
-import com.intellij.util.containers.ContainerUtil;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-
+// this service connects to jetbrains.com resources and gets actual info
+// about statistics service connections
+// 1. url: wheren to post statistics data
+// 2. permitted: true/false. statistics could be stopped remotely. if false UsageCollectors won't be started
 public class StatisticsConnectionService extends SettingsConnectionService {
   private static final String PERMISSION_ATTR_NAME = "permitted";
   private static final String DISABLED = "disabled";
 
   public StatisticsConnectionService() {
-    this(((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getStatisticsSettingsUrl(),
-         ((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getStatisticsServiceUrl());
+    this("localhost", null);
   }
 
   public StatisticsConnectionService(@NotNull String settingsUrl, @Nullable String defaultServiceUrl) {
@@ -43,22 +40,11 @@ public class StatisticsConnectionService extends SettingsConnectionService {
   @NotNull
   @Override
   public String[] getAttributeNames() {
-    return ArrayUtil.mergeArrays(super.getAttributeNames(), PERMISSION_ATTR_NAME, DISABLED);
+    return ArrayUtil.mergeArrays(super.getAttributeNames(), PERMISSION_ATTR_NAME);
   }
 
   public boolean isTransmissionPermitted() {
     final String permitted = getSettingValue(PERMISSION_ATTR_NAME);
     return permitted == null || Boolean.parseBoolean(permitted);
   }
-
-  @NotNull
-  public Set<String> getDisabledGroups() {
-    final String disabledGroupsString = getSettingValue(DISABLED);
-    if (disabledGroupsString == null) {
-      return Collections.emptySet();
-    }
-
-    final List<String> disabledGroupsList = StringUtil.split(disabledGroupsString, ",");
-    return ContainerUtil.map2Set(disabledGroupsList, s -> s.trim());
-  }
 }

platform/platform-impl/src/com/intellij/internal/statistic/connect/StatisticsDataSender.java

@@ -17,9 +17,8 @@ package com.intellij.internal.statistic.connect;
 
 
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 public interface StatisticsDataSender {
 
-   void send(@NotNull String url, @NotNull String content) throws StatServiceException;
+   void send(@NotNull String url) throws StatServiceException;
 }

platform/platform-impl/src/com/intellij/internal/statistic/connect/StatisticsHttpClientSender.java

@@ -1,61 +0,0 @@
-/*
- * Copyright 2000-2014 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.internal.statistic.connect;
-
-import com.intellij.openapi.application.ApplicationNamesInfo;
-import com.intellij.openapi.application.PermanentInstallationID;
-import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.util.net.HttpConfigurable;
-import org.apache.http.Consts;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.client.fluent.Form;
-import org.apache.http.client.fluent.Request;
-import org.apache.http.client.fluent.Response;
-import org.jetbrains.annotations.NotNull;
-
-public class StatisticsHttpClientSender implements StatisticsDataSender {
-  @Override
-  public void send(@NotNull String url, @NotNull String content) throws StatServiceException {
-    try {
-      HttpConfigurable.getInstance().prepareURL(url);
-
-      Response response = Request.Post(url).bodyForm(Form.form().
-        add("content", content).
-        add("uuid", PermanentInstallationID.get()).
-        add("patch", String.valueOf(false)).
-        add("ide", ApplicationNamesInfo.getInstance().getProductName()).build(), Consts.UTF_8).execute();
-
-      final HttpResponse httpResponse = response.returnResponse();
-      if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
-        throw new StatServiceException("Error during data sending... Code: " + httpResponse.getStatusLine().getStatusCode());
-      }
-
-      final Header errors = httpResponse.getFirstHeader("errors");
-      if (errors != null) {
-        String value = errors.getValue();
-        throw new StatServiceException("Error during updating statistics " + (!StringUtil.isEmptyOrSpaces(value) ? " : " + value : ""));
-      }
-    }
-    catch (StatServiceException e) {
-      throw e;
-    }
-    catch (Exception e) {
-      throw new StatServiceException("Error during data sending...", e);
-    }
-  }
-}

platform/platform-impl/src/com/intellij/internal/statistic/connect/StatisticsService.java

@@ -15,7 +15,6 @@
  */
 package com.intellij.internal.statistic.connect;
 
-
 import com.intellij.notification.Notification;
 import com.intellij.notification.NotificationListener;
 import com.intellij.openapi.extensions.ExtensionPointName;
@@ -25,24 +24,13 @@ import org.jetbrains.annotations.Nullable;
 
 import java.util.Map;
 
+@Deprecated // to be removed in 2018.1. it will not be allowed to use own services
 public interface StatisticsService {
 
+  @Deprecated  // to be removed in 2018.1
   ExtensionPointName<StatisticsServiceEP> EP_NAME = ExtensionPointName.create("com.intellij.statisticsService");
 
-  @NonNls String TITLE = "title";
-  @NonNls String DETAILS = "details";
-  @NonNls String ALLOW_CHECKBOX = "allow-checkbox";
-
   StatisticsResult send();
 
   Notification createNotification(@NotNull String groupDisplayId, @Nullable NotificationListener listener);
-
-  /**
-   * Returns the custom text to be displayed in the statistics dialog. The keys in the map are control IDs ({@link #TITLE}, {@link #DETAILS}
-   * and {@link #ALLOW_CHECKBOX} and the values are the text to be displayed in the dialog.
-   *
-   * @return the text override map, or null if the standard text should be used.
-   */
-  @Nullable
-  Map<String, String> getStatisticsConfigurationLabels();
 }

platform/platform-impl/src/com/intellij/internal/statistic/connect/StatisticsServiceEP.java

@@ -22,6 +22,7 @@ import com.intellij.util.KeyedLazyInstance;
 import com.intellij.util.xmlb.annotations.Attribute;
 import org.jetbrains.annotations.NotNull;
 
+@Deprecated  // to be removed in 2018.1x
 public class StatisticsServiceEP extends AbstractExtensionPointBean implements KeyedLazyInstance<StatisticsService> {
   @Attribute("key")
   public String key;

platform/platform-impl/src/com/intellij/internal/statistic/service/ConfigurableStatisticsService.java

@@ -0,0 +1,44 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service;
+
+import com.intellij.internal.statistic.connect.*;
+import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
+import com.intellij.notification.Notification;
+import com.intellij.notification.NotificationListener;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+public abstract class ConfigurableStatisticsService<T extends StatisticsConnectionService> implements
+                                                                                           StatisticsService {
+  @Override
+  public final StatisticsResult send() {
+    final String serviceUrl = getConnectionService().getServiceUrl();
+    if (serviceUrl == null) {
+      return new StatisticsResult(StatisticsResult.ResultCode.ERROR_IN_CONFIG, "ERROR");
+    }
+
+    if (!getConnectionService().isTransmissionPermitted()) {
+      return new StatisticsResult(StatisticsResult.ResultCode.NOT_PERMITTED_SERVER, "NOT_PERMITTED");
+    }
+
+    try {
+      String sentData = sendData();
+      StatisticsUploadAssistant.updateSentTime();
+      return new StatisticsResult(StatisticsResult.ResultCode.SEND, sentData);
+    }
+    catch (Exception e) {
+      return new StatisticsResult(StatisticsResult.ResultCode.SENT_WITH_ERRORS, e.getMessage() != null ? e.getMessage() : "NPE");
+    }
+  }
+
+  @Override
+  public Notification createNotification(@NotNull final String groupDisplayId, @Nullable NotificationListener listener) {
+    return new StatisticsNotification(groupDisplayId, listener);
+  }
+
+  @Nullable
+  // result: the content was sent.
+  protected abstract String sendData();
+
+  public abstract T getConnectionService();
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/FUStatisticsService.java

@@ -0,0 +1,65 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus;
+
+import com.google.gson.Gson;
+import com.intellij.internal.statistic.connect.StatServiceException;
+import com.intellij.internal.statistic.service.ConfigurableStatisticsService;
+import com.intellij.internal.statistic.service.fus.beans.gson.FSContent;
+import com.intellij.internal.statistic.service.fus.collectors.FUStatisticsAggregator;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.io.StreamUtil;
+import com.intellij.openapi.vfs.CharsetToolkit;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+
+public class FUStatisticsService extends ConfigurableStatisticsService<FUStatisticsSettingsService> {
+  private static final Logger LOG = Logger.getInstance("com.intellij.internal.statistic.service.whiteList.FUStatisticsService");
+
+  private static final FUStatisticsSettingsService myConnections = FUStatisticsSettingsService.getInstance();
+  private static final FUStatisticsAggregator myAggregator = FUStatisticsAggregator.create();
+
+  @Override
+  protected String sendData() {
+
+    String serviceUrl = myConnections.getServiceUrl();
+    if (serviceUrl == null) return null;
+
+    FSContent gsonContent = myAggregator.getUsageCollectorsData(myConnections.getApprovedGroups());
+
+    if (gsonContent == null) return null;
+    try {
+      Gson gson = new Gson();
+      HttpClient httpClient = HttpClientBuilder.create().build();
+      String content = gson.toJson(gsonContent);
+
+      HttpPost post = new HttpPost(serviceUrl);
+      StringEntity postingString = new StringEntity(content);
+      post.setEntity(postingString);
+      post.setHeader("Content-type", "application/json");
+      HttpResponse response = httpClient.execute(post);
+
+      if (LOG.isDebugEnabled()) {
+        HttpEntity entity = response.getEntity();
+        if (entity != null) {
+          LOG.debug(StreamUtil.readText(entity.getContent(), CharsetToolkit.UTF8));
+        }
+      }
+      return content;
+    }
+    catch (StatServiceException e) {
+      throw e;
+    }
+    catch (Exception e) {
+      throw new StatServiceException("Error during data sending...", e);
+    }
+  }
+
+  @Override
+  public FUStatisticsSettingsService getConnectionService() {
+    return myConnections;
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/FUStatisticsSettingsService.java

@@ -0,0 +1,43 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus;
+
+import com.intellij.internal.statistic.connect.StatisticsConnectionService;
+import com.intellij.openapi.application.ApplicationInfo;
+import com.intellij.openapi.application.impl.ApplicationInfoImpl;
+import com.intellij.util.ArrayUtil;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Collections;
+import java.util.Set;
+
+// this service connects to jetbrains.com resources and gets actual info about running statistics services
+// 1. url: where to post statistics data.
+// 2. white-list-service: WhiteListService url: this service returns approved UsagesCollectors(groups)
+// 3. permitted: true/false. statistics could be stopped remotely. if false UsageCollectors won't be started
+public class FUStatisticsSettingsService extends StatisticsConnectionService {
+  private static final String APPROVED_GROUPS_SERVICE = "white-list-service";
+  public  static FUStatisticsSettingsService getInstance() {return  new FUStatisticsSettingsService();}
+  private FUStatisticsSettingsService() {
+    super(((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getFUStatisticsSettingsUrl(), null);
+  }
+
+  @NotNull
+  @Override
+  public String[] getAttributeNames() {
+    return ArrayUtil.mergeArrays(super.getAttributeNames(), APPROVED_GROUPS_SERVICE);
+  }
+
+  @NotNull
+  public Set<String> getApprovedGroups() {
+    final String approvedGroupsServiceUrl = getSettingValue(APPROVED_GROUPS_SERVICE);
+    if (approvedGroupsServiceUrl == null) {
+      return Collections.emptySet();
+    }
+    return FUStatisticsWhiteListGroupsService.getApprovedGroups(getProductRelatedUrl(approvedGroupsServiceUrl));
+  }
+
+  @NotNull
+  public String getProductRelatedUrl(@NotNull  String approvedGroupsServiceUrl) {
+    return approvedGroupsServiceUrl + ApplicationInfo.getInstance().getBuild().getProductCode() + ".json";
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/FUStatisticsWhiteListGroupsService.java

@@ -0,0 +1,65 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus;
+
+import com.google.gson.GsonBuilder;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.util.io.HttpRequests;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+// StatisticsWhiteListGroups service returns actual "approved" UsagesCollectors(groups)
+// the result is gson file:
+// {
+//  "groups" : [{
+//    "id" : "statistics.Productivity",
+//    "builds" : [{ "from" : "173.4127.37" }]
+//  }, {
+//    "id" : "spring-example"
+//  }]
+//}
+public class FUStatisticsWhiteListGroupsService {
+  private static final Logger LOG = Logger.getInstance("com.intellij.internal.statistic.service.whiteList.FUStatisticsWhiteListGroupsService");
+
+  @NotNull
+  public static Set<String> getApprovedGroups(@NotNull String serviceUrl) {
+    String content = null;
+    try {
+      content = HttpRequests.request(serviceUrl)
+                            .productNameAsUserAgent()
+                            .readString(null);
+    }  catch (IOException e) {
+      LOG.error(e);
+    }
+    if (content == null) return Collections.emptySet();
+
+    WLGroups groups = new GsonBuilder().create().fromJson(content, WLGroups.class);
+    return groups.groups.stream().map(group -> group.id).collect(Collectors.toSet());
+  }
+
+  private static class WLGroups {
+    @NotNull
+    public final ArrayList<WLGroup> groups = new ArrayList<>();
+  }
+
+  private static class WLGroup {
+    @NotNull
+    public final String id;
+    @NotNull
+    public final ArrayList<WLBuild> builds = new ArrayList<>();
+
+    public WLGroup(@NotNull String id) {
+      this.id = id;
+    }
+  }
+
+  private static class WLBuild {
+    public final String from;
+
+    private WLBuild(String from) {this.from = from;}
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/beans/gson/FSContent.java

@@ -0,0 +1,52 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.beans.gson;
+
+import com.intellij.openapi.application.ApplicationInfo;
+import com.intellij.openapi.application.PermanentInstallationID;
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Objects;
+
+public class FSContent {
+  public String product;
+  public String user;
+
+  public ArrayList<FSSession> sessions = null;
+
+  public static FSContent create() {return new FSContent();}
+
+  public FSContent() {
+    product = ApplicationInfo.getInstance().getBuild().getProductCode();
+    user = PermanentInstallationID.get();
+  }
+
+  @Nullable
+  public ArrayList<FSSession> getSessions() {
+    return sessions;
+  }
+
+  public void addSession(@NotNull FSSession session) {
+    if (sessions == null) {
+      sessions = ContainerUtil.newArrayList();
+    }
+    sessions.add(session);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    FSContent content = (FSContent)o;
+    return Objects.equals(product, content.product) &&
+           Objects.equals(user, content.user);
+  }
+
+  @Override
+  public int hashCode() {
+
+    return Objects.hash(product, user);
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/beans/gson/FSGroup.java

@@ -0,0 +1,50 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.beans.gson;
+
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+public class FSGroup {
+
+  public String id;
+  public Map<String, Integer> metrics ;
+
+  private FSGroup(String id, Set<UsageDescriptor> usages) {
+    this.id = id;
+    for (UsageDescriptor usage : usages) {
+      getMetrics().put(usage.getKey(), usage.getValue());
+    }
+  }
+
+  @NotNull
+  private Map<String, Integer> getMetrics() {
+    if (metrics == null) {
+      metrics = ContainerUtil.newHashMap();
+    }
+    return metrics;
+  }
+
+  public static FSGroup create(@NotNull String groupId, @NotNull Set<UsageDescriptor> usages) {
+     return new FSGroup(groupId, usages);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    FSGroup group = (FSGroup)o;
+    return Objects.equals(id, group.id) &&
+           Objects.equals(metrics, group.metrics);
+  }
+
+  @Override
+  public int hashCode() {
+
+    return Objects.hash(id, metrics);
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/beans/gson/FSSession.java

@@ -0,0 +1,56 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.beans.gson;
+
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.ArrayList;
+import java.util.Objects;
+
+public class FSSession {
+  public String id;
+  public String build;
+
+  public static FSSession create(String id, String build) {
+    return new FSSession(id, build);
+  }
+
+  private FSSession(String id, String build) {
+    this.id = id;
+    this.build = build;
+  }
+
+  public ArrayList<FSGroup> groups = null;
+
+  public void addGroup(@NotNull  FSGroup group) {
+    if (groups == null) {
+      groups = ContainerUtil.newArrayList();
+    }
+    groups.add(group);
+  }
+
+  @Nullable
+  public ArrayList<FSGroup> getGroups() {
+    return groups;
+  }
+
+  public boolean hasGroups() {
+    return groups != null;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+    FSSession session = (FSSession)o;
+    return Objects.equals(id, session.id) &&
+           Objects.equals(build, session.build);
+  }
+
+  @Override
+  public int hashCode() {
+
+    return Objects.hash(id, build);
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/ApplicationUsagesCollector.java

@@ -0,0 +1,32 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+import com.intellij.ide.plugins.cl.PluginClassLoader;
+import com.intellij.internal.statistic.CollectUsagesException;
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.openapi.extensions.ExtensionPointName;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public abstract class ApplicationUsagesCollector extends FeatureUsagesCollector {
+  private static final ExtensionPointName<ApplicationUsagesCollector> EP_NAME =
+    ExtensionPointName.create("com.intellij.statistics.applicationUsagesCollector");
+
+  @NotNull
+  public static Set<ApplicationUsagesCollector> getExtensions(@NotNull UsagesCollectorConsumer invoker) {
+    if (invoker.getClass().getClassLoader() instanceof PluginClassLoader) return Collections.emptySet();
+    return Arrays.stream(EP_NAME.getExtensions()).filter(u -> u.isValid()).collect(Collectors.toSet());
+  }
+
+  @NotNull
+  public static String getExtensionPointName() {
+    return EP_NAME.getName();
+  }
+
+  @NotNull
+  public abstract Set<UsageDescriptor> getUsages() throws CollectUsagesException;
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/FUSession.java

@@ -0,0 +1,52 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+import com.intellij.openapi.application.ApplicationInfo;
+import com.intellij.openapi.project.Project;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Objects;
+
+public class FUSession {
+  public final static FUSession APPLICATION_SESSION = create("application_session");
+
+  private int id;
+  private String buildId;
+
+  public static FUSession create(@NotNull String sessionId) {
+    return new FUSession(sessionId);
+  }
+
+  public static FUSession create(@NotNull Project project) {
+    return new FUSession(project.getName());
+  }
+
+  public FUSession(@NotNull String id) {
+    this.id = id.hashCode();
+    this.buildId = ApplicationInfo.getInstance().getBuild().asStringWithoutProductCodeAndSnapshot();
+    if (buildId.endsWith(".")) buildId += "0";
+  }
+
+  public int getId() {
+    return id;
+  }
+
+  public String getBuildId() {
+    return buildId;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof FUSession)) return false;
+    FUSession session = (FUSession)o;
+    return Objects.equals(id, session.id) &&
+           Objects.equals(buildId, session.buildId);
+  }
+
+  @Override
+  public int hashCode() {
+
+    return Objects.hash(id, buildId);
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/FUStatisticsAggregator.java

@@ -0,0 +1,123 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+import com.intellij.internal.statistic.CollectUsagesException;
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.internal.statistic.service.fus.beans.gson.FSContent;
+import com.intellij.internal.statistic.service.fus.beans.gson.FSGroup;
+import com.intellij.internal.statistic.service.fus.beans.gson.FSSession;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.project.Project;
+import com.intellij.openapi.project.ProjectManager;
+import com.intellij.util.containers.ContainerUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static com.intellij.internal.statistic.utils.StatisticsUploadAssistant.LOCK;
+
+public class FUStatisticsAggregator implements UsagesCollectorConsumer {
+  private static final Logger LOG = Logger.getInstance("com.intellij.internal.statistic.service.whiteList.collectors.FUStatisticsAggregator.");
+
+  public FUStatisticsAggregator() {
+  }
+
+  public static FUStatisticsAggregator create() {
+     return new FUStatisticsAggregator();
+  }
+
+  @Nullable
+  public  FSContent getUsageCollectorsData(@NotNull Set<String> approvedGroups ) {
+    if (approvedGroups.isEmpty()) return null;
+
+    FSContent content = FSContent.create();
+
+    collectApplicationUsages(content, approvedGroups);
+    collectOpenProjectUsages(content, approvedGroups);
+    collectPersistedProjectUsages(content, approvedGroups);
+
+    return content;
+  }
+
+  public void collectApplicationUsages(FSContent content, @NotNull Set<String> approvedGroups) {
+    writeContent(content, FUSession.APPLICATION_SESSION, getApplicationUsages(approvedGroups));
+  }
+
+  public void collectOpenProjectUsages(@NotNull FSContent content, @NotNull Set<String> approvedGroups) {
+    for (Project project : ProjectManager.getInstance().getOpenProjects()) {
+      if (project.isDisposed()) continue;
+      Map<String, Set<UsageDescriptor>> usages = getProjectUsages(project, approvedGroups);
+      if (!usages.isEmpty()) {
+        writeContent(content, FUSession.create(project), usages);
+      }
+    }
+  }
+
+  private void collectPersistedProjectUsages(@NotNull FSContent content,
+                                             @NotNull Set<String> approvedGroups) {
+    // todo: implement persistence component
+  }
+
+  @NotNull
+  public Map<String, Set<UsageDescriptor>> getApplicationUsages(@NotNull Set<String> approvedGroups) {
+    synchronized (LOCK) {
+      Map<String, Set<UsageDescriptor>> usageDescriptors = new LinkedHashMap<>();
+      for (ApplicationUsagesCollector usagesCollector : ApplicationUsagesCollector.getExtensions(this)) {
+        if (!usagesCollector.isValid()) continue;
+        String groupDescriptor = usagesCollector.getGroupId();
+        if (!approvedGroups.contains(groupDescriptor)) continue;
+
+        try {
+          usageDescriptors.merge(groupDescriptor, usagesCollector.getUsages(), ContainerUtil::union);
+        }
+        catch (CollectUsagesException e) {
+          LOG.info(e);
+        }
+      }
+
+      return usageDescriptors;
+    }
+  }
+
+  @NotNull
+  public Map<String, Set<UsageDescriptor>> getProjectUsages(@NotNull Project project, @NotNull Set<String> approvedGroups) {
+    synchronized (LOCK) {
+      Map<String, Set<UsageDescriptor>> usageDescriptors = new LinkedHashMap<>();
+      for (ProjectUsagesCollector usagesCollector : ProjectUsagesCollector.getExtensions(this)) {
+        if (!usagesCollector.isValid()) continue;
+        String groupDescriptor = usagesCollector.getGroupId();
+        if (!approvedGroups.contains(groupDescriptor)) continue;
+
+        try {
+          usageDescriptors.merge(groupDescriptor, usagesCollector.getUsages(project), ContainerUtil::union);
+        }
+        catch (CollectUsagesException e) {
+          LOG.info(e);
+        }
+      }
+
+      return usageDescriptors;
+    }
+  }
+
+
+  public void writeContent(FSContent content,
+                           @NotNull FUSession fuSession,
+                           Map<String, Set<UsageDescriptor>> usages) {
+    FSSession session = FSSession.create(Integer.toString(fuSession.getId()), fuSession.getBuildId());
+    if (content.getSessions() !=null && content.getSessions().contains(session)) return;
+
+    for (Map.Entry<String, Set<UsageDescriptor>> group : usages.entrySet()) {
+      Set<UsageDescriptor> value = group.getValue();
+      if (!value.isEmpty()) {
+        session.addGroup(FSGroup.create(group.getKey(), value));
+      }
+    }
+    if (session.hasGroups()) {
+      content.addSession(session);
+    }
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/FeatureUsagesCollector.java

@@ -0,0 +1,17 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.util.regex.Pattern;
+
+public abstract class FeatureUsagesCollector {
+  private static final String GROUP_ID_PATTERN = "([a-zA-Z]*\\.)*[a-zA-Z]*";
+
+  public final boolean isValid() {
+    return Pattern.compile(GROUP_ID_PATTERN).matcher(getGroupId()).matches();
+  }
+
+  @NotNull
+  public abstract String getGroupId();
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/ProjectUsagesCollector.java

@@ -0,0 +1,33 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+import com.intellij.ide.plugins.cl.PluginClassLoader;
+import com.intellij.internal.statistic.CollectUsagesException;
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.openapi.extensions.ExtensionPointName;
+import com.intellij.openapi.project.Project;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public abstract class ProjectUsagesCollector extends FeatureUsagesCollector {
+  private static final ExtensionPointName<ProjectUsagesCollector> EP_NAME =
+    ExtensionPointName.create("com.intellij.statistics.projectUsagesCollector");
+
+  @NotNull
+  public static Set<ProjectUsagesCollector> getExtensions(@NotNull UsagesCollectorConsumer invoker) {
+    if (invoker.getClass().getClassLoader() instanceof PluginClassLoader) return Collections.emptySet();
+    return Arrays.stream(EP_NAME.getExtensions()).filter(u -> u.isValid()).collect(Collectors.toSet());
+  }
+
+  @NotNull
+  public static String getExtensionPointName() {
+    return EP_NAME.getName();
+  }
+
+  @NotNull
+  public abstract Set<UsageDescriptor> getUsages(@NotNull Project project) throws CollectUsagesException;
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/fus/collectors/UsagesCollectorConsumer.java

@@ -0,0 +1,7 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.fus.collectors;
+
+// markup interface.
+// UsageCollector extentions could be requested only by UsagesCollectorConsumer
+interface UsagesCollectorConsumer {
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/old/OldConfigurableStatisticsService.java

@@ -0,0 +1,96 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.old;
+
+import com.intellij.internal.statistic.CollectUsagesException;
+import com.intellij.internal.statistic.UsagesCollector;
+import com.intellij.internal.statistic.beans.ConvertUsagesUtil;
+import com.intellij.internal.statistic.beans.UsageDescriptor;
+import com.intellij.internal.statistic.connect.StatServiceException;
+import com.intellij.internal.statistic.connect.StatisticsConnectionService;
+import com.intellij.internal.statistic.service.ConfigurableStatisticsService;
+import com.intellij.openapi.application.ApplicationNamesInfo;
+import com.intellij.openapi.application.PermanentInstallationID;
+import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.openapi.util.text.StringUtil;
+import com.intellij.util.containers.ContainerUtil;
+import com.intellij.util.net.HttpConfigurable;
+import org.apache.http.Consts;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.fluent.Form;
+import org.apache.http.client.fluent.Request;
+import org.apache.http.client.fluent.Response;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static com.intellij.internal.statistic.utils.StatisticsUploadAssistant.LOCK;
+
+@Deprecated  // to be removed in 2018.1x
+public class OldConfigurableStatisticsService extends
+                                              ConfigurableStatisticsService<StatisticsConnectionService> {
+
+  private static final Logger LOG = Logger.getInstance(OldConfigurableStatisticsService.class);
+  private final StatisticsConnectionService connectionService = new OldStatisticsConnectionService();
+
+  @Override
+  public String sendData() {
+    String url = connectionService.getServiceUrl();
+    Map<String, Set<UsageDescriptor>> content = getContentToSend();
+
+    try {
+      HttpConfigurable.getInstance().prepareURL(url);
+      String usages = ConvertUsagesUtil.convertUsages(content);
+      Response response = Request.Post(url).bodyForm(Form.form().
+                                                       add("content", usages).
+                                                           add("uuid", PermanentInstallationID.get()).
+                                                           add("patch", String.valueOf(false)).
+                                                           add("ide", ApplicationNamesInfo.getInstance().getProductName()).build(),
+                                                     Consts.UTF_8).execute();
+
+      final HttpResponse httpResponse = response.returnResponse();
+      if (httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
+        throw new StatServiceException("Error during data sending... Code: " + httpResponse.getStatusLine().getStatusCode());
+      }
+
+      final Header errors = httpResponse.getFirstHeader("errors");
+      if (errors != null) {
+        String value = errors.getValue();
+        throw new StatServiceException("Error during updating statistics " + (!StringUtil.isEmptyOrSpaces(value) ? " : " + value : ""));
+      }
+      return usages;
+    }
+    catch (StatServiceException e) {
+      throw e;
+    }
+    catch (Exception e) {
+      throw new StatServiceException("Error during data sending...", e);
+    }
+  }
+
+  @NotNull
+  public Map<String, Set<UsageDescriptor>> getContentToSend() {
+    synchronized (LOCK) {
+      Map<String, Set<UsageDescriptor>> usageDescriptors = new LinkedHashMap<>();
+      for (UsagesCollector usagesCollector : UsagesCollector.EP_NAME.getExtensions()) {
+        String groupDescriptor = usagesCollector.getGroupId().getId();
+        try {
+          usageDescriptors.merge(groupDescriptor, usagesCollector.getUsages(), ContainerUtil::union);
+        }
+        catch (CollectUsagesException e) {
+          LOG.info(e);
+        }
+      }
+
+      return usageDescriptors;
+    }
+  }
+
+  @Override
+  public StatisticsConnectionService getConnectionService() {
+    return connectionService;
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/service/old/OldStatisticsConnectionService.java

@@ -0,0 +1,15 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal.statistic.service.old;
+
+import com.intellij.internal.statistic.connect.StatisticsConnectionService;
+import com.intellij.openapi.application.impl.ApplicationInfoImpl;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+@Deprecated  // to be removed in 2018.1x
+public class OldStatisticsConnectionService extends StatisticsConnectionService {
+  public OldStatisticsConnectionService() {
+    super(((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getStatisticsSettingsUrl(),
+          ((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getStatisticsServiceUrl());
+  }
+}

platform/platform-impl/src/com/intellij/internal/statistic/updater/SendStatisticsComponent.java

@@ -20,13 +20,11 @@ import com.intellij.ide.FrameStateListener;
 import com.intellij.ide.FrameStateManager;
 import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
 import com.intellij.internal.statistic.connect.StatisticsService;
-import com.intellij.internal.statistic.connect.StatisticsServiceEP;
 import com.intellij.notification.NotificationDisplayType;
 import com.intellij.notification.NotificationsConfiguration;
 import com.intellij.notification.impl.NotificationsConfigurationImpl;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.components.ApplicationComponent;
-import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.wm.WindowManager;
 import com.intellij.openapi.wm.ex.WindowManagerEx;
 import com.intellij.openapi.wm.impl.IdeFrameImpl;
@@ -35,13 +33,10 @@ import com.intellij.ui.BalloonLayoutImpl;
 import org.jetbrains.annotations.NotNull;
 
 import java.awt.*;
-import java.util.Arrays;
 import java.util.concurrent.TimeUnit;
 
 public class SendStatisticsComponent implements ApplicationComponent {
 
-  private static final Logger LOG = Logger.getInstance(SendStatisticsComponent.class);
-
   private static final int DELAY_IN_MIN = 10;
 
   private final FrameStateManager myFrameStateManager;
@@ -68,7 +63,7 @@ public class SendStatisticsComponent implements ApplicationComponent {
   }
 
   private void runStatisticsService() {
-    final StatisticsService statisticsService = StatisticsUploadAssistant.getStatisticsService();
+    final StatisticsService statisticsService = StatisticsUploadAssistant.getApprovedGroupsStatisticsService();
 
     if (StatisticsUploadAssistant.isShouldShowNotification()) {
       myFrameStateManager.addListener(new FrameStateListener.Adapter() {
@@ -82,12 +77,15 @@ public class SendStatisticsComponent implements ApplicationComponent {
       });
     }
     else if (StatisticsUploadAssistant.isSendAllowed() && StatisticsUploadAssistant.isTimeToSend()) {
-      runWithDelay(statisticsService);
+      runWithDelay(statisticsService, DELAY_IN_MIN);
+
+      // TODO: to be removed in 2018.1
+      runWithDelay(StatisticsUploadAssistant.getOldStatisticsService(), 2 * DELAY_IN_MIN);
     }
   }
 
-  private static void runWithDelay(@NotNull final StatisticsService statisticsService) {
-    JobScheduler.getScheduler().schedule(statisticsService::send, DELAY_IN_MIN, TimeUnit.MINUTES);
+  private static void runWithDelay(@NotNull final StatisticsService statisticsService, int delayInMin) {
+    JobScheduler.getScheduler().schedule(statisticsService::send, delayInMin, TimeUnit.MINUTES);
   }
 
   @Override

platform/platform-impl/src/com/intellij/internal/statistic/utils/StatisticsUploadAssistant.java

@@ -17,37 +17,18 @@ package com.intellij.internal.statistic.utils;
 
 import com.intellij.featureStatistics.FeatureUsageTracker;
 import com.intellij.featureStatistics.FeatureUsageTrackerImpl;
-import com.intellij.internal.statistic.CollectUsagesException;
-import com.intellij.internal.statistic.UsagesCollector;
-import com.intellij.internal.statistic.beans.ConvertUsagesUtil;
-import com.intellij.internal.statistic.beans.GroupDescriptor;
-import com.intellij.internal.statistic.beans.UsageDescriptor;
-import com.intellij.internal.statistic.connect.RemotelyConfigurableStatisticsService;
-import com.intellij.internal.statistic.connect.StatisticsConnectionService;
-import com.intellij.internal.statistic.connect.StatisticsHttpClientSender;
+import com.intellij.internal.statistic.service.old.OldConfigurableStatisticsService;
 import com.intellij.internal.statistic.connect.StatisticsService;
 import com.intellij.internal.statistic.persistence.SentUsagesPersistence;
 import com.intellij.internal.statistic.persistence.UsageStatisticsPersistenceComponent;
-import com.intellij.openapi.application.impl.ApplicationInfoImpl;
-import com.intellij.openapi.diagnostic.Logger;
+import com.intellij.internal.statistic.service.fus.FUStatisticsService;
 import com.intellij.openapi.util.KeyedExtensionCollector;
-import com.intellij.openapi.vfs.CharsetToolkit;
 import com.intellij.util.Time;
-import com.intellij.util.containers.ContainerUtil;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
 
 public class StatisticsUploadAssistant {
-  private static final Logger LOG = Logger.getInstance(UsagesCollector.class);
   public static final Object LOCK = new Object();
 
-  public String getData() {
-    return getData(Collections.emptySet());
-  }
+  private StatisticsUploadAssistant(){};
 
   public static boolean isShouldShowNotification() {
     return UsageStatisticsPersistenceComponent.getInstance().isShowNotification() &&
@@ -72,68 +53,17 @@ public class StatisticsUploadAssistant {
     return settings != null && settings.isAllowed();
   }
 
-  public String getData(@NotNull Set<String> disabledGroups) {
-    return getDataString(disabledGroups);
-  }
-
   public static void updateSentTime() {
     UsageStatisticsPersistenceComponent.getInstance().setSentTime(System.currentTimeMillis());
   }
 
-  @NotNull
-  public static String getDataString(@NotNull Set<String> disabledGroups) {
-    return getDataString(disabledGroups, 0);
+  @Deprecated  // to be removed in 2018.1x
+  public static StatisticsService getOldStatisticsService() {
+    return new OldConfigurableStatisticsService();
   }
 
-  @NotNull
-  public static String getDataString(@NotNull Set<String> disabledGroups,
-                                     int maxSize) {
-    return getDataString(getAllUsages(disabledGroups), maxSize);
+  public static StatisticsService getApprovedGroupsStatisticsService() {
+    return new FUStatisticsService();
   }
 
-  public static <T extends UsageDescriptor> String getDataString(@NotNull Map<GroupDescriptor, Set<T>> usages, int maxSize) {
-    if (usages.isEmpty()) {
-      return "";
-    }
-
-    String dataStr = ConvertUsagesUtil.convertUsages(usages);
-    return maxSize > 0 && dataStr.getBytes(CharsetToolkit.UTF8_CHARSET).length > maxSize ? ConvertUsagesUtil.cutDataString(dataStr, maxSize) : dataStr;
-  }
-
-  private static final KeyedExtensionCollector<StatisticsService, String> COLLECTOR;
-
-  static {
-    COLLECTOR = new KeyedExtensionCollector<>("com.intellij.statisticsService");
-  }
-
-  public static StatisticsService getStatisticsService() {
-    String key = ((ApplicationInfoImpl)ApplicationInfoImpl.getShadowInstance()).getStatisticsServiceKey();
-    StatisticsService service = key == null ? null : COLLECTOR.findSingle(key);
-    if (service != null) {
-      return service;
-    }
-
-    return new RemotelyConfigurableStatisticsService(new StatisticsConnectionService(),
-                                                     new StatisticsHttpClientSender(),
-                                                     new StatisticsUploadAssistant());
-  }
-
-  @NotNull
-  private static Map<GroupDescriptor, Set<UsageDescriptor>> getAllUsages(@NotNull Set<String> disabledGroups) {
-    synchronized (LOCK) {
-      Map<GroupDescriptor, Set<UsageDescriptor>> usageDescriptors = new LinkedHashMap<>();
-      for (UsagesCollector usagesCollector : UsagesCollector.EP_NAME.getExtensions()) {
-        GroupDescriptor groupDescriptor = usagesCollector.getGroupId();
-        if (!disabledGroups.contains(groupDescriptor.getId())) {
-          try {
-            usageDescriptors.merge(groupDescriptor, usagesCollector.getUsages(), ContainerUtil::union);
-          }
-          catch (CollectUsagesException e) {
-            LOG.info(e);
-          }
-        }
-      }
-      return usageDescriptors;
-    }
-  }
 }

platform/platform-resources/src/META-INF/PlatformExtensionPoints.xml

@@ -193,6 +193,8 @@
     <extensionPoint name="editorNavigation" interface="com.intellij.openapi.editor.EditorNavigationDelegate"/>
 
     <extensionPoint name="statistics.usagesCollector" interface="com.intellij.internal.statistic.UsagesCollector"/>
+    <extensionPoint name="statistics.applicationUsagesCollector" interface="com.intellij.internal.statistic.service.fus.collectors.ApplicationUsagesCollector"/>
+    <extensionPoint name="statistics.projectUsagesCollector" interface="com.intellij.internal.statistic.service.fus.collectors.ProjectUsagesCollector"/>
 
     <extensionPoint name="editorHighlighterProvider" beanClass="com.intellij.openapi.fileTypes.FileTypeExtensionPoint">
       <with attribute="implementationClass" implements="com.intellij.openapi.fileTypes.EditorHighlighterProvider"/>

platform/platform-resources/src/META-INF/PlatformExtensions.xml

@@ -385,6 +385,9 @@
     <statistics.usagesCollector implementation="com.intellij.ide.plugins.DisabledPluginsUsagesCollector"/>
     <statistics.usagesCollector implementation="com.intellij.ide.plugins.NonBundledPluginsUsagesCollector"/>
     <statistics.usagesCollector implementation="com.intellij.featureStatistics.FeaturesUsageCollector"/>
+
+    <statistics.applicationUsagesCollector implementation="com.intellij.featureStatistics.fusCollectors.ProductivityUsageCollector"/>
+
     <statistics.usagesCollector implementation="com.intellij.openapi.vcs.statistics.VcsUsagesCollector"/>
     <statistics.usagesCollector implementation="com.intellij.openapi.vcs.statistics.VcsOptionsUsagesCollector"/>
     <statistics.usagesCollector implementation="com.intellij.openapi.vcs.statistics.VcsStatsUsagesCollector"/>

platform/platform-resources/src/idea/PlatformActions.xml

@@ -825,7 +825,8 @@
       <separator/>
       <reference ref="MaintenanceGroup"/>
 
-      <action internal="true" id="SendStatistics" class="com.intellij.internal.statistic.actions.SendStatisticsAction" text="Collect And Send Statistics"/>
+      <action internal="true" id="SendStatistics" class="com.intellij.internal.statistic.actions.SendStatisticsAction" text="OLD: Collect And Send Statistics"/>
+      <action internal="true" id="SendFUStatistics" class="com.intellij.internal.statistic.actions.SendFUStatisticsAction" text="NEW: Collect And Send Statistics"/>
 
       <add-to-group group-id="ToolsMenu" anchor="last"/>
     </group>

platform/platform-tests/testSrc/com/intellij/usagesStatistics/RemotelyConfigurableStatServiceTest.java

@@ -1,112 +0,0 @@
-/*
- * Copyright 2000-2013 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.usagesStatistics;
-
-import com.intellij.internal.statistic.utils.StatisticsUploadAssistant;
-import com.intellij.internal.statistic.connect.RemotelyConfigurableStatisticsService;
-import com.intellij.internal.statistic.connect.StatisticsConnectionService;
-import com.intellij.internal.statistic.connect.StatisticsHttpClientSender;
-import com.intellij.internal.statistic.connect.StatisticsResult;
-import com.intellij.util.net.NetUtils;
-import org.jetbrains.annotations.NotNull;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class RemotelyConfigurableStatServiceTest {
-  private static String STAT_URL;
-  private static String STAT_CONFIG_URL;
-
-  @BeforeClass
-  public static void init() throws Exception {
-    int port = NetUtils.findAvailableSocketPort();
-    STAT_URL = "http://localhost:" + port + "/stat.jsp";
-    STAT_CONFIG_URL = "http://localhost:" + port + "/config.jsp";
-  }
-
-  @Test
-  public void testStatisticsConnectionServiceDefaultSettings() {
-    StatisticsConnectionService connectionService = new StatisticsConnectionService(STAT_CONFIG_URL, STAT_URL);
-    assertEquals(STAT_URL, connectionService.getServiceUrl());
-
-    assertTrue(connectionService.isTransmissionPermitted());
-    String[] attributeNames = connectionService.getAttributeNames();
-
-    assertEquals(3, attributeNames.length);
-    assertEquals("url", attributeNames[0]);
-    assertEquals("permitted", attributeNames[1]);
-    assertEquals("disabled", attributeNames[2]);
-  }
-
-  @Test
-  public void testEmptyDataSending() {
-    RemotelyConfigurableStatisticsService service =
-      new RemotelyConfigurableStatisticsService(new StatisticsConnectionService(),
-                                                new StatisticsHttpClientSender(),
-                                                new StatisticsUploadAssistant() {
-                                                  @Override
-                                                  public String getData(@NotNull Set<String> disabledGroups) {
-                                                    return "";
-                                                  }
-                                                });
-    StatisticsResult result = service.send();
-    assertEquals(StatisticsResult.ResultCode.NOTHING_TO_SEND, result.getCode());
-  }
-
-  @Test
-  public void testIncorrectUrlSending() {
-    RemotelyConfigurableStatisticsService service =
-      new RemotelyConfigurableStatisticsService(new StatisticsConnectionService(STAT_CONFIG_URL, STAT_URL),
-                                                new StatisticsHttpClientSender(),
-                                                new StatisticsUploadAssistant() {
-                                                  @Override
-                                                  public String getData(@NotNull Set<String> disabledGroups) {
-                                                    return "group:key1=11";
-                                                  }
-                                                });
-    StatisticsResult result = service.send();
-    assertEquals(StatisticsResult.ResultCode.SENT_WITH_ERRORS, result.getCode());
-  }
-
-  @Test
-  public void testRemotelyDisabledTransmission() {
-    RemotelyConfigurableStatisticsService service =
-      new RemotelyConfigurableStatisticsService(new StatisticsConnectionService() {
-                                                  @Override
-                                                  public boolean isTransmissionPermitted() {
-                                                    return false;
-                                                  }
-                                                },
-                                                new StatisticsHttpClientSender(),
-                                                new StatisticsUploadAssistant());
-    StatisticsResult result = service.send();
-    assertEquals(StatisticsResult.ResultCode.NOT_PERMITTED_SERVER, result.getCode());
-  }
-
-  @Test
-  public void testErrorInRemoteConfiguration() {
-    RemotelyConfigurableStatisticsService service =
-      new RemotelyConfigurableStatisticsService(new StatisticsConnectionService(STAT_CONFIG_URL, null),
-                                                new StatisticsHttpClientSender(),
-                                                new StatisticsUploadAssistant());
-    StatisticsResult result = service.send();
-    assertEquals(StatisticsResult.ResultCode.ERROR_IN_CONFIG, result.getCode());
-  }
-}

platform/platform-tests/testSrc/com/intellij/usagesStatistics/StatisticsUploadAssistantTest.java

@@ -1,153 +0,0 @@
-/*
- * Copyright 2000-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.intellij.usagesStatistics;
-
-import com.intellij.internal.statistic.beans.ConvertUsagesUtil;
-import com.intellij.internal.statistic.beans.GroupDescriptor;
-import com.intellij.internal.statistic.beans.UsageDescriptor;
-import com.intellij.openapi.util.text.StringUtil;
-import com.intellij.util.containers.hash.HashMap;
-import com.intellij.util.containers.hash.LinkedHashMap;
-import junit.framework.TestCase;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class StatisticsUploadAssistantTest extends TestCase {
-
-    @Override
-    protected void setUp() throws Exception {
-        super.setUp();
-    }
-
-    @Override
-    protected void tearDown() throws Exception {
-        super.tearDown();
-    }
-
-    public void testConvertUsagesWithPriority() {
-        final Map<GroupDescriptor, Set<UsageDescriptor>> patchedUsages = new HashMap<>();
-
-        createPatchDescriptor(patchedUsages, "low", GroupDescriptor.LOWER_PRIORITY, "l1", 1);
-        createPatchDescriptor(patchedUsages, "low", GroupDescriptor.LOWER_PRIORITY, "l2", 1);
-        createPatchDescriptor(patchedUsages, "high", GroupDescriptor.HIGHER_PRIORITY, "h", 1);
-        createPatchDescriptor(patchedUsages, "high", GroupDescriptor.HIGHER_PRIORITY, "h2", 1);
-        createPatchDescriptor(patchedUsages, "default_1", GroupDescriptor.DEFAULT_PRIORITY, "d11", 1);
-        createPatchDescriptor(patchedUsages, "default_2", GroupDescriptor.DEFAULT_PRIORITY, "d21", 1);
-        createPatchDescriptor(patchedUsages, "default_1", GroupDescriptor.DEFAULT_PRIORITY, "d12", 1);
-
-
-        assertEquals(ConvertUsagesUtil.convertUsages(patchedUsages),
-                     "high:h=1,h2=1;default_1:d11=1,d12=1;default_2:d21=1;low:l1=1,l2=1;");
-    }
-
-    public void testConvertUsagesWithEqualPriority() {
-        final Map<GroupDescriptor, Set<UsageDescriptor>> patchedUsages = new HashMap<>();
-
-  createPatchDescriptor(patchedUsages, "g4", GroupDescriptor.HIGHER_PRIORITY, "1", 1);
-        createPatchDescriptor(patchedUsages, "g2", GroupDescriptor.HIGHER_PRIORITY, "2", 1);
-        createPatchDescriptor(patchedUsages, "g1", GroupDescriptor.HIGHER_PRIORITY, "3", 1);
-        createPatchDescriptor(patchedUsages, "g3", GroupDescriptor.HIGHER_PRIORITY, "4", 1);
-
-
-        assertEquals(ConvertUsagesUtil.convertUsages(patchedUsages), "g1:3=1;g2:2=1;g3:4=1;g4:1=1;");
-    }
-
-    public void testConvertString() {
-        final Map<GroupDescriptor, Set<UsageDescriptor>> patchedUsages = new HashMap<>();
-
-        createPatchDescriptor(patchedUsages, "g4", GroupDescriptor.HIGHER_PRIORITY, "1", 1);
-        createPatchDescriptor(patchedUsages, "g2", GroupDescriptor.HIGHER_PRIORITY, "2", 1);
-        createPatchDescriptor(patchedUsages, "g1", GroupDescriptor.HIGHER_PRIORITY, "3", 1);
-        createPatchDescriptor(patchedUsages, "g3", GroupDescriptor.HIGHER_PRIORITY, "4", 1);
-
-        assertMapEquals(patchedUsages, ConvertUsagesUtil.convertString("g3:4=1;g1:3=1;g4:1=1;g2:2=1;"));
-    }
-
-    public void testConvertEmptyString() {
-        assertEquals(ConvertUsagesUtil.convertString("").size(), 0);
-    }
-
-    public void testConvertBrokenString() {
-        assertEquals(ConvertUsagesUtil.convertString("asdfasdfsad").size(), 0);
-        assertEquals(ConvertUsagesUtil.convertString("asdf:asdfsad").size(), 0);
-        assertEquals(ConvertUsagesUtil.convertString("asdfa:sd;fsad").size(), 0);
-        assertEquals(ConvertUsagesUtil.convertString("asdfa:sd;fs,ad").size(), 0);
-        assertEquals(ConvertUsagesUtil.convertString("asdfa:sd;f;sad").size(), 0);
-        assertEquals(ConvertUsagesUtil.convertString("asdfa:sd=ds2,f=f,sad=;").size(), 0);
-    }
-
-    private static <T extends UsageDescriptor> void  assertMapEquals(@NotNull Map<GroupDescriptor, Set<T>> expected, @NotNull Map<GroupDescriptor, Set<UsageDescriptor>> actual) {
-        assertEquals(expected.size(), actual.size());
-
-        for (Map.Entry<GroupDescriptor, Set<T>> expectedEntry : expected.entrySet()) {
-            final GroupDescriptor expectedGroupDescriptor = expectedEntry.getKey();
-
-            assertTrue(actual.containsKey(expectedGroupDescriptor));
-
-            assertSetEquals(expectedEntry.getValue(), actual.get(expectedEntry.getKey()));
-        }
-    }
-
-    private static void assertSetEquals(@NotNull Set<? extends UsageDescriptor> expected, @NotNull Set<? extends UsageDescriptor> actual) {
-        assertEquals(expected.size(), actual.size());
-        for (UsageDescriptor usageDescriptor : expected) {
-            boolean exists = false;
-            for (UsageDescriptor descriptor : actual) {
-                if (usageDescriptor.getKey().equals(descriptor.getKey()) && usageDescriptor.getValue() == descriptor.getValue()) {
-                    exists = true;
-                    break;
-                }
-            }
-            assertTrue(asString(usageDescriptor) + " usage doesn't exist", exists);
-        }
-    }
-
-    private static String asString(UsageDescriptor usage) {
-        return usage.getKey() + "=" + usage.getValue();
-    }
-
-    private static void createPatchDescriptor(Map<GroupDescriptor, Set<UsageDescriptor>> patchedUsages, String groupId, double priority, String key, int i) {
-        final GroupDescriptor groupDescriptor = GroupDescriptor.create(groupId, priority);
-
-        if (!patchedUsages.containsKey(groupDescriptor)){
-            patchedUsages.put(groupDescriptor, new LinkedHashSet<>());
-        }
-        patchedUsages.get(groupDescriptor).add(new UsageDescriptor(key, i));
-    }
-
-
-    protected static UsageDescriptor createDescriptor(String k, int i) {
-        return new UsageDescriptor(k, i);
-    }
-
-    protected static Map<GroupDescriptor, Set<UsageDescriptor>> createDescriptors(String... strs) {
-        Map<GroupDescriptor, Set<UsageDescriptor>> set = new LinkedHashMap<>();
-        for (String str : strs) {
-            final List<String> list = StringUtil.split(str, ":");
-            final GroupDescriptor g = GroupDescriptor.create(list.get(0));
-            if (!set.containsKey(g)) {
-                set.put(g, new LinkedHashSet<>());
-            }
-            set.get(g).add(createDescriptor(list.get(1), Integer.parseInt(list.get(2))));
-        }
-
-        return set;
-    }
-}