move shared index metadata generation to the DumpJdkIndexStarter.kt side (a dedicated plugin/module in the future)

GitOrigin-RevId: 0704a20026b6c3dded409640018c35f8d69f1f92

java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt

@@ -21,7 +21,6 @@ import com.intellij.util.text.nullize
 import org.tukaani.xz.LZMA2Options
 import org.tukaani.xz.XZOutputStream
 import java.io.File
-import java.util.zip.ZipFile
 import kotlin.system.exitProcess
 
 class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
@@ -52,10 +51,10 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
     val jdkHome = args.argFile(jdkHomeKey, System.getProperty("java.home"))
     val tempDir = args.argFile(tempKey).recreateDir()
     val outputDir = args.argFile(outputKey).apply { mkdirs() }
-    val projectDir = File(tempDir, "project").recreateDir()
-    val zipsDir = File(tempDir, "zips").recreateDir()
-    val indexZip = File(zipsDir, "index.zip")
-    val indexZipXZ = File(zipsDir, "index.zip.xz")
+    val projectDir = (tempDir / "project").recreateDir()
+    val zipsDir = (tempDir / "zips").recreateDir()
+    val indexZip = zipsDir / "index.zip"
+    val indexZipXZ = zipsDir / "index.zip.xz"
 
     LOG.info("Resolved jdkHome = $jdkHome")
     LOG.info("Resolved outputDir = $outputDir")
@@ -116,12 +115,12 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
     val allRoots = (classesRoot + sourcesRoot).toSet()
 
     LOG.info("Collected ${allRoots.size} SDK roots...")
-    val indexChunk = IndexChunk(allRoots, "jdk-$jdkVersion-${os.osName}${nameHint?.let {"-$it"} ?: ""}")
-    indexChunk.contentsHash = hash
-    indexChunk.kind = "jdk"
+    val indexName = "jdk-$jdkVersion-${os.osName}${nameHint?.let { "-$it" } ?: ""}"
+    val indexKind = "jdk"
+    val indexChunk = IndexChunk(allRoots, indexName)
 
     LOG.info("Indexing...")
-    val indexerInfra = IndexesExporter.getInstance(project).exportIndexesChunk(indexChunk, indexZip.toPath())
+    val infraVersion = IndexesExporter.getInstance(project).exportIndexesChunk(indexChunk, indexZip.toPath())
 
     LOG.info("Packing the indexes to XZ...")
     xz(indexZip, indexZipXZ)
@@ -135,11 +134,7 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
     LOG.info("Generated index in $indexZip")
 
     val indexMetadata = runAndCatchNotNull("extract JSON metadata from $indexZip"){
-      ZipFile(indexZip).use { zipFile ->
-        val entry = zipFile.getEntry("metadata.json") ?: error("metadata.json is not found")
-        val data = zipFile.getInputStream(entry) ?: error("metadata.json is not found")
-        data.readBytes()
-      }
+      SharedIndexMetadata.writeIndexMetadata(indexName, indexKind, hash, infraVersion)
     }
 
     //we generate production layout here:
@@ -154,8 +149,8 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
     //       | <entry>.sha256 // hashcode of the entry
     //
 
-    val indexDir = File(File(outputDir, indexChunk.kind!!), indexChunk.contentsHash!!).apply { mkdirs() }
-    fun indexFile(nameSuffix: String) = File(indexDir, indexChunk.name + "-${indexerInfra.weakVersionHash}" + nameSuffix)
+    val indexDir = (outputDir / indexKind / hash).apply { mkdirs() }
+    fun indexFile(nameSuffix: String) = indexDir / "${indexChunk.name}-${infraVersion.weakVersionHash}$nameSuffix"
 
     FileUtil.copy(indexZipXZ, indexFile(".ijx"))
     FileUtil.writeToFile(indexFile(".json"), indexMetadata)
@@ -181,5 +176,7 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") {
       LOG.error("Failed to generate index.zip.xz package from $file to $output. ${e.message}", e)
     }
   }
+
+  private operator fun File.div(x: String) = File(this, x)
 }
 

java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt

@@ -0,0 +1,56 @@
+// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+package com.intellij.internal
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.databind.node.ObjectNode
+import com.intellij.openapi.application.ApplicationInfo
+import com.intellij.openapi.util.SystemInfo
+import com.intellij.util.indexing.IndexInfrastructureVersion
+
+object SharedIndexMetadata {
+  fun writeIndexMetadata(indexName: String,
+                         indexKind: String,
+                         sourcesHash: String,
+                         infrastructureVersion: IndexInfrastructureVersion): ByteArray {
+    try {
+      val om = ObjectMapper()
+
+      val root = om.createObjectNode()
+
+      root.put("metadata_version", "2")
+
+      root.putObject("sources").also { sources ->
+        sources.put("os", IndexInfrastructureVersion.getOs().osName)
+        sources.put("hash", sourcesHash)
+        sources.put("kind", indexKind)
+        sources.put("name", indexName)
+      }
+
+      root.putObject("sources").also { build ->
+        build.put("os", IndexInfrastructureVersion.getOs().osName)
+        build.put("os_name", SystemInfo.getOsNameAndVersion())
+        build.put("intellij_version", ApplicationInfo.getInstance().fullVersion)
+        build.put("intellij_build", ApplicationInfo.getInstance().build.toString())
+        build.put("intellij_product_code", ApplicationInfo.getInstance().build.productCode)
+      }
+
+      root.putObject("indexes").also { indexes ->
+        indexes.put("weak_hash", infrastructureVersion.weakVersionHash)
+        indexes.putObjectFromMap("base_versions", infrastructureVersion.baseIndexes)
+        indexes.putObjectFromMap("file_index_versions", infrastructureVersion.fileBasedIndexVersions)
+        indexes.putObjectFromMap("stub_index_versions", infrastructureVersion.stubIndexVersions)
+      }
+
+      return om.writerWithDefaultPrettyPrinter().writeValueAsBytes(root)
+      //NOTE: should we include information about index sizes here too?
+    } catch (t: Throwable) {
+      throw RuntimeException("Failed to generate shared index metadata JSON. ${t.message}", t)
+    }
+  }
+
+  private fun ObjectNode.putObjectFromMap(name: String, map: Map<String, String>) {
+    putObject(name).also { obj ->
+      map.toSortedMap().forEach { (k, v) -> obj.put(k, v) }
+    }
+  }
+}

platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexChunk.java

@@ -11,8 +11,6 @@ import java.util.Set;
 public final class IndexChunk {
   private final Set<VirtualFile> myRoots;
   private final String myName;
-  private String myKind;
-  private String myContentsHash;
 
   public IndexChunk(@NotNull Set<VirtualFile> roots,
                     @NotNull String name) {
@@ -30,24 +28,6 @@ public final class IndexChunk {
     return myRoots;
   }
 
-  @Nullable
-  public String getContentsHash() {
-    return myContentsHash;
-  }
-
-  public void setContentsHash(@Nullable String contentsHash) {
-    myContentsHash = contentsHash;
-  }
-
-  @Nullable
-  public String getKind() {
-    return myKind;
-  }
-
-  public void setKind(@Nullable String kind) {
-    myKind = kind;
-  }
-
   @Override
   public boolean equals(Object o) {
     if (this == o) return true;

platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexesExporter.java

@@ -1,16 +1,12 @@
 // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
 package com.intellij.util.indexing.hash.building;
 
-import com.google.gson.Gson;
-import com.google.gson.stream.JsonWriter;
 import com.intellij.concurrency.JobLauncher;
-import com.intellij.openapi.application.ApplicationInfo;
 import com.intellij.openapi.application.ReadAction;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.progress.EmptyProgressIndicator;
 import com.intellij.openapi.progress.ProgressIndicator;
 import com.intellij.openapi.project.Project;
-import com.intellij.openapi.util.SystemInfo;
 import com.intellij.openapi.vfs.VfsUtilCore;
 import com.intellij.openapi.vfs.VirtualFile;
 import com.intellij.openapi.vfs.VirtualFileVisitor;
@@ -30,10 +26,8 @@ import com.intellij.util.io.zip.JBZipEntry;
 import com.intellij.util.io.zip.JBZipFile;
 import gnu.trove.THashSet;
 import org.jetbrains.annotations.NotNull;
-import org.jetbrains.annotations.Nullable;
 
 import java.io.IOException;
-import java.io.StringWriter;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
@@ -120,87 +114,10 @@ public class IndexesExporter {
 
     IndexInfrastructureVersion indexInfrastructureVersion = IndexInfrastructureVersion.fromExtensions(exportableFileBasedIndexExtensions,
                                                                                                       exportableStubIndexExtensions);
-    Path metadataFile = chunkRoot.resolve("metadata.json");
-    writeIndexVersionsMetadata(metadataFile, chunk, indexInfrastructureVersion);
     printStatistics(chunk, fileBasedGenerators, stubGenerator);
-    //printMetadata(metadataFile);
     return indexInfrastructureVersion;
   }
 
-  private static void writeIndexVersionsMetadata(@NotNull Path metadataFile,
-                                                 @NotNull IndexChunk indexChunk,
-                                                 @NotNull IndexInfrastructureVersion infrastructureVersion) {
-
-    StringWriter sw = new StringWriter();
-    try(JsonWriter writer = new Gson().newBuilder().setPrettyPrinting().create().newJsonWriter(sw)) {
-      writer.beginObject();
-
-      writeNamedValue(writer, "metadata_version", "1");
-
-      writer.name("chunk");
-      writer.beginObject();
-      writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName());
-      writeNamedValue(writer, "kind", indexChunk.getKind());
-      writeNamedValue(writer, "name", indexChunk.getName());
-      writer.endObject();
-
-      writer.name("sources");
-      writer.beginObject();
-      writeNamedValue(writer, "hash", indexChunk.getContentsHash());
-      writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName());
-      writer.endObject();
-
-      writer.name("build");
-      writer.beginObject();
-      writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName());
-      writeNamedValue(writer, "os_name", SystemInfo.getOsNameAndVersion());
-      writeNamedValue(writer, "intellij_version", ApplicationInfo.getInstance().getFullVersion());
-      writeNamedValue(writer, "intellij_build", ApplicationInfo.getInstance().getBuild().toString());
-      writeNamedValue(writer, "intellij_product_code",ApplicationInfo.getInstance().getBuild().getProductCode());
-      writer.endObject();
-
-      writer.name("indexes");
-      writer.beginObject();
-      writeNamedValue(writer, "weak_hash", infrastructureVersion.getWeakVersionHash());
-      writer.name("versions");
-      writer.beginObject();
-      writeNamedMap(writer, "base_versions", infrastructureVersion.getBaseIndexes());
-      writeNamedMap(writer, "file_index_versions", infrastructureVersion.getFileBasedIndexVersions());
-      writeNamedMap(writer, "stub_index_versions", infrastructureVersion.getStubIndexVersions());
-      writer.endObject();
-      writer.endObject();
-
-      writer.endObject();
-    } catch (Exception e) {
-      throw new RuntimeException("Failed to generate versions JSON. " + e.getMessage(), e);
-    }
-
-    try {
-      PathKt.write(metadataFile, sw.toString());
-    } catch (IOException e) {
-      throw new RuntimeException("Failed to write versions JSON to " + metadataFile + ". " + e.getMessage(), e);
-    }
-  }
-
-  private static void writeNamedValue(@NotNull JsonWriter writer,
-                                      @NotNull String name,
-                                      @Nullable String value) throws IOException {
-    writer.name(name);
-    writer.value(value);
-  }
-
-  private static void writeNamedMap(@NotNull JsonWriter writer,
-                                    @NotNull String name,
-                                    @NotNull Map<String, String> map) throws IOException {
-    writer.name(name);
-    writer.beginObject();
-    for (Map.Entry<String, String> e : map.entrySet()) {
-      writer.name(e.getKey());
-      writer.value(e.getValue());
-    }
-    writer.endObject();
-  }
-
   @NotNull
   private static <K, V> HashBasedIndexGenerator<K, V> getGenerator(Path chunkRoot, FileBasedIndexExtension<K, V> extension) {
     return new HashBasedIndexGenerator<>(extension, chunkRoot);
@@ -327,15 +244,6 @@ public class IndexesExporter {
     }
   }
 
-  private static void printMetadata(@NotNull Path metadataFile) {
-    try {
-      String text = PathKt.readText(metadataFile);
-      LOG.warn(metadataFile.getFileName().toString() + ":\n" + text + "\n\n");
-    } catch (IOException e){
-      throw new RuntimeException("Failed to read " + metadataFile + ". " + e.getMessage(), e);
-    }
-  }
-
   private static void printStatistics(@NotNull IndexChunk chunk,
                                       @NotNull List<HashBasedIndexGenerator<?, ?>> fileBasedGenerators,
                                       @NotNull StubHashBasedIndexGenerator stubGenerator) {