From 60982e9081655c359fbb3ae610309ff458563c67 Mon Sep 17 00:00:00 2001 From: Eugene Petrenko Date: Wed, 29 Jan 2020 14:15:32 +0100 Subject: [PATCH] move shared index metadata generation to the DumpJdkIndexStarter.kt side (a dedicated plugin/module in the future) GitOrigin-RevId: 0704a20026b6c3dded409640018c35f8d69f1f92 --- .../intellij/internal/DumpJdkIndexStarter.kt | 29 +++--- .../intellij/internal/SharedIndexMetadata.kt | 56 +++++++++++ .../indexing/hash/building/IndexChunk.java | 20 ---- .../hash/building/IndexesExporter.java | 92 ------------------- 4 files changed, 69 insertions(+), 128 deletions(-) create mode 100644 java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt diff --git a/java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt b/java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt index 6c13d937e0cf..d806f523ee5e 100644 --- a/java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt +++ b/java/idea-ui/src/com/intellij/internal/DumpJdkIndexStarter.kt @@ -21,7 +21,6 @@ import com.intellij.util.text.nullize import org.tukaani.xz.LZMA2Options import org.tukaani.xz.XZOutputStream import java.io.File -import java.util.zip.ZipFile import kotlin.system.exitProcess class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { @@ -52,10 +51,10 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { val jdkHome = args.argFile(jdkHomeKey, System.getProperty("java.home")) val tempDir = args.argFile(tempKey).recreateDir() val outputDir = args.argFile(outputKey).apply { mkdirs() } - val projectDir = File(tempDir, "project").recreateDir() - val zipsDir = File(tempDir, "zips").recreateDir() - val indexZip = File(zipsDir, "index.zip") - val indexZipXZ = File(zipsDir, "index.zip.xz") + val projectDir = (tempDir / "project").recreateDir() + val zipsDir = (tempDir / "zips").recreateDir() + val indexZip = zipsDir / "index.zip" + val indexZipXZ = zipsDir / "index.zip.xz" LOG.info("Resolved jdkHome = $jdkHome") LOG.info("Resolved outputDir = $outputDir") @@ -116,12 +115,12 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { val allRoots = (classesRoot + sourcesRoot).toSet() LOG.info("Collected ${allRoots.size} SDK roots...") - val indexChunk = IndexChunk(allRoots, "jdk-$jdkVersion-${os.osName}${nameHint?.let {"-$it"} ?: ""}") - indexChunk.contentsHash = hash - indexChunk.kind = "jdk" + val indexName = "jdk-$jdkVersion-${os.osName}${nameHint?.let { "-$it" } ?: ""}" + val indexKind = "jdk" + val indexChunk = IndexChunk(allRoots, indexName) LOG.info("Indexing...") - val indexerInfra = IndexesExporter.getInstance(project).exportIndexesChunk(indexChunk, indexZip.toPath()) + val infraVersion = IndexesExporter.getInstance(project).exportIndexesChunk(indexChunk, indexZip.toPath()) LOG.info("Packing the indexes to XZ...") xz(indexZip, indexZipXZ) @@ -135,11 +134,7 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { LOG.info("Generated index in $indexZip") val indexMetadata = runAndCatchNotNull("extract JSON metadata from $indexZip"){ - ZipFile(indexZip).use { zipFile -> - val entry = zipFile.getEntry("metadata.json") ?: error("metadata.json is not found") - val data = zipFile.getInputStream(entry) ?: error("metadata.json is not found") - data.readBytes() - } + SharedIndexMetadata.writeIndexMetadata(indexName, indexKind, hash, infraVersion) } //we generate production layout here: @@ -154,8 +149,8 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { // | .sha256 // hashcode of the entry // - val indexDir = File(File(outputDir, indexChunk.kind!!), indexChunk.contentsHash!!).apply { mkdirs() } - fun indexFile(nameSuffix: String) = File(indexDir, indexChunk.name + "-${indexerInfra.weakVersionHash}" + nameSuffix) + val indexDir = (outputDir / indexKind / hash).apply { mkdirs() } + fun indexFile(nameSuffix: String) = indexDir / "${indexChunk.name}-${infraVersion.weakVersionHash}$nameSuffix" FileUtil.copy(indexZipXZ, indexFile(".ijx")) FileUtil.writeToFile(indexFile(".json"), indexMetadata) @@ -181,5 +176,7 @@ class DumpJdkIndexStarter : IndexesStarterBase("dump-jdk-index") { LOG.error("Failed to generate index.zip.xz package from $file to $output. ${e.message}", e) } } + + private operator fun File.div(x: String) = File(this, x) } diff --git a/java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt b/java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt new file mode 100644 index 000000000000..5f3b50144227 --- /dev/null +++ b/java/idea-ui/src/com/intellij/internal/SharedIndexMetadata.kt @@ -0,0 +1,56 @@ +// Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.intellij.internal + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import com.intellij.openapi.application.ApplicationInfo +import com.intellij.openapi.util.SystemInfo +import com.intellij.util.indexing.IndexInfrastructureVersion + +object SharedIndexMetadata { + fun writeIndexMetadata(indexName: String, + indexKind: String, + sourcesHash: String, + infrastructureVersion: IndexInfrastructureVersion): ByteArray { + try { + val om = ObjectMapper() + + val root = om.createObjectNode() + + root.put("metadata_version", "2") + + root.putObject("sources").also { sources -> + sources.put("os", IndexInfrastructureVersion.getOs().osName) + sources.put("hash", sourcesHash) + sources.put("kind", indexKind) + sources.put("name", indexName) + } + + root.putObject("sources").also { build -> + build.put("os", IndexInfrastructureVersion.getOs().osName) + build.put("os_name", SystemInfo.getOsNameAndVersion()) + build.put("intellij_version", ApplicationInfo.getInstance().fullVersion) + build.put("intellij_build", ApplicationInfo.getInstance().build.toString()) + build.put("intellij_product_code", ApplicationInfo.getInstance().build.productCode) + } + + root.putObject("indexes").also { indexes -> + indexes.put("weak_hash", infrastructureVersion.weakVersionHash) + indexes.putObjectFromMap("base_versions", infrastructureVersion.baseIndexes) + indexes.putObjectFromMap("file_index_versions", infrastructureVersion.fileBasedIndexVersions) + indexes.putObjectFromMap("stub_index_versions", infrastructureVersion.stubIndexVersions) + } + + return om.writerWithDefaultPrettyPrinter().writeValueAsBytes(root) + //NOTE: should we include information about index sizes here too? + } catch (t: Throwable) { + throw RuntimeException("Failed to generate shared index metadata JSON. ${t.message}", t) + } + } + + private fun ObjectNode.putObjectFromMap(name: String, map: Map) { + putObject(name).also { obj -> + map.toSortedMap().forEach { (k, v) -> obj.put(k, v) } + } + } +} diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexChunk.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexChunk.java index 90db2b386eba..d87e8e759c7a 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexChunk.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexChunk.java @@ -11,8 +11,6 @@ import java.util.Set; public final class IndexChunk { private final Set myRoots; private final String myName; - private String myKind; - private String myContentsHash; public IndexChunk(@NotNull Set roots, @NotNull String name) { @@ -30,24 +28,6 @@ public final class IndexChunk { return myRoots; } - @Nullable - public String getContentsHash() { - return myContentsHash; - } - - public void setContentsHash(@Nullable String contentsHash) { - myContentsHash = contentsHash; - } - - @Nullable - public String getKind() { - return myKind; - } - - public void setKind(@Nullable String kind) { - myKind = kind; - } - @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexesExporter.java b/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexesExporter.java index e5770437100a..686b012d054f 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexesExporter.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/hash/building/IndexesExporter.java @@ -1,16 +1,12 @@ // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package com.intellij.util.indexing.hash.building; -import com.google.gson.Gson; -import com.google.gson.stream.JsonWriter; import com.intellij.concurrency.JobLauncher; -import com.intellij.openapi.application.ApplicationInfo; import com.intellij.openapi.application.ReadAction; import com.intellij.openapi.diagnostic.Logger; import com.intellij.openapi.progress.EmptyProgressIndicator; import com.intellij.openapi.progress.ProgressIndicator; import com.intellij.openapi.project.Project; -import com.intellij.openapi.util.SystemInfo; import com.intellij.openapi.vfs.VfsUtilCore; import com.intellij.openapi.vfs.VirtualFile; import com.intellij.openapi.vfs.VirtualFileVisitor; @@ -30,10 +26,8 @@ import com.intellij.util.io.zip.JBZipEntry; import com.intellij.util.io.zip.JBZipFile; import gnu.trove.THashSet; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import java.io.IOException; -import java.io.StringWriter; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; @@ -120,87 +114,10 @@ public class IndexesExporter { IndexInfrastructureVersion indexInfrastructureVersion = IndexInfrastructureVersion.fromExtensions(exportableFileBasedIndexExtensions, exportableStubIndexExtensions); - Path metadataFile = chunkRoot.resolve("metadata.json"); - writeIndexVersionsMetadata(metadataFile, chunk, indexInfrastructureVersion); printStatistics(chunk, fileBasedGenerators, stubGenerator); - //printMetadata(metadataFile); return indexInfrastructureVersion; } - private static void writeIndexVersionsMetadata(@NotNull Path metadataFile, - @NotNull IndexChunk indexChunk, - @NotNull IndexInfrastructureVersion infrastructureVersion) { - - StringWriter sw = new StringWriter(); - try(JsonWriter writer = new Gson().newBuilder().setPrettyPrinting().create().newJsonWriter(sw)) { - writer.beginObject(); - - writeNamedValue(writer, "metadata_version", "1"); - - writer.name("chunk"); - writer.beginObject(); - writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName()); - writeNamedValue(writer, "kind", indexChunk.getKind()); - writeNamedValue(writer, "name", indexChunk.getName()); - writer.endObject(); - - writer.name("sources"); - writer.beginObject(); - writeNamedValue(writer, "hash", indexChunk.getContentsHash()); - writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName()); - writer.endObject(); - - writer.name("build"); - writer.beginObject(); - writeNamedValue(writer, "os", IndexInfrastructureVersion.getOs().getOsName()); - writeNamedValue(writer, "os_name", SystemInfo.getOsNameAndVersion()); - writeNamedValue(writer, "intellij_version", ApplicationInfo.getInstance().getFullVersion()); - writeNamedValue(writer, "intellij_build", ApplicationInfo.getInstance().getBuild().toString()); - writeNamedValue(writer, "intellij_product_code",ApplicationInfo.getInstance().getBuild().getProductCode()); - writer.endObject(); - - writer.name("indexes"); - writer.beginObject(); - writeNamedValue(writer, "weak_hash", infrastructureVersion.getWeakVersionHash()); - writer.name("versions"); - writer.beginObject(); - writeNamedMap(writer, "base_versions", infrastructureVersion.getBaseIndexes()); - writeNamedMap(writer, "file_index_versions", infrastructureVersion.getFileBasedIndexVersions()); - writeNamedMap(writer, "stub_index_versions", infrastructureVersion.getStubIndexVersions()); - writer.endObject(); - writer.endObject(); - - writer.endObject(); - } catch (Exception e) { - throw new RuntimeException("Failed to generate versions JSON. " + e.getMessage(), e); - } - - try { - PathKt.write(metadataFile, sw.toString()); - } catch (IOException e) { - throw new RuntimeException("Failed to write versions JSON to " + metadataFile + ". " + e.getMessage(), e); - } - } - - private static void writeNamedValue(@NotNull JsonWriter writer, - @NotNull String name, - @Nullable String value) throws IOException { - writer.name(name); - writer.value(value); - } - - private static void writeNamedMap(@NotNull JsonWriter writer, - @NotNull String name, - @NotNull Map map) throws IOException { - writer.name(name); - writer.beginObject(); - for (Map.Entry e : map.entrySet()) { - writer.name(e.getKey()); - writer.value(e.getValue()); - } - writer.endObject(); - } - @NotNull private static HashBasedIndexGenerator getGenerator(Path chunkRoot, FileBasedIndexExtension extension) { return new HashBasedIndexGenerator<>(extension, chunkRoot); @@ -327,15 +244,6 @@ public class IndexesExporter { } } - private static void printMetadata(@NotNull Path metadataFile) { - try { - String text = PathKt.readText(metadataFile); - LOG.warn(metadataFile.getFileName().toString() + ":\n" + text + "\n\n"); - } catch (IOException e){ - throw new RuntimeException("Failed to read " + metadataFile + ". " + e.getMessage(), e); - } - } - private static void printStatistics(@NotNull IndexChunk chunk, @NotNull List> fileBasedGenerators, @NotNull StubHashBasedIndexGenerator stubGenerator) {