From bde33431ba5a8be44d894af4aeaea08b80b69cb8 Mon Sep 17 00:00:00 2001 From: Dmitry Trofimov Date: Thu, 14 Sep 2017 18:36:34 +0200 Subject: [PATCH] Extract common code to reuse for JS and Go --- .../com/intellij/psi/stubs/PrebuiltStubs.kt | 135 ++++++++++++++++-- .../impl/stubs/PyPrebuiltStubsProvider.java | 134 +---------------- 2 files changed, 131 insertions(+), 138 deletions(-) diff --git a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt index f62877395c6d..1db3521abee7 100644 --- a/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt +++ b/platform/lang-impl/src/com/intellij/psi/stubs/PrebuiltStubs.kt @@ -15,17 +15,26 @@ */ package com.intellij.psi.stubs +import com.google.common.annotations.VisibleForTesting import com.google.common.hash.HashCode import com.google.common.hash.Hashing -import com.intellij.openapi.extensions.ExtensionPointName +import com.intellij.openapi.Disposable +import com.intellij.openapi.application.PathManager +import com.intellij.openapi.diagnostic.Logger import com.intellij.openapi.fileTypes.FileTypeExtension +import com.intellij.openapi.util.Disposer +import com.intellij.openapi.util.io.FileUtil import com.intellij.util.indexing.FileContent +import com.intellij.util.indexing.IndexInfrastructure import com.intellij.util.io.DataExternalizer import com.intellij.util.io.DataInputOutputUtil import com.intellij.util.io.KeyDescriptor +import com.intellij.util.io.PersistentHashMap import org.jetbrains.annotations.ApiStatus import java.io.DataInput import java.io.DataOutput +import java.io.File +import java.io.IOException /** * @author traff @@ -33,9 +42,7 @@ import java.io.DataOutput val EP_NAME = "com.intellij.filetype.prebuiltStubsProvider" -object PrebuiltStubsProviders : FileTypeExtension(EP_NAME) { - val EXTENSION_POINT_NAME = ExtensionPointName.create(EP_NAME) -} +object PrebuiltStubsProviders : FileTypeExtension(EP_NAME) @ApiStatus.Experimental interface PrebuiltStubsProvider { @@ -45,16 +52,13 @@ interface PrebuiltStubsProvider { class FileContentHashing { private val hashing = Hashing.sha256() - fun hashString(fileContent: FileContent): HashCode { - return hashing.hashBytes(fileContent.content) - } + fun hashString(fileContent: FileContent) = hashing.hashBytes(fileContent.content) } class HashCodeDescriptor : HashCodeExternalizers(), KeyDescriptor { override fun getHashCode(value: HashCode): Int = value.hashCode() - override fun isEqual(val1: HashCode, val2: HashCode): Boolean = val1 == val2 companion object { @@ -83,4 +87,119 @@ class StubTreeExternalizer : DataExternalizer { } override fun read(`in`: DataInput) = SerializedStubTree(`in`) +} + +abstract class PrebuiltStubsProviderBase : PrebuiltStubsProvider, Disposable { + private val myFileContentHashing = FileContentHashing() + private var myPrebuiltStubsStorage: PersistentHashMap? = null + private var mySerializationManager: SerializationManagerImpl? = null + + protected abstract val stubVersion: Int + + init { + init() + } + + companion object { + val PREBUILT_INDICES_PATH_PROPERTY = "prebuilt_indices_path" + val SDK_STUBS_STORAGE_NAME = "sdk-stubs" + private val LOG = Logger.getInstance("#com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProviderBase") + } + + @VisibleForTesting + internal fun init() { + var indexesRoot = findPrebuiltIndicesRoot() + try { + if (indexesRoot != null) { + // we should copy prebuilt indexes to a writable folder + indexesRoot = copyPrebuiltIndexesToIndexRoot(indexesRoot) + // otherwise we can get access denied error, because persistent hash map opens file for read and write + + val versionInFile = FileUtil.loadFile(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".version")) + + if (Integer.parseInt(versionInFile) == stubVersion) { + myPrebuiltStubsStorage = object : PersistentHashMap( + File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".input"), + HashCodeDescriptor.instance, + StubTreeExternalizer()) { + override fun isReadOnly(): Boolean { + return true + } + } + + mySerializationManager = SerializationManagerImpl(File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".names")) + + Disposer.register(this, mySerializationManager!!) + + LOG.info("Using prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.absolutePath) + } + else { + LOG.error("Prebuilt stubs version mismatch: $versionInFile, current version is $versionInFile") + } + } + } + catch (e: Exception) { + myPrebuiltStubsStorage = null + LOG.warn("Prebuilt stubs can't be loaded at " + indexesRoot!!, e) + } + + } + + override fun findStub(fileContent: FileContent): Stub? { + if (myPrebuiltStubsStorage != null) { + val hashCode = myFileContentHashing.hashString(fileContent) + var stub: Stub? = null + try { + val stubTree = myPrebuiltStubsStorage!!.get(hashCode) + if (stubTree != null) { + stub = stubTree.getStub(false, mySerializationManager!!) + } + } + catch (e: SerializerNotFoundException) { + LOG.error("Can't deserialize stub tree", e) + } + catch (e: Exception) { + LOG.error("Error reading prebuilt stubs from " + myPrebuiltStubsStorage!!.baseFile.path, e) + myPrebuiltStubsStorage = null + stub = null + } + + if (stub != null) { + return stub + } + } + return null + } + + override fun dispose() { + if (myPrebuiltStubsStorage != null) { + try { + myPrebuiltStubsStorage!!.close() + } + catch (e: IOException) { + LOG.error(e) + } + } + } + + @Throws(IOException::class) + private fun copyPrebuiltIndexesToIndexRoot(prebuiltIndicesRoot: File): File { + val indexRoot = File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt") + + FileUtil.copyDir(prebuiltIndicesRoot, indexRoot) + + return indexRoot + } + + private fun findPrebuiltIndicesRoot(): File? { + var path: String? = System.getProperty(PREBUILT_INDICES_PATH_PROPERTY) + if (path != null && File(path).exists()) { + return File(path) + } + path = PathManager.getHomePath() + var f = File(path, "python/index") // from sources + if (f.exists()) return f + f = File(path, "index") // compiled binary + return if (f.exists()) f else null + } } \ No newline at end of file diff --git a/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java index 778caf6c8fb3..077b9f0f13c9 100644 --- a/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java +++ b/python/src/com/jetbrains/python/psi/impl/stubs/PyPrebuiltStubsProvider.java @@ -15,141 +15,15 @@ */ package com.jetbrains.python.psi.impl.stubs; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.hash.HashCode; -import com.intellij.openapi.Disposable; -import com.intellij.openapi.application.PathManager; -import com.intellij.openapi.diagnostic.Logger; -import com.intellij.openapi.util.Disposer; -import com.intellij.openapi.util.io.FileUtil; -import com.intellij.psi.stubs.*; -import com.intellij.util.indexing.FileContent; -import com.intellij.util.indexing.IndexInfrastructure; -import com.intellij.util.io.PersistentHashMap; +import com.intellij.psi.stubs.PrebuiltStubsProviderBase; import com.jetbrains.python.psi.PyFileElementType; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -import java.io.File; -import java.io.IOException; /** * @author traff */ -public class PyPrebuiltStubsProvider implements PrebuiltStubsProvider, Disposable { - private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.psi.impl.stubs.PyPrebuiltStubsProvider"); - public static final String PREBUILT_INDICES_PATH_PROPERTY = "prebuilt_indices_path"; - - public static final String SDK_STUBS_STORAGE_NAME = "sdk-stubs"; - - private final FileContentHashing myFileContentHashing = new FileContentHashing(); - - private PersistentHashMap myPrebuiltStubsStorage; - private SerializationManagerImpl mySerializationManager; - - public PyPrebuiltStubsProvider() { - init(); - } - - @VisibleForTesting - void init() { - File indexesRoot = findPrebuiltIndicesRoot(); - try { - if (indexesRoot != null) { - // we should copy prebuilt indexes to a writable folder - indexesRoot = copyPrebuiltIndexesToIndexRoot(indexesRoot); - // otherwise we can get access denied error, because persistent hash map opens file for read and write - - String stubVersion = FileUtil.loadFile(new File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".version")); - - int currentVersion = PyFileElementType.INSTANCE.getStubVersion(); - if (Integer.parseInt(stubVersion) == currentVersion) { - myPrebuiltStubsStorage = - new PersistentHashMap(new File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".input"), - HashCodeDescriptor.Companion.getInstance(), - new StubTreeExternalizer()) { - @Override - protected boolean isReadOnly() { - return true; - } - }; - - mySerializationManager = new SerializationManagerImpl(new File(indexesRoot, SDK_STUBS_STORAGE_NAME + ".names")); - - Disposer.register(this, mySerializationManager); - - LOG.info("Using prebuilt stubs from " + myPrebuiltStubsStorage.getBaseFile().getAbsolutePath()); - } - else { - LOG.error("Prebuilt stubs version mismatch: " + stubVersion + ", current version is " + currentVersion); - } - } - } - catch (Exception e) { - myPrebuiltStubsStorage = null; - LOG.warn("Prebuilt stubs can't be loaded at " + indexesRoot, e); - } - } - - private static File copyPrebuiltIndexesToIndexRoot(File prebuiltIndicesRoot) throws IOException { - File indexRoot = new File(IndexInfrastructure.getPersistentIndexRoot(), "prebuilt"); - - FileUtil.copyDir(prebuiltIndicesRoot, indexRoot); - - return indexRoot; - } - - - @Nullable +public class PyPrebuiltStubsProvider extends PrebuiltStubsProviderBase { @Override - public Stub findStub(@NotNull FileContent fileContent) { - if (myPrebuiltStubsStorage != null) { - HashCode hashCode = myFileContentHashing.hashString(fileContent); - Stub stub = null; - try { - SerializedStubTree stubTree = myPrebuiltStubsStorage.get(hashCode); - if (stubTree != null) { - stub = stubTree.getStub(false, mySerializationManager); - } - } - catch (SerializerNotFoundException e) { - LOG.error("Can't deserialize stub tree", e); - } - catch (Exception e) { - LOG.error("Error reading prebuilt stubs from " + myPrebuiltStubsStorage.getBaseFile().getPath(), e); - myPrebuiltStubsStorage = null; - stub = null; - } - if (stub != null) { - return stub; - } - } - return null; - } - - @Nullable - private static File findPrebuiltIndicesRoot() { - String path = System.getProperty(PREBUILT_INDICES_PATH_PROPERTY); - if (path != null && new File(path).exists()) { - return new File(path); - } - path = PathManager.getHomePath(); - File f = new File(path, "python/index"); // from sources - if (f.exists()) return f; - f = new File(path, "index"); // compiled binary - if (f.exists()) return f; - return null; - } - - @Override - public void dispose() { - if (myPrebuiltStubsStorage != null) { - try { - myPrebuiltStubsStorage.close(); - } - catch (IOException e) { - LOG.error(e); - } - } + protected int getStubVersion() { + return PyFileElementType.INSTANCE.getStubVersion(); } }