From 87f388f7fe4d3e38bd2fc1014107fe052599bd02 Mon Sep 17 00:00:00 2001 From: Bogdan Kirilenko Date: Thu, 21 Mar 2024 14:26:25 +0100 Subject: [PATCH] [pycharm] PY-63670 - fixes according to IJ-CR-125957, part 1 - without touching async and json parsing part GitOrigin-RevId: acd02edbe98a47c0147797edd2a2131e1e67074e --- .../icons => huggingFaceIcons}/download.svg | 0 .../icons => huggingFaceIcons}/like.svg | 0 .../icons => huggingFaceIcons}/logo.svg | 0 ...llij.python.community.impl.huggingFace.xml | 4 +- .../messages/PyHuggingFaceBundle.properties | 9 ++- .../impl/huggingFace/HuggingFaceConstants.kt | 4 +- .../impl/huggingFace/HuggingFaceEntityKind.kt | 3 + .../impl/huggingFace/HuggingFaceUtil.kt | 52 +++++++++------- .../HuggingFaceEntityIdentitiers.kt | 6 ++ .../HuggingFaceEntityNameAnnotator.kt | 12 ++-- ....kt => HuggingFaceIdentifierPsiElement.kt} | 42 ++++++------- .../annotation/HuggingFaceReference.kt | 21 +++---- .../HuggingFaceReferenceContributor.kt | 17 ++---- .../HuggingFaceReferenceProvider.kt | 32 +++------- .../impl/huggingFace/api/HuggingFaceApi.kt | 39 +++++++++++- .../api/HuggingFaceEntityBasicApiData.kt | 35 +++-------- .../huggingFace/api/HuggingFaceHttpClient.kt | 5 +- .../api/HuggingFaceModelSortKey.kt | 13 ++++ .../huggingFace/api/HuggingFaceURLProvider.kt | 32 ++++------ .../cache/HuggingFaceEntitiesCache.kt | 19 +++--- .../cache/HuggingFaceLastCacheRefresh.kt | 6 +- .../cache/HuggingFaceMdCardsCache.kt | 4 +- ...uggingFaceDocumentationPlaceholdersUtil.kt | 14 ++--- .../HuggingFaceDocumentationTarget.kt | 59 +++---------------- .../HuggingFaceDocumentationTargetProvider.kt | 29 +++++---- .../documentation/HuggingFaceHtmlBuilder.kt | 24 ++++---- .../HuggingFaceMarkdownToHtmlConverter.kt | 36 ++++++----- .../HuggingFaceQuickDocStyles.kt | 11 ++-- .../documentation/HuggingFaceReadmeCleaner.kt | 34 +---------- .../PythonCommunityImplHuggingFaceIcons.java | 20 ------- .../service/HuggingFaceExecutorService.kt | 4 +- .../HuggingFaceImportedLibrariesManager.kt | 23 ++------ .../HuggingFaceTypoInspectionSuppressor.kt | 2 +- 33 files changed, 263 insertions(+), 348 deletions(-) rename python/huggingFace/resources/{com/intellij/python/community/impl/huggingFace/icons => huggingFaceIcons}/download.svg (100%) rename python/huggingFace/resources/{com/intellij/python/community/impl/huggingFace/icons => huggingFaceIcons}/like.svg (100%) rename python/huggingFace/resources/{com/intellij/python/community/impl/huggingFace/icons => huggingFaceIcons}/logo.svg (100%) create mode 100644 python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityIdentitiers.kt rename python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/{HuggingFacePsiElements.kt => HuggingFaceIdentifierPsiElement.kt} (55%) create mode 100644 python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceModelSortKey.kt delete mode 100644 python/huggingFace/src/com/intellij/python/community/impl/huggingFace/icons/PythonCommunityImplHuggingFaceIcons.java diff --git a/python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/download.svg b/python/huggingFace/resources/huggingFaceIcons/download.svg similarity index 100% rename from python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/download.svg rename to python/huggingFace/resources/huggingFaceIcons/download.svg diff --git a/python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/like.svg b/python/huggingFace/resources/huggingFaceIcons/like.svg similarity index 100% rename from python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/like.svg rename to python/huggingFace/resources/huggingFaceIcons/like.svg diff --git a/python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/logo.svg b/python/huggingFace/resources/huggingFaceIcons/logo.svg similarity index 100% rename from python/huggingFace/resources/com/intellij/python/community/impl/huggingFace/icons/logo.svg rename to python/huggingFace/resources/huggingFaceIcons/logo.svg diff --git a/python/huggingFace/resources/intellij.python.community.impl.huggingFace.xml b/python/huggingFace/resources/intellij.python.community.impl.huggingFace.xml index 0e77e54f92b0..80ca2b7aae48 100644 --- a/python/huggingFace/resources/intellij.python.community.impl.huggingFace.xml +++ b/python/huggingFace/resources/intellij.python.community.impl.huggingFace.xml @@ -4,9 +4,7 @@ - - diff --git a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties index c8883017324a..b3ec8f09c40d 100644 --- a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties +++ b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties @@ -4,14 +4,13 @@ python.hugging.face.default.value=Default value python.hugging.face.unknown.element=Unknown element python.hugging.face.not.found=Error fetching model data: {0} python.hugging.face.base.url=https://huggingface.co -python.hugging.face.documentation.hint=Open in browser -updated.0= Updated {0} -python.hugging.face.html.head="Documentation for {0}" -open.on.hugging.face=Open on Hugging Face -python.hugging.face.dictionary.name=Hugging Face model and dataset dictionary +python.hugging.face.updated.suffix= Updated {0} +python.hugging.face.open.on.link.text=Open on Hugging Face + python.hugging.face.placeholder.gated.model=Sorry, this {0} is gated.
Please visit the [HuggingFace website]({1})
for more details about the {2} card. python.hugging.face.placeholder.no.readme=This model has no description available. For more details, please follow the [link]({0}) python.hugging.face.placeholder.trimmed=*Please find the rest of the {0} card [here]({1})* python.hugging.face.placeholder.no.internet=Failed to fetch data for {0}. Please check your internet connection. python.hugging.face.dataset=Dataset +python.hugging.face.tags.style= diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceConstants.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceConstants.kt index 467f6691230c..eebf680c724b 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceConstants.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceConstants.kt @@ -1,12 +1,14 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.impl.huggingFace +import org.jetbrains.annotations.ApiStatus + +@ApiStatus.Internal object HuggingFaceConstants { const val HF_API_FETCH_PAGE_SIZE = 500 const val MAX_MODELS_IN_CACHE = HF_API_FETCH_PAGE_SIZE * 20 const val MAX_DATASETS_IN_CACHE = HF_API_FETCH_PAGE_SIZE * 20 const val HF_EMOJI = "\uD83E\uDD17" - const val MAX_MD_CHAR_NUM = 5000 const val API_FETCH_SORT_KEY = "likes" const val DATASET_FAKE_PIPELINE_TAG = "dataset" const val UNDEFINED_PIPELINE_TAG = "undefined" diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceEntityKind.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceEntityKind.kt index 24fdf7b60250..7dfa4ca0782f 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceEntityKind.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceEntityKind.kt @@ -1,6 +1,9 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.impl.huggingFace +import org.jetbrains.annotations.ApiStatus + +@ApiStatus.Internal enum class HuggingFaceEntityKind(val urlFragment: String, val printName: String) { MODEL("models", "model"), DATASET("datasets", "dataset") diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceUtil.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceUtil.kt index 16513a678aa2..2830b8930368 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceUtil.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/HuggingFaceUtil.kt @@ -6,8 +6,7 @@ import com.intellij.openapi.roots.ProjectFileIndex import com.intellij.openapi.util.NlsSafe import com.intellij.psi.PsiElement import com.intellij.psi.PsiManager -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceDatasetPsiElement -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceModelPsiElement +import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceIdentifierPsiElement import com.intellij.python.community.impl.huggingFace.api.HuggingFaceApi import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceDatasetsCache import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceLastCacheRefresh @@ -15,16 +14,18 @@ import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceModelsCac import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle import com.jetbrains.python.psi.* import com.jetbrains.python.psi.impl.PyPlainStringElementImpl +import org.jetbrains.annotations.ApiStatus import java.time.Duration -val huggingFaceRelevantLibraries = setOf( - "diffusers", "transformers", "allennlp", "spacy", - "asteroid", "flair", "keras", "sentence-transformers", - "stable-baselines3", "adapters", "huggingface_hub", -) - +@ApiStatus.Internal object HuggingFaceUtil { + private val huggingFaceRelevantLibraries = setOf( + "diffusers", "transformers", "allennlp", "spacy", + "asteroid", "flair", "keras", "sentence-transformers", + "stable-baselines3", "adapters", "huggingface_hub", + ) + private fun isAnyHFLibraryImportedInFile(file: PyFile): Boolean { val isDirectlyImported = file.importTargets.any { importStmt -> huggingFaceRelevantLibraries.any { lib -> importStmt.importedQName.toString().contains(lib) } @@ -56,7 +57,7 @@ object HuggingFaceUtil { return isLibraryImported } - private fun isWhatHuggingFaceEntity(text: String): HuggingFaceEntityKind? { + fun isWhatHuggingFaceEntity(text: String): HuggingFaceEntityKind? { return when { isHuggingFaceModel(text) -> HuggingFaceEntityKind.MODEL isHuggingFaceDataset(text) -> HuggingFaceEntityKind.DATASET @@ -64,9 +65,9 @@ object HuggingFaceUtil { } } - fun isHuggingFaceModel(text: String): Boolean = HuggingFaceModelsCache.isInCache(text) + private fun isHuggingFaceModel(text: String): Boolean = HuggingFaceModelsCache.isInCache(text) - fun isHuggingFaceDataset(text: String): Boolean = HuggingFaceDatasetsCache.isInCache(text) + private fun isHuggingFaceDataset(text: String): Boolean = HuggingFaceDatasetsCache.isInCache(text) fun isHuggingFaceEntity(text: String): Boolean = isHuggingFaceModel(text) || isHuggingFaceDataset(text) @@ -81,35 +82,34 @@ object HuggingFaceUtil { } } - private fun extractStringAndKindFromStringLiteralExpression(stringValue: String?): Pair { + private fun extractStringAndKindFromStringLiteralExpression(stringValue: String?): Pair? { val entityKind = if (stringValue != null) isWhatHuggingFaceEntity(stringValue) else null - return Pair(stringValue, entityKind) + return stringValue?.takeIf { entityKind != null }?.let { Pair(it, entityKind!!) } } - fun extractStringValueAndEntityKindFromElement(element: PsiElement): Pair { + fun extractStringValueAndEntityKindFromElement(element: PsiElement): Pair? { return when (val parent = element.parent) { is PyAssignmentStatement -> when (element) { is PyTargetExpression -> { val stringValue = (parent.assignedValue as? PyStringLiteralExpression)?.stringValue extractStringAndKindFromStringLiteralExpression(stringValue) } - else -> Pair(null, null) + else -> null } is PyStringLiteralExpression -> when (element) { - is HuggingFaceModelPsiElement -> Pair(parent.stringValue, HuggingFaceEntityKind.MODEL) - is HuggingFaceDatasetPsiElement -> Pair(parent.stringValue, HuggingFaceEntityKind.DATASET) + is HuggingFaceIdentifierPsiElement -> Pair(element.entityName, element.entityKind) is PyPlainStringElementImpl -> { extractStringAndKindFromStringLiteralExpression(parent.stringValue) } - else -> Pair(null, null) + else -> null } is PyArgumentList -> when (element) { is PyStringLiteralExpression -> { extractStringAndKindFromStringLiteralExpression(element.stringValue) } - else -> Pair(null, null) + else -> null } - else -> Pair(null, null) + else -> null } } @@ -127,4 +127,16 @@ object HuggingFaceUtil { HuggingFaceDatasetsCache, HuggingFaceConstants.MAX_DATASETS_IN_CACHE) } + + fun humanReadableNumber(rawNumber: Int): String { + if (rawNumber < 1000) return rawNumber.toString() + val suffixes = arrayOf("K", "M", "B", "T") + var count = rawNumber.toDouble() + var i = 0 + while (i < suffixes.size && count >= 1000) { + count /= 1000 + i++ + } + return String.format("%.1f%s", count, suffixes[i - 1]) + } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityIdentitiers.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityIdentitiers.kt new file mode 100644 index 000000000000..a3f626d428e2 --- /dev/null +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityIdentitiers.kt @@ -0,0 +1,6 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.impl.huggingFace.annotation + +@JvmInline value class HuggingFaceModelId(val id: String) +@JvmInline value class HuggingFaceDatasetId(val id: String) +@JvmInline value class HuggingFaceEntityId(val id: String) diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityNameAnnotator.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityNameAnnotator.kt index 1578281ce742..1d72de2c58c0 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityNameAnnotator.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceEntityNameAnnotator.kt @@ -8,17 +8,18 @@ import com.intellij.openapi.editor.colors.CodeInsightColors import com.intellij.openapi.util.TextRange import com.intellij.psi.PsiElement import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceEntityPsiElement.Companion.HUGGING_FACE_ENTITY_NAME_KEY -import com.intellij.python.community.impl.huggingFace.service.HuggingFaceImportedLibrariesManagerService +import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceIdentifierPsiElement.Companion.HUGGING_FACE_ENTITY_NAME_KEY +import com.intellij.python.community.impl.huggingFace.service.HuggingFaceImportedLibrariesManager import com.jetbrains.python.psi.PyStringLiteralExpression +import org.jetbrains.annotations.ApiStatus +@ApiStatus.Internal class HuggingFaceEntityNameAnnotator : Annotator { override fun annotate(element: PsiElement, holder: AnnotationHolder) { if (element !is PyStringLiteralExpression) return val project = element.project - val service = project.getService(HuggingFaceImportedLibrariesManagerService::class.java) - val manager = service.getManager() + val manager = project.getService(HuggingFaceImportedLibrariesManager::class.java) if (!manager.isLibraryImported()) return val text = element.stringValue @@ -36,7 +37,6 @@ class HuggingFaceEntityNameAnnotator : Annotator { } private fun isValidHfString(element: PyStringLiteralExpression, text: String): Boolean { - val userData = element.getUserData(HUGGING_FACE_ENTITY_NAME_KEY) - return (userData == true) || HuggingFaceUtil.isHuggingFaceEntity(text) + return (element.getUserData(HUGGING_FACE_ENTITY_NAME_KEY) == true) || HuggingFaceUtil.isHuggingFaceEntity(text) } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFacePsiElements.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceIdentifierPsiElement.kt similarity index 55% rename from python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFacePsiElements.kt rename to python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceIdentifierPsiElement.kt index 2879b698a2ae..468a9ecc8c5c 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFacePsiElements.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceIdentifierPsiElement.kt @@ -7,15 +7,19 @@ import com.intellij.openapi.util.NlsSafe import com.intellij.psi.PsiElement import com.intellij.psi.impl.FakePsiElement import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants +import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceModelsCache import com.intellij.python.community.impl.huggingFace.service.HuggingFaceCardsUsageCollector +import org.jetbrains.annotations.ApiStatus - -abstract class HuggingFaceEntityPsiElement( +@ApiStatus.Internal +class HuggingFaceIdentifierPsiElement( private val parent: PsiElement, - private val entityName: String + val entityName: String, + val entityKind: HuggingFaceEntityKind ) : FakePsiElement() { + init { parent.putUserData(HUGGING_FACE_ENTITY_NAME_KEY, true) } @@ -25,24 +29,22 @@ abstract class HuggingFaceEntityPsiElement( @NlsSafe fun stringValue(): String = entityName + override fun navigate(requestFocus: Boolean) { + when (entityKind) { + HuggingFaceEntityKind.MODEL -> { + val pipelineTag = HuggingFaceModelsCache.getPipelineTagForEntity(entityName) ?: + HuggingFaceConstants.UNDEFINED_PIPELINE_TAG + HuggingFaceCardsUsageCollector.NAVIGATION_LINK_IN_EDITOR_CLICKED.log(pipelineTag) + BrowserUtil.browse(HuggingFaceURLProvider.getModelCardLink(entityName)) + } + HuggingFaceEntityKind.DATASET -> { + HuggingFaceCardsUsageCollector.NAVIGATION_LINK_IN_EDITOR_CLICKED.log(HuggingFaceConstants.DATASET_FAKE_PIPELINE_TAG) + BrowserUtil.browse(HuggingFaceURLProvider.getDatasetCardLink(entityName)) + } + } + } + companion object { val HUGGING_FACE_ENTITY_NAME_KEY = Key.create("HUGGING_FACE_ENTITY_NAME_KEY") } } - -class HuggingFaceModelPsiElement(parent: PsiElement, private val modelName: String -) : HuggingFaceEntityPsiElement(parent, modelName) { - override fun navigate(requestFocus: Boolean) { - val pipelineTag = HuggingFaceModelsCache.getPipelineTagForEntity(modelName) ?: HuggingFaceConstants.UNDEFINED_PIPELINE_TAG - HuggingFaceCardsUsageCollector.NAVIGATION_LINK_IN_EDITOR_CLICKED.log(pipelineTag) - BrowserUtil.browse(HuggingFaceURLProvider.getModelCardLink(modelName)) - } -} - -class HuggingFaceDatasetPsiElement(parent: PsiElement, private val datasetName: String -) : HuggingFaceEntityPsiElement(parent, datasetName) { - override fun navigate(requestFocus: Boolean) { - HuggingFaceCardsUsageCollector.NAVIGATION_LINK_IN_EDITOR_CLICKED.log(HuggingFaceConstants.DATASET_FAKE_PIPELINE_TAG) - BrowserUtil.browse(HuggingFaceURLProvider.getDatasetCardLink(datasetName)) - } -} diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReference.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReference.kt index 721364911651..ef690678d3f9 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReference.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReference.kt @@ -4,23 +4,18 @@ package com.intellij.python.community.impl.huggingFace.annotation import com.intellij.openapi.util.TextRange import com.intellij.psi.PsiElement import com.intellij.psi.PsiReferenceBase +import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind +import org.jetbrains.annotations.ApiStatus -class HuggingFaceModelReference( + +@ApiStatus.Internal +class HuggingFaceReference( element: PsiElement, textRange: TextRange, - private val modelName: String + private val identifier: String, + private val entityKind: HuggingFaceEntityKind, ) : PsiReferenceBase(element, textRange) { override fun resolve(): PsiElement { - return HuggingFaceModelPsiElement(myElement, modelName) - } -} - -class HuggingFaceDatasetReference( - element: PsiElement, - textRange: TextRange, - private val datasetName: String -) : PsiReferenceBase(element, textRange) { - override fun resolve(): PsiElement { - return HuggingFaceDatasetPsiElement(myElement, datasetName) + return HuggingFaceIdentifierPsiElement(myElement, identifier, entityKind) } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceContributor.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceContributor.kt index 84d3d664b612..5c8c7306556f 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceContributor.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceContributor.kt @@ -5,23 +5,14 @@ import com.intellij.patterns.PlatformPatterns import com.intellij.psi.PsiReferenceContributor import com.intellij.psi.PsiReferenceRegistrar import com.jetbrains.python.PyElementTypes +import org.jetbrains.annotations.ApiStatus - -class HuggingFaceModelReferenceContributor: PsiReferenceContributor() { +@ApiStatus.Internal +class HuggingFaceIdentifierReferenceContributor: PsiReferenceContributor() { override fun registerReferenceProviders(registrar: PsiReferenceRegistrar) { registrar.registerReferenceProvider( PlatformPatterns.psiElement(PyElementTypes.STRING_LITERAL_EXPRESSION), - HuggingFaceModelReferenceProvider(), - PsiReferenceRegistrar.DEFAULT_PRIORITY - ) - } -} - -class HuggingFaceDatasetReferenceContributor: PsiReferenceContributor() { - override fun registerReferenceProviders(registrar: PsiReferenceRegistrar) { - registrar.registerReferenceProvider( - PlatformPatterns.psiElement(PyElementTypes.STRING_LITERAL_EXPRESSION), - HuggingFaceDatasetReferenceProvider(), + HuggingFaceIdentifierReferenceProvider(), PsiReferenceRegistrar.DEFAULT_PRIORITY ) } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceProvider.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceProvider.kt index 914cc1a91e46..6c64a417b3a5 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceProvider.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/annotation/HuggingFaceReferenceProvider.kt @@ -5,32 +5,28 @@ import com.intellij.openapi.util.TextRange import com.intellij.psi.PsiElement import com.intellij.psi.PsiReference import com.intellij.psi.PsiReferenceProvider -import com.intellij.python.community.impl.huggingFace.service.HuggingFaceImportedLibrariesManagerService import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil +import com.intellij.python.community.impl.huggingFace.service.HuggingFaceImportedLibrariesManager import com.intellij.util.ProcessingContext import com.jetbrains.python.psi.PyStringLiteralExpression +import org.jetbrains.annotations.ApiStatus -abstract class HuggingFaceReferenceProvider : PsiReferenceProvider() { - +@ApiStatus.Internal +class HuggingFaceIdentifierReferenceProvider : PsiReferenceProvider() { override fun getReferencesByElement(element: PsiElement, context: ProcessingContext): Array { val pyStringLiteralExpression = element as? PyStringLiteralExpression ?: return PsiReference.EMPTY_ARRAY val project = element.project - val service = project.getService(HuggingFaceImportedLibrariesManagerService::class.java) - val manager = service.getManager() + val manager = project.getService(HuggingFaceImportedLibrariesManager::class.java) if (!manager.isLibraryImported()) return PsiReference.EMPTY_ARRAY val text = pyStringLiteralExpression.stringValue - if (!isValidReference(text)) return PsiReference.EMPTY_ARRAY + val entityKind = HuggingFaceUtil.isWhatHuggingFaceEntity(text) ?: return PsiReference.EMPTY_ARRAY val textRange = getTextRange(element, text) - return getReferenceArray(element, textRange, text) + return arrayOf(HuggingFaceReference(element, textRange, text, entityKind)) } - abstract fun isValidReference(text: String) : Boolean - - abstract fun getReferenceArray(element: PsiElement, textRange: TextRange, text: String) : Array - private fun getTextRange(element: PsiElement, text: String): TextRange { val startOffset = element.text.indexOf(text) return if (startOffset >= 0) { @@ -40,17 +36,3 @@ abstract class HuggingFaceReferenceProvider : PsiReferenceProvider() { } } } - -class HuggingFaceModelReferenceProvider : HuggingFaceReferenceProvider() { - override fun isValidReference(text: String) = HuggingFaceUtil.isHuggingFaceModel(text) - - override fun getReferenceArray(element: PsiElement, textRange: TextRange, text: String): Array = - arrayOf(HuggingFaceModelReference(element, textRange, text)) -} - -class HuggingFaceDatasetReferenceProvider : HuggingFaceReferenceProvider() { - override fun isValidReference(text: String) = HuggingFaceUtil.isHuggingFaceDataset(text) - - override fun getReferenceArray(element: PsiElement, textRange: TextRange, text: String): Array = - arrayOf(HuggingFaceDatasetReference(element, textRange, text)) -} diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceApi.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceApi.kt index 934133b04262..f8b66879ff1e 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceApi.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceApi.kt @@ -8,9 +8,18 @@ import com.intellij.openapi.util.NlsSafe import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceCache +import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceMdCacheEntry +import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceMdCardsCache +import com.intellij.python.community.impl.huggingFace.documentation.HuggingFaceDocumentationPlaceholdersUtil +import com.intellij.python.community.impl.huggingFace.documentation.HuggingFaceReadmeCleaner import com.intellij.python.community.impl.huggingFace.service.HuggingFaceSafeExecutor +import org.jetbrains.annotations.ApiStatus +import java.time.Instant +@ApiStatus.Internal object HuggingFaceApi { + private val nextLinkRegex = Regex("""<(.+)>; rel="next"""") + fun fillCacheWithBasicApiData(endpoint: HuggingFaceEntityKind, cache: HuggingFaceCache, maxCount: Int) { val executor = HuggingFaceSafeExecutor.instance @@ -28,8 +37,8 @@ object HuggingFaceApi { } } - suspend fun performSearch(query: String, tags: String?): Map { - val queryUrl = HuggingFaceURLProvider.getModelSearchQuery(query, tags) + suspend fun performSearch(query: String, tags: String?, sortKey: HuggingFaceModelSortKey): Map { + val queryUrl = HuggingFaceURLProvider.getModelSearchQuery(query, tags, sortKey) val executor = HuggingFaceSafeExecutor.instance return executor.asyncSuspend("SearchHuggingFace") { @@ -44,7 +53,7 @@ object HuggingFaceApi { } private fun extractNextPageUrl(linkHeader: String?): String? { - val nextLinkMatch = Regex("""<(.+)>; rel="next"""").find(linkHeader ?: "") + val nextLinkMatch = nextLinkRegex.find(linkHeader ?: "") return nextLinkMatch?.groupValues?.get(1) } @@ -83,4 +92,28 @@ object HuggingFaceApi { return modelDataMap } + + fun fetchOrRetrieveModelCard(entityDataApiContent: HuggingFaceEntityBasicApiData, + entityId: String, + entityKind: HuggingFaceEntityKind, + prefix: String = "markdown"): String { + return if (entityDataApiContent.gated != "false") { + HuggingFaceDocumentationPlaceholdersUtil.generateGatedEntityMarkdownString(entityId, entityKind) + } + else { + val cached = HuggingFaceMdCardsCache.getData("${prefix}_${entityId}") + cached?.data + ?: try { + val mdUrl = HuggingFaceURLProvider.getEntityMarkdownURL(entityId, entityKind).toString() + val rawData = HuggingFaceHttpClient.downloadFile(mdUrl) + ?: HuggingFaceDocumentationPlaceholdersUtil.noInternetConnectionPlaceholder(entityId) + val cleanedData = HuggingFaceReadmeCleaner(rawData, entityId, entityKind).doCleanUp().getMarkdown() + HuggingFaceMdCardsCache.saveData("markdown_$entityId", HuggingFaceMdCacheEntry(cleanedData, Instant.now())) + cleanedData + } + catch (e: Exception) { + HuggingFaceDocumentationPlaceholdersUtil.noInternetConnectionPlaceholder(entityId) + } + } + } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceEntityBasicApiData.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceEntityBasicApiData.kt index f820683e265a..91fe96af220c 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceEntityBasicApiData.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceEntityBasicApiData.kt @@ -3,12 +3,15 @@ package com.intellij.python.community.impl.huggingFace.api import com.intellij.openapi.util.NlsSafe import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind +import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil.humanReadableNumber +import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.Nls import java.time.LocalDateTime import java.time.format.DateTimeFormatter import java.time.format.FormatStyle +@ApiStatus.Internal data class HuggingFaceEntityBasicApiData ( val kind: HuggingFaceEntityKind, @Nls val itemId: String, @@ -19,31 +22,9 @@ data class HuggingFaceEntityBasicApiData ( val libraryName: String?, @NlsSafe val pipelineTag: String ) { - @Nls - private fun humanReadableNumber(rawNumber: Int): String { - if (rawNumber < 1000) return rawNumber.toString() - val suffixes = arrayOf("K", "M", "B", "T") - var count = rawNumber.toDouble() - var i = 0 - while (i < suffixes.size && count >= 1000) { - count /= 1000 - i++ - } - return String.format("%.1f%s", count, suffixes[i - 1]) - } - - @Nls - fun humanReadableLikes(): String { - return humanReadableNumber(likes) - } - - @Nls - fun humanReadableDownloads(): String { - return humanReadableNumber(downloads) - } - - fun humanReadableLastUpdated(): String { - val parsedDate = LocalDateTime.parse(lastModified, DateTimeFormatter.ISO_DATE_TIME) - return parsedDate.format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM)) - } + val humanReadableLikes: String = humanReadableNumber(likes) + val humanReadableDownloads: String = humanReadableNumber(downloads) + val humanReadableLastUpdated: String = + LocalDateTime.parse(lastModified, DateTimeFormatter.ISO_DATE_TIME) + .format(DateTimeFormatter.ofLocalizedDate(FormatStyle.MEDIUM)) } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceHttpClient.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceHttpClient.kt index 401e84531b29..699276747e89 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceHttpClient.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceHttpClient.kt @@ -3,12 +3,13 @@ package com.intellij.python.community.impl.huggingFace.api import com.intellij.openapi.diagnostic.thisLogger import com.intellij.util.io.HttpRequests +import org.jetbrains.annotations.ApiStatus import java.net.HttpURLConnection import java.net.URL -data class HfHttpResponseWithHeaders(val content: String?, val linkHeader: String?) - +@ApiStatus.Internal data class HfHttpResponseWithHeaders(val content: String?, val linkHeader: String?) +@ApiStatus.Internal object HuggingFaceHttpClient { fun exists(url: String, retries: Int = 2): Boolean { repeat(retries) { diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceModelSortKey.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceModelSortKey.kt new file mode 100644 index 000000000000..b1deafce3a93 --- /dev/null +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceModelSortKey.kt @@ -0,0 +1,13 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package com.intellij.python.community.impl.huggingFace.api + +import org.jetbrains.annotations.ApiStatus + +@ApiStatus.Internal +enum class HuggingFaceModelSortKey(val value: String, val displayName: String) { + AUTHOR("author", "Author"), + CREATED_AT("createdAt", "Created At"), + LAST_MODIFIED("lastModified", "Last Modified"), + DOWNLOADS("downloads", "Downloads"), + LIKES("likes", "Likes"); +} diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt index 0ef0eee147df..f5781969de08 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt @@ -4,13 +4,15 @@ package com.intellij.python.community.impl.huggingFace.api import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle +import org.jetbrains.annotations.ApiStatus import java.net.URL +@ApiStatus.Internal object HuggingFaceURLProvider { private val baseURL = PyHuggingFaceBundle.message("python.hugging.face.base.url") - private val models_expand_parameters = listOf("gated", "downloads", "likes", "lastModified", "pipeline_tag") - private val datasets_expand_parameters = listOf("gated", "downloads", "likes", "lastModified") + private val modelsExpandParameters = listOf("gated", "downloads", "likes", "lastModified", "pipeline_tag") + private val datasetsExpandParameters = listOf("gated", "downloads", "likes", "lastModified") fun getEntityMarkdownURL(entityId: String, entityKind: HuggingFaceEntityKind): URL { return when(entityKind) { @@ -30,29 +32,24 @@ object HuggingFaceURLProvider { fun getDatasetCardLink(datasetId: String): URL = URL("$baseURL/datasets/$datasetId") - fun getModelApiLink(modelId: String): URL = URL("$baseURL/api/models/$modelId") - - fun getModelSearchQuery(query: String, tags: String? = null, sort: String = "downloads", limit: Int = 20): URL { - return URL("$baseURL/api/models?search=$query&sort=$sort&limit=$limit&direction=-1" - ).let { url -> - tags?.let { URL("${url}&filter=${tags}") } ?: url - } + fun getModelSearchQuery(query: String, + tags: String? = null, + sortKey: HuggingFaceModelSortKey = HuggingFaceModelSortKey.DOWNLOADS, + limit: Int = 20 + ): URL { + val urlString = "$baseURL/api/models?search=$query&sort=${sortKey.value}&limit=$limit&direction=-1&expand[]=gated&expand[]=downloads&expand[]=likes&expand[]=lastModified&expand[]=pipeline_tag&expand=library_name" + return URL(urlString + if (tags.isNullOrBlank()) "" else "&filter=$tags") } - // Will be necessary to fetch detailed information for dataset - // fun getDatasetApiLink(datasetId: String): URL = URL("$baseURL/api/models/$datasetId") - // may be needed for dataset viewer button: - // fun getDatasetViewerLink(datasetName: String): URL = URL("$baseURL/datasets/$datasetName/viewer") - fun fetchApiDataUrl( entityKind: HuggingFaceEntityKind, limit: Int = HuggingFaceConstants.HF_API_FETCH_PAGE_SIZE, sort: String = HuggingFaceConstants.API_FETCH_SORT_KEY ): URL { val expand = if (entityKind == HuggingFaceEntityKind.MODEL) { - models_expand_parameters + modelsExpandParameters } else { - datasets_expand_parameters + datasetsExpandParameters } val parameters = mapOf( @@ -73,9 +70,6 @@ object HuggingFaceURLProvider { return url } - fun makeAbsoluteImageLink(entityId: String, relativeImagePath: String): URL = - URL("$baseURL/$entityId/resolve/main/$relativeImagePath") - fun makeAbsoluteFileLink(entityId: String, relativeFilePath: String): URL = URL("$baseURL/$entityId/blob/main/$relativeFilePath") } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceEntitiesCache.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceEntitiesCache.kt index ac9a31ba3203..ebcad4ac0bcc 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceEntitiesCache.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceEntitiesCache.kt @@ -7,22 +7,21 @@ import com.intellij.openapi.components.Storage import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants import com.intellij.python.community.impl.huggingFace.api.HuggingFaceEntityBasicApiData import com.intellij.util.xmlb.XmlSerializerUtil +import org.jetbrains.annotations.ApiStatus -@State( - name = "HuggingFaceCache", - storages = [Storage("HuggingFaceCache.xml")] -) +@ApiStatus.Internal abstract class HuggingFaceCache(private val maxSize: Int) : PersistentStateComponent { + private var nameSet: MutableSet = mutableSetOf() + private val hotCacheMaxSize = 32 + private var cacheMap: LinkedHashMap = object : LinkedHashMap(maxSize, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { return size > maxSize } } - private var nameSet: MutableSet = mutableSetOf() - private val hotCacheMaxSize = 50 // smaller cache for IDs met on practice private val hotCache: LinkedHashMap = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { @@ -30,6 +29,7 @@ abstract class HuggingFaceCache(private val maxSize: Int) : PersistentStateCompo } } + @Synchronized fun saveEntities(entitiesMap: Map) { cacheMap.putAll(entitiesMap) entitiesMap.keys.forEach { key -> @@ -37,6 +37,7 @@ abstract class HuggingFaceCache(private val maxSize: Int) : PersistentStateCompo } } + @Synchronized fun isInCache(entityId: String): Boolean { if (hotCache.containsKey(entityId)) return true @@ -46,16 +47,18 @@ abstract class HuggingFaceCache(private val maxSize: Int) : PersistentStateCompo } else false } + @Synchronized fun getBasicData(entityId: String): HuggingFaceEntityBasicApiData? = cacheMap[entityId] + @Synchronized fun getPipelineTagForEntity(entityId: String): String? { hotCache[entityId]?.let { return it.pipelineTag } return cacheMap[entityId]?.pipelineTag } - override fun getState(): HuggingFaceCache = this - + @Synchronized fun getCacheSize() = cacheMap.size + override fun getState(): HuggingFaceCache = this override fun loadState(state: HuggingFaceCache) { XmlSerializerUtil.copyBean(state, this) diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceLastCacheRefresh.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceLastCacheRefresh.kt index 368aeb11490e..943b973c1c54 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceLastCacheRefresh.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceLastCacheRefresh.kt @@ -1,12 +1,14 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.impl.huggingFace.cache +import com.intellij.openapi.components.PersistentStateComponent +import com.intellij.openapi.components.Service import com.intellij.openapi.components.State import com.intellij.openapi.components.Storage -import com.intellij.openapi.components.Service -import com.intellij.openapi.components.PersistentStateComponent import com.intellij.util.xmlb.XmlSerializerUtil +import org.jetbrains.annotations.ApiStatus +@ApiStatus.Internal @Service(Service.Level.PROJECT) @State(name = "HuggingFaceLastCacheRefresh", storages = [Storage("HuggingFaceLastCacheRefresh.xml")]) class HuggingFaceLastCacheRefresh : PersistentStateComponent { diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceMdCardsCache.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceMdCardsCache.kt index 14522983b681..76bbfa5b9e83 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceMdCardsCache.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/cache/HuggingFaceMdCardsCache.kt @@ -5,11 +5,13 @@ import com.intellij.openapi.components.PersistentStateComponent import com.intellij.openapi.components.State import com.intellij.openapi.components.Storage import com.intellij.util.xmlb.XmlSerializerUtil +import org.jetbrains.annotations.ApiStatus import java.time.Instant -data class HuggingFaceMdCacheEntry(val data: String, val timeFetched: Instant) +@ApiStatus.Internal data class HuggingFaceMdCacheEntry(val data: String, val timeFetched: Instant) +@ApiStatus.Internal @State(name = "HuggingFaceMdCardsCache", storages = [Storage("HuggingFaceMdCardsCache.xml")]) object HuggingFaceMdCardsCache: PersistentStateComponent { diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationPlaceholdersUtil.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationPlaceholdersUtil.kt index b82c440139b4..349e765c22e6 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationPlaceholdersUtil.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationPlaceholdersUtil.kt @@ -4,7 +4,9 @@ package com.intellij.python.community.impl.huggingFace.documentation import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle +import org.jetbrains.annotations.ApiStatus +@ApiStatus.Internal object HuggingFaceDocumentationPlaceholdersUtil { fun generateGatedEntityMarkdownString(entityId: String, entityKind: HuggingFaceEntityKind): String { val msg = PyHuggingFaceBundle.message( @@ -21,14 +23,6 @@ object HuggingFaceDocumentationPlaceholdersUtil { return msg } - fun trimmedMdPlaceholder(entityId: String, entityKind: HuggingFaceEntityKind): String { - val cardUrl = HuggingFaceURLProvider.getEntityCardLink(entityId, entityKind) - val msg = PyHuggingFaceBundle.message("python.hugging.face.placeholder.trimmed", entityKind.printName, cardUrl) - return msg - } - - fun noInternetConnectionPlaceholder(entityId: String): String { - val msg = PyHuggingFaceBundle.message("python.hugging.face.placeholder.no.internet", entityId) - return msg - } + fun noInternetConnectionPlaceholder(entityId: String): String = + PyHuggingFaceBundle.message("python.hugging.face.placeholder.no.internet", entityId) } \ No newline at end of file diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTarget.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTarget.kt index 630f73c0fd61..2942bb16be01 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTarget.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTarget.kt @@ -11,51 +11,29 @@ import com.intellij.psi.PsiElement import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceDatasetPsiElement -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceModelPsiElement -import com.intellij.python.community.impl.huggingFace.api.HuggingFaceEntityBasicApiData -import com.intellij.python.community.impl.huggingFace.api.HuggingFaceHttpClient -import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider +import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceIdentifierPsiElement +import com.intellij.python.community.impl.huggingFace.api.HuggingFaceApi.fetchOrRetrieveModelCard import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceDatasetsCache -import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceMdCacheEntry -import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceMdCardsCache import com.intellij.python.community.impl.huggingFace.cache.HuggingFaceModelsCache import com.intellij.python.community.impl.huggingFace.service.HuggingFaceCardsUsageCollector import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle import com.intellij.refactoring.suggested.createSmartPointer import com.jetbrains.python.psi.PyTargetExpression -import java.time.Instant internal class HuggingFaceDocumentationTarget(private val myElement : PsiElement) : DocumentationTarget { override fun createPointer(): Pointer { val psiElementPointer = myElement.createSmartPointer() - - return object : Pointer { - override fun dereference(): DocumentationTarget? { - return psiElementPointer.element?.let { HuggingFaceDocumentationTarget(it) } - } - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (other == null || this::class != other::class) return false - other as HuggingFaceDocumentationTarget - return myElement == other.myElement - } - - override fun hashCode(): Int { - return myElement.hashCode() - } - } + return Pointer { psiElementPointer.element?.let { HuggingFaceDocumentationTarget(it) } } } override fun computePresentation(): TargetPresentation { val elementText = when (myElement) { is PyTargetExpression -> "${HuggingFaceConstants.HF_EMOJI} ${HuggingFaceUtil.extractTextFromPyTargetExpression(myElement)}" - is HuggingFaceModelPsiElement -> "${HuggingFaceConstants.HF_EMOJI} ${myElement.stringValue()}" - is HuggingFaceDatasetPsiElement -> "${HuggingFaceConstants.HF_EMOJI} ${myElement.stringValue()}" + is HuggingFaceIdentifierPsiElement -> "${HuggingFaceConstants.HF_EMOJI} ${myElement.stringValue()}" else -> PyHuggingFaceBundle.message("python.hugging.face.unknown.element") } + // todo: add HF logo icon here once it is incorporated return TargetPresentation.builder(elementText).icon(null).presentation() } @@ -63,9 +41,7 @@ internal class HuggingFaceDocumentationTarget(private val myElement : PsiElement override fun computeDocumentation(): DocumentationResult = DocumentationResult.asyncDocumentation { val (entityId, entityKind) = HuggingFaceUtil.extractStringValueAndEntityKindFromElement(myElement) - if (entityId == null || entityKind == null) { - return@asyncDocumentation DocumentationResult.documentation(PyHuggingFaceBundle.message("python.hugging.face.no.string.value.found")) - } + ?: return@asyncDocumentation DocumentationResult.documentation(PyHuggingFaceBundle.message("python.hugging.face.no.string.value.found")) try { val entityDataApiContent = if (entityKind == HuggingFaceEntityKind.MODEL) { @@ -73,7 +49,7 @@ internal class HuggingFaceDocumentationTarget(private val myElement : PsiElement } else { HuggingFaceDatasetsCache.getBasicData(entityId) } - // shall we log failures? + if (entityDataApiContent == null) { return@asyncDocumentation DocumentationResult.documentation(PyHuggingFaceBundle.message("python.hugging.face.could.not.fetch")) } val modelCardContent = fetchOrRetrieveModelCard(entityDataApiContent, entityId, entityKind) val project = readAction { myElement.project } @@ -96,25 +72,4 @@ internal class HuggingFaceDocumentationTarget(private val myElement : PsiElement DocumentationResult.documentation(PyHuggingFaceBundle.message("python.hugging.face.not.found", e.message ?: "")) } } - - private fun fetchOrRetrieveModelCard(entityDataApiContent: HuggingFaceEntityBasicApiData, - entityId: String, - entityKind: HuggingFaceEntityKind): String { - return if (entityDataApiContent.gated != "false") { - HuggingFaceDocumentationPlaceholdersUtil.generateGatedEntityMarkdownString(entityId, entityKind) - } else { - val cached = HuggingFaceMdCardsCache.getData("markdown_$entityId") - cached?.data - ?: try { - val mdUrl = HuggingFaceURLProvider.getEntityMarkdownURL(entityId, entityKind).toString() - val rawData = HuggingFaceHttpClient.downloadFile(mdUrl) - ?: HuggingFaceDocumentationPlaceholdersUtil.noInternetConnectionPlaceholder(entityId) - val cleanedData = HuggingFaceReadmeCleaner(rawData, entityId, entityKind).doCleanUp().getMarkdown() - HuggingFaceMdCardsCache.saveData("markdown_$entityId", HuggingFaceMdCacheEntry(cleanedData, Instant.now())) - cleanedData - } catch (e: Exception) { - HuggingFaceDocumentationPlaceholdersUtil.noInternetConnectionPlaceholder(entityId) - } - } - } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTargetProvider.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTargetProvider.kt index df5ae073e140..d49716446ba4 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTargetProvider.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceDocumentationTargetProvider.kt @@ -5,34 +5,33 @@ import com.intellij.platform.backend.documentation.DocumentationTarget import com.intellij.platform.backend.documentation.PsiDocumentationTargetProvider import com.intellij.psi.PsiElement import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceDatasetPsiElement -import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceModelPsiElement +import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceIdentifierPsiElement import com.jetbrains.python.psi.PyStringLiteralExpression import com.jetbrains.python.psi.PyTargetExpression +import org.jetbrains.annotations.ApiStatus +@ApiStatus.Internal class HuggingFaceDocumentationTargetProvider : PsiDocumentationTargetProvider { override fun documentationTargets(element: PsiElement, originalElement: PsiElement?): List { val documentationTargets = mutableListOf() - if (element is PyTargetExpression) { - val referencedElement = element.findAssignedValue() - - if (referencedElement is PyStringLiteralExpression) { - val stringValue = referencedElement.stringValue - if (HuggingFaceUtil.isHuggingFaceEntity(stringValue)) { + when (element) { + is PyTargetExpression -> { + val referencedElement = element.findAssignedValue() + if (referencedElement is PyStringLiteralExpression + && HuggingFaceUtil.isHuggingFaceEntity(referencedElement.stringValue)) { documentationTargets.add(HuggingFaceDocumentationTarget(element)) } } - } else if (element is HuggingFaceModelPsiElement) { - documentationTargets.add(HuggingFaceDocumentationTarget(element)) - } else if (element is HuggingFaceDatasetPsiElement) { - documentationTargets.add(HuggingFaceDocumentationTarget(element)) - } else if (element is PyStringLiteralExpression) { - val stringValue = element.stringValue - if (HuggingFaceUtil.isHuggingFaceEntity(stringValue)) { + is HuggingFaceIdentifierPsiElement -> { documentationTargets.add(HuggingFaceDocumentationTarget(element)) } + is PyStringLiteralExpression -> { + if (HuggingFaceUtil.isHuggingFaceEntity(element.stringValue)) { + documentationTargets.add(HuggingFaceDocumentationTarget(element)) + } + } } return documentationTargets diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceHtmlBuilder.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceHtmlBuilder.kt index 33954c65ea56..e64e40de67a8 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceHtmlBuilder.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceHtmlBuilder.kt @@ -10,8 +10,10 @@ import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.api.HuggingFaceEntityBasicApiData import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle +import org.jetbrains.annotations.ApiStatus +@ApiStatus.Internal class HuggingFaceHtmlBuilder( private val project: Project, private val modelDataApiContent: HuggingFaceEntityBasicApiData, @@ -19,9 +21,13 @@ class HuggingFaceHtmlBuilder( private val entityKind: HuggingFaceEntityKind ) { @NlsSafe - suspend fun build(): String { + suspend fun build(noHeader: Boolean = false): String { val headChunk = HtmlChunk.tag("head").child(HuggingFaceQuickDocStyles.styleChunk()) - val cardHeaderChunk = generateCardHeader(modelDataApiContent) + val cardHeaderChunk = if (noHeader) { + HtmlChunk.empty() + } else { + generateCardHeader(modelDataApiContent) + } @NlsSafe val convertedHtml = readAction { HuggingFaceMarkdownToHtmlConverter(project).convert(modelCardContent) } val wrappedBodyContent = HtmlChunk.div() @@ -54,23 +60,21 @@ class HuggingFaceHtmlBuilder( val modelInfoRow = DocumentationMarkup.GRAYED_ELEMENT .children( *conditionalChunks.toTypedArray(), - HtmlChunk.text(PyHuggingFaceBundle.message("updated.0", modelInfo.humanReadableLastUpdated())), + HtmlChunk.text(PyHuggingFaceBundle.message("python.hugging.face.updated.suffix", modelInfo.humanReadableLastUpdated)), HtmlChunk.nbsp(2), DOWNLOADS_ICON, - HtmlChunk.raw(HuggingFaceQuickDocStyles.HAIR_SPACE), - HtmlChunk.text(modelInfo.humanReadableDownloads()), + HtmlChunk.text(modelInfo.humanReadableDownloads), HtmlChunk.nbsp(2), LIKES_ICON, - HtmlChunk.raw(HuggingFaceQuickDocStyles.HAIR_SPACE), - HtmlChunk.raw(modelInfo.humanReadableLikes()), + HtmlChunk.raw(modelInfo.humanReadableLikes), HtmlChunk.nbsp(), ) val linkRow = HtmlChunk.tag("a") .attr("href", HuggingFaceURLProvider.getEntityCardLink(modelInfo.itemId, entityKind).toString()) - .child(HtmlChunk.text(PyHuggingFaceBundle.getMessage("open.on.hugging.face"))) + .child(HtmlChunk.text(PyHuggingFaceBundle.getMessage("python.hugging.face.open.on.link.text"))) .wrapWith("p") val headerContainer = HtmlChunk.div() @@ -85,12 +89,8 @@ class HuggingFaceHtmlBuilder( companion object { private val DOWNLOADS_ICON = HtmlChunk.tag("icon") - // .attr("src", "com.intellij.python.community.impl.huggingFace.icons.PythonCommunityImplHuggingFaceIcons.Download") .attr("src", "AllIcons.Actions.Download") private val LIKES_ICON = HtmlChunk.tag("icon") - // .attr(".src", "com.intellij.python.community.impl.huggingFace.icons.PythonCommunityImplHuggingFaceIcons.Like") .attr(".src", "AllIcons.Toolwindows.ToolWindowFavorites") - //private val LOGO_ICON = HtmlChunk.tag("icon") - // .attr(".src", "/com/intellij/python/community/impl/huggingFace/icons/logo.svg") } } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt index 58323191f8fc..4b022d6ba908 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt @@ -1,7 +1,7 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.impl.huggingFace.documentation -import com.intellij.json.JsonLanguage +import com.intellij.json.json5.Json5Language import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTION import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTIONS import com.intellij.markdown.utils.convertMarkdownToHtml @@ -10,6 +10,7 @@ import com.intellij.openapi.project.Project import com.intellij.openapi.util.NlsSafe import com.intellij.openapi.util.text.StringUtil import com.jetbrains.python.PythonLanguage +import org.jetbrains.annotations.ApiStatus import org.jetbrains.annotations.Nls @@ -17,25 +18,23 @@ import org.jetbrains.annotations.Nls * Justification for the complexity of the HuggingFaceMarkdownToHtmlConverter class: * it is driven by the unique rendering needs within the quickDoc display context. * Specifically, pre-tags were enhanced with a subclass to ensure they adapt to the quickDoc window width. - * Similarly, processing for blockquotes and tables was customized to respect the display area's constraints. + * Similarly, processing for blockquotes and tables was customized to respect the display area's constraints. * These adjustments guarantee that the converted HTML content seamlessly fits within the quickDoc environment, * enhancing readability and user interaction. */ +@ApiStatus.Internal class HuggingFaceMarkdownToHtmlConverter(private val project: Project) { @NlsSafe @Nls fun convert(markdown: String): String { val convertedHtml = convertMarkdownToHtml(markdown) - val htmlWithoutBaseTags = convertedHtml.replace( - Regex("<(/)?(html|head|body)( [^>]*)?>"), - "" - ) + val htmlWithoutBaseTags = convertedHtml.replace(HTML_BASE_TAGS_REGEX, "") val htmlWithFixedPreTags = htmlWithoutBaseTags.replace( - Regex("
"),
+      PRE_TAG_REGEX,
       "
"
     )
     val htmlWithFixedPTags = htmlWithFixedPreTags.replace(
-      Regex("
"),
+      P_TAG_REGEX,
       "
"
     )
     val htmlWithFixedQuotes = addDivToBlockquotes(htmlWithFixedPTags)
@@ -45,21 +44,19 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) {
   }
 
   private fun addDivToBlockquotes(html: String): String {
-    val blockquoteRegex = Regex("
(.*?)
", RegexOption.DOT_MATCHES_ALL)
-    return html.replace(blockquoteRegex) {
+    return html.replace(BLOCKQUOTE_REGEX) {
       "
${it.groupValues[1]}
"
     }
   }
 
   private fun formatTablesInHtml(html: String): String {
     val htmlNoIntellijRowEven = html.replace(" class=\"intellij-row-even\"", "")
-    val tableRegex = Regex("(.*?)", RegexOption.DOT_MATCHES_ALL)
-    val tdRegex = Regex("(.*?)", RegexOption.DOT_MATCHES_ALL)
 
-    return htmlNoIntellijRowEven.replace(tableRegex) { tableMatch ->
+
+    return htmlNoIntellijRowEven.replace(TABLE_REGEX) { tableMatch ->
       val formattedTable = "${tableMatch.groupValues[1]}
"
 
-      formattedTable.replace(tdRegex) { tdMatch ->
+      formattedTable.replace(TD_REGEX) { tdMatch ->
         "${tdMatch.groupValues[1]}"
       }
     }
@@ -78,7 +75,7 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) {
       val language = when {
         // PY-70540 - if additional languages are needed - to be added here
         "language-python" in codeAttributes -> PythonLanguage.INSTANCE
-        "language-json" in codeAttributes -> JsonLanguage.INSTANCE
+        "language-json" in codeAttributes -> Json5Language.INSTANCE
         else -> null
       }
 
@@ -109,4 +106,13 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) {
       .replace("'", "'")
       .replace("&", "&")
   }
+
+  companion object {
+    private val HTML_BASE_TAGS_REGEX = Regex("<(/)?(html|head|body)( [^>]*)?>")
+    private val PRE_TAG_REGEX = Regex("
")
+    private val P_TAG_REGEX = Regex("
")
+    private val BLOCKQUOTE_REGEX = Regex("
(.*?)
", RegexOption.DOT_MATCHES_ALL)
+    private val TABLE_REGEX = Regex("(.*?)", RegexOption.DOT_MATCHES_ALL)
+    private val TD_REGEX = Regex("(.*?)", RegexOption.DOT_MATCHES_ALL)
+  }
 }
\ No newline at end of file
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceQuickDocStyles.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceQuickDocStyles.kt
index 8a133b7c1324..a763fa63463b 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceQuickDocStyles.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceQuickDocStyles.kt
@@ -2,6 +2,7 @@ package com.intellij.python.community.impl.huggingFace.documentation
 
 import com.intellij.openapi.util.NlsSafe
 import com.intellij.openapi.util.text.HtmlChunk
+import com.intellij.python.community.impl.huggingFace.service.PyHuggingFaceBundle
 import com.intellij.ui.scale.JBUIScale
 import org.intellij.lang.annotations.Language
 
@@ -28,8 +29,7 @@ object HuggingFaceQuickDocStyles {  // see PY-70541
   const val HAIR_SPACE = " "
 
   @Language("CSS")
-  @NlsSafe
-  private val styleContent = listOf(
+  val styleRules = listOf(
     "$HF_P_TAG_CLASS { margin-top: ${scale(4)}px; margin-bottom: ${scale(6)}px; }",
 
     ".$CODE_DIV_CLASS { padding-top: ${scale(4)}px; padding-bottom: ${scale(4)}px;  padding-left: ${scale(4)}px; " +
@@ -47,7 +47,10 @@ object HuggingFaceQuickDocStyles {  // see PY-70541
     "blockquote { border-left: ${scale(4)}px solid #cccccc; }",
 
     "blockquote p { border-left: none; }",
-  ).joinToString(separator = " ")
+  )
 
-  fun styleChunk(): HtmlChunk = HtmlChunk.raw("")
+  @NlsSafe
+  private val styleContent = styleRules.joinToString(separator = " ")
+
+  fun styleChunk(): HtmlChunk = HtmlChunk.raw(PyHuggingFaceBundle.message("python.hugging.face.tags.style", styleContent))
 }
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
index 318c8034f39e..537769d5694a 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
@@ -1,12 +1,12 @@
 package com.intellij.python.community.impl.huggingFace.documentation
 
-import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants
 import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind
 import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider
+import org.jetbrains.annotations.ApiStatus
 import java.net.URL
 import java.util.*
 
-
+@ApiStatus.Internal
 class HuggingFaceReadmeCleaner(
   private var markdown: String,
   private val entityId: String,
@@ -180,36 +180,6 @@ class HuggingFaceReadmeCleaner(
     }
   }
 
-  @Suppress("unused")  // may be activated if we decide to trim model cards
-  private fun trimLongMd() {
-    markdown = if (markdown.length < HuggingFaceConstants.MAX_MD_CHAR_NUM) markdown else {
-      var trimmedMd = markdown.substring(0, HuggingFaceConstants.MAX_MD_CHAR_NUM)
-      val numCodeFences = trimmedMd.split(CODE_FENCE_MARKER).size - 1
-
-      // Check for an odd number of code fences
-      if (numCodeFences % 2 != 0) {
-        val lastCodeFenceIndex = trimmedMd.lastIndexOf("```")
-        if (lastCodeFenceIndex != -1) {
-          trimmedMd = trimmedMd.substring(0, lastCodeFenceIndex)
-        }
-      }
-
-      val sentenceBoundary = trimmedMd.lastIndexOf(". ")
-      val imageBoundary = trimmedMd.lastIndexOf("![")
-      val tableBoundary = trimmedMd.lastIndexOf("|")
-      val lastNewLine = trimmedMd.lastIndexOf("\n")
-
-      val maxBoundary = maxOf(sentenceBoundary, imageBoundary, tableBoundary, lastNewLine)
-
-      if (maxBoundary != -1) {
-        trimmedMd = trimmedMd.substring(0, maxBoundary + 1)
-      }
-
-      val placeholder = HuggingFaceDocumentationPlaceholdersUtil.trimmedMdPlaceholder(entityId, entityKind)
-      "${trimmedMd.trimEnd()}\n\n${placeholder}"
-    }
-  }
-
   companion object {
     private const val HF_MD_HEADER_SEPARATOR = "---\n"
     private const val ERR_PY_CODE_FENCE_HEADER = "```py\n"
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/icons/PythonCommunityImplHuggingFaceIcons.java b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/icons/PythonCommunityImplHuggingFaceIcons.java
deleted file mode 100644
index 82e9fe5a385a..000000000000
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/icons/PythonCommunityImplHuggingFaceIcons.java
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
-package com.intellij.python.community.impl.huggingFace.icons;
-
-import com.intellij.ui.IconManager;
-import org.jetbrains.annotations.NotNull;
-
-import javax.swing.*;
-
-/**
- * NOTE THIS FILE IS AUTO-GENERATED
- * DO NOT EDIT IT BY HAND, run "Generate icon classes" configuration instead
- */
-public final class PythonCommunityImplHuggingFaceIcons {
-  private static @NotNull Icon load(@NotNull String path, int cacheKey, int flags) {
-    return IconManager.getInstance().loadRasterizedIcon(path, PythonCommunityImplHuggingFaceIcons.class.getClassLoader(), cacheKey, flags);
-  }
-  /** 14x14 */ public static final @NotNull Icon Download = load("com/intellij/python/community/impl/huggingFace/icons/download.svg", -459821553, 0);
-  /** 14x14 */ public static final @NotNull Icon Like = load("com/intellij/python/community/impl/huggingFace/icons/like.svg", -1702501286, 0);
-  /** 16x16 */ public static final @NotNull Icon Logo = load("com/intellij/python/community/impl/huggingFace/icons/logo.svg", 59143455, 0);
-}
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceExecutorService.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceExecutorService.kt
index 9681e28ac929..a7a538f0d1bd 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceExecutorService.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceExecutorService.kt
@@ -7,12 +7,14 @@ import com.intellij.openapi.components.service
 import com.intellij.openapi.diagnostic.Logger
 import com.intellij.platform.util.coroutines.childScope
 import kotlinx.coroutines.*
+import org.jetbrains.annotations.ApiStatus
 import kotlin.time.Duration
 
-
+@ApiStatus.Internal
 @Service(Service.Level.APP)
 class HuggingFaceExecutorService(val coroutineScope: CoroutineScope)
 
+@ApiStatus.Internal
 @Service(Service.Level.APP)
 class HuggingFaceSafeExecutor(private val coroutineScope: CoroutineScope, private val defaultTimeout: Duration): Disposable {
 
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceImportedLibrariesManager.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceImportedLibrariesManager.kt
index d7eab04ba0c0..9e31acf5f4bf 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceImportedLibrariesManager.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceImportedLibrariesManager.kt
@@ -5,39 +5,26 @@ import com.intellij.openapi.components.Service
 import com.intellij.openapi.editor.EditorFactory
 import com.intellij.openapi.editor.event.DocumentEvent
 import com.intellij.openapi.editor.event.DocumentListener
+import com.intellij.openapi.project.Project
 import com.intellij.openapi.vfs.VirtualFileManager
 import com.intellij.openapi.vfs.newvfs.BulkFileListener
 import com.intellij.openapi.vfs.newvfs.events.VFileEvent
-import com.intellij.openapi.project.Project
 import com.intellij.python.community.impl.huggingFace.HuggingFaceUtil
 import com.intellij.util.messages.MessageBusConnection
+import org.jetbrains.annotations.ApiStatus
 
-
+@ApiStatus.Internal
 @Service(Service.Level.PROJECT)
-class HuggingFaceImportedLibrariesManagerService(project: Project) {
-  private val manager = HuggingFaceImportedLibrariesManager(project)
-
-  fun getManager(): HuggingFaceImportedLibrariesManager {
-    return manager
-  }
-}
-
-
-class HuggingFaceImportedLibrariesManager(project: Project) {
-
+class HuggingFaceImportedLibrariesManager(val project: Project) {
   private var cachedResult: Result? = null
   private var cacheTimestamp: Long = 0
   private val invalidationThresholdMinutes = 5
-
   private var changesCount: Int = 0
   private val changesThreshold = 10
-  private var project: Project? = null
 
   private data class Result(val isImported: Boolean)
 
   init {
-    this.project = project
-
     val connection: MessageBusConnection = project.messageBus.connect()
     connection.subscribe(VirtualFileManager.VFS_CHANGES, object : BulkFileListener {
       override fun after(events: List) {
@@ -60,7 +47,7 @@ class HuggingFaceImportedLibrariesManager(project: Project) {
   }
 
   fun isLibraryImported(): Boolean {
-    val project = this.project ?: throw IllegalStateException("Init the ImportedLibrariesManager with project first.")
+    val project = this.project
     if (System.currentTimeMillis() - cacheTimestamp > invalidationThresholdMinutes * 60 * 1000 || cachedResult == null) {
       val isImported = HuggingFaceUtil.isAnyHFLibraryImportedInProject(project)
       if (isImported) {
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceTypoInspectionSuppressor.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceTypoInspectionSuppressor.kt
index 7236ee217f54..2c62bfc6a1b3 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceTypoInspectionSuppressor.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/service/HuggingFaceTypoInspectionSuppressor.kt
@@ -5,7 +5,7 @@ import com.intellij.codeInspection.InspectionSuppressor
 import com.intellij.codeInspection.SuppressQuickFix
 import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiFileSystemItem
-import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceEntityPsiElement.Companion.HUGGING_FACE_ENTITY_NAME_KEY
+import com.intellij.python.community.impl.huggingFace.annotation.HuggingFaceIdentifierPsiElement.Companion.HUGGING_FACE_ENTITY_NAME_KEY
 import com.jetbrains.python.psi.PyFile
 import com.jetbrains.python.psi.impl.PyPlainStringElementImpl