diff --git a/python/huggingFace/intellij.python.community.impl.huggingFace.iml b/python/huggingFace/intellij.python.community.impl.huggingFace.iml index a8f7a3650701..afd0f03a8540 100644 --- a/python/huggingFace/intellij.python.community.impl.huggingFace.iml +++ b/python/huggingFace/intellij.python.community.impl.huggingFace.iml @@ -28,5 +28,6 @@ + \ No newline at end of file diff --git a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties index e760bceaf907..c8883017324a 100644 --- a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties +++ b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties @@ -12,6 +12,6 @@ python.hugging.face.dictionary.name=Hugging Face model and dataset dictionary python.hugging.face.placeholder.gated.model=Sorry, this {0} is gated.
Please visit the [HuggingFace website]({1})
for more details about the {2} card.
python.hugging.face.placeholder.no.readme=This model has no description available. For more details, please follow the [link]({0}) python.hugging.face.placeholder.trimmed=*Please find the rest of the {0} card [here]({1})* -python.hugging.face.placeholder.no.internet="Failed to fetch data for {0}. Please check your internet connection." +python.hugging.face.placeholder.no.internet=Failed to fetch data for {0}. Please check your internet connection. python.hugging.face.dataset=Dataset diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt index 91c00ceacc97..0ef0eee147df 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt @@ -73,6 +73,9 @@ object HuggingFaceURLProvider { return url } - //fun makeAbsoluteImageLink(modelName: String, relativeImagePath: String): URL = - // URL("$baseURL/$modelName/resolve/main/$relativeImagePath") + fun makeAbsoluteImageLink(entityId: String, relativeImagePath: String): URL = + URL("$baseURL/$entityId/resolve/main/$relativeImagePath") + + fun makeAbsoluteFileLink(entityId: String, relativeFilePath: String): URL = + URL("$baseURL/$entityId/blob/main/$relativeFilePath") } diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt index 767b6ac36521..58323191f8fc 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt @@ -1,15 +1,16 @@ // Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package com.intellij.python.community.impl.huggingFace.documentation +import com.intellij.json.JsonLanguage import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTION import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTIONS -import com.intellij.openapi.util.NlsSafe -import org.jetbrains.annotations.Nls import com.intellij.markdown.utils.convertMarkdownToHtml import com.intellij.markdown.utils.lang.HtmlSyntaxHighlighter import com.intellij.openapi.project.Project +import com.intellij.openapi.util.NlsSafe import com.intellij.openapi.util.text.StringUtil import com.jetbrains.python.PythonLanguage +import org.jetbrains.annotations.Nls /** @@ -75,8 +76,10 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) { rawCode = decodeHtmlEntities(rawCode) val language = when { + // PY-70540 - if additional languages are needed - to be added here "language-python" in codeAttributes -> PythonLanguage.INSTANCE - else -> null // it seems like only python is crucial + "language-json" in codeAttributes -> JsonLanguage.INSTANCE + else -> null } val highlightedHtmlChunk = if (language != null) { diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt index e9e541114813..318c8034f39e 100644 --- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt +++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt @@ -4,14 +4,7 @@ import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider import java.net.URL - - -private const val HF_MD_HEADER_SEPARATOR = "---\n" -private const val ERR_PY_CODE_FENCE_HEADER = "```py\n" -private const val PY_CODE_FENCE_HEADER = "```python\n" -private const val CODE_FENCE_MARKER = "```" -private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)""" -private const val HTML_IMG_PATTERN = """]+)?>""" +import java.util.* class HuggingFaceReadmeCleaner( @@ -22,10 +15,15 @@ class HuggingFaceReadmeCleaner( private val cardUrl: URL = HuggingFaceURLProvider.getEntityCardLink(entityId, entityKind) fun doCleanUp(): HuggingFaceReadmeCleaner { + // todo: some optimisation is needed: + // headers are collected twice - in the increaseHeaderLevels and fixContentTables removeMetaData() increaseHeaderLevels() fixCodeFences() + cleanupNotSupportedElements() cleanupImages() + convertRelativeFileLinksToAbsolute() + fixContentTables() processMarkdownTables() removeMarkdownSeparators() // trimLongMd() @@ -48,15 +46,51 @@ class HuggingFaceReadmeCleaner( } } - private fun fixCodeFences() { - markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER) - .replace("
", "") - .replace("
", "") - .replace(Regex(".*"), "") + private fun fixContentTables() { + val internalLinksRegex = INTERNAL_LINK_PATTERN.toRegex() + val headersRegex = MARKDOWN_HEADER_PATTERN.toRegex() + + val internalLinks = internalLinksRegex.findAll(markdown).map { it.groupValues[2] }.toList() + val headers = headersRegex.findAll(markdown).map { it.value.trim() }.toList() + + internalLinks.forEach { link -> + val anchor = "" + if (markdown.contains(anchor)) { + return@forEach + } + + val normalizedLink = link.replace("-", "").lowercase(Locale.getDefault()) + + headers.forEach { header -> + val normalizedHeader = header + .replace(Regex("^#{1,6}\\s"), "") + .replace(" ", "").lowercase(Locale.getDefault()) + if (normalizedLink == normalizedHeader) { + // Find the position of the header and insert the anchor above it + val headerIndex = markdown.indexOf(header) + if (headerIndex != -1) { + markdown = markdown.substring(0, headerIndex) + "$anchor\n" + markdown.substring(headerIndex) + } + } + } + } } - private fun cleanupImages() { // See PY-70539 - // Pattern to match ![alt text](url) + private fun fixCodeFences() { + markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER) + } + + private fun cleanupNotSupportedElements() { + markdown = markdown + .replace("
", "") + .replace("
", "") + .replace(Regex(SUMMARY_TAGS_PATTERN)) { matchResult -> + matchResult.groupValues[1] // Return only the content captured between tags + } + } + + private fun cleanupImages() { + // See PY-70539 -> potentially we could keep svgs val markdownImgPattern = Regex(MD_IMG_PATTERN) markdown = markdownImgPattern.replace(markdown) { matchResult -> val altText = matchResult.groupValues[1].ifBlank { matchResult.groupValues[2].split("/").last() } @@ -77,6 +111,16 @@ class HuggingFaceReadmeCleaner( } } + private fun convertRelativeFileLinksToAbsolute() { + // Catch relative links to files excluding internal markdown links (like in tables of content) + val regex = RELATIVE_LINK_PATTERN.toRegex() + markdown = regex.replace(markdown) { matchResult -> + val (linkText, relativePath) = matchResult.destructured + val absoluteUrl = HuggingFaceURLProvider.makeAbsoluteFileLink(entityId, relativePath).toString() + "[$linkText]($absoluteUrl)" + } + } + private fun processMarkdownTables() { val lines = markdown.split("\n") val processedLines = mutableListOf() @@ -107,7 +151,7 @@ class HuggingFaceReadmeCleaner( val header = table.first() val columnCount = header.split("|").filter { it.isNotBlank() }.size - if (columnCount <= 3) return table + if (columnCount <= 4) return table val truncatedTable = mutableListOf() truncatedTable.add(header.split("|").take(4).joinToString("|") + "|...|") @@ -165,4 +209,17 @@ class HuggingFaceReadmeCleaner( "${trimmedMd.trimEnd()}\n\n${placeholder}" } } + + companion object { + private const val HF_MD_HEADER_SEPARATOR = "---\n" + private const val ERR_PY_CODE_FENCE_HEADER = "```py\n" + private const val PY_CODE_FENCE_HEADER = "```python\n" + private const val CODE_FENCE_MARKER = "```" + private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)""" + private const val HTML_IMG_PATTERN = """]+)?>""" + private const val MARKDOWN_HEADER_PATTERN = """(?m)^#{1,6}\s(.*?)$""" + private const val INTERNAL_LINK_PATTERN = """\[(.*?)\]\(#(.*?)\)""" + private const val RELATIVE_LINK_PATTERN = """\[(.*?)\]\((?!http|#)(.*?)(?(.*?)" + } } \ No newline at end of file