diff --git a/python/huggingFace/intellij.python.community.impl.huggingFace.iml b/python/huggingFace/intellij.python.community.impl.huggingFace.iml
index a8f7a3650701..afd0f03a8540 100644
--- a/python/huggingFace/intellij.python.community.impl.huggingFace.iml
+++ b/python/huggingFace/intellij.python.community.impl.huggingFace.iml
@@ -28,5 +28,6 @@
+
\ No newline at end of file
diff --git a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties
index e760bceaf907..c8883017324a 100644
--- a/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties
+++ b/python/huggingFace/resources/messages/PyHuggingFaceBundle.properties
@@ -12,6 +12,6 @@ python.hugging.face.dictionary.name=Hugging Face model and dataset dictionary
python.hugging.face.placeholder.gated.model=Sorry, this {0} is gated.
Please visit the [HuggingFace website]({1})
for more details about the {2} card.
python.hugging.face.placeholder.no.readme=This model has no description available. For more details, please follow the [link]({0})
python.hugging.face.placeholder.trimmed=*Please find the rest of the {0} card [here]({1})*
-python.hugging.face.placeholder.no.internet="Failed to fetch data for {0}. Please check your internet connection."
+python.hugging.face.placeholder.no.internet=Failed to fetch data for {0}. Please check your internet connection.
python.hugging.face.dataset=Dataset
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt
index 91c00ceacc97..0ef0eee147df 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt
@@ -73,6 +73,9 @@ object HuggingFaceURLProvider {
return url
}
- //fun makeAbsoluteImageLink(modelName: String, relativeImagePath: String): URL =
- // URL("$baseURL/$modelName/resolve/main/$relativeImagePath")
+ fun makeAbsoluteImageLink(entityId: String, relativeImagePath: String): URL =
+ URL("$baseURL/$entityId/resolve/main/$relativeImagePath")
+
+ fun makeAbsoluteFileLink(entityId: String, relativeFilePath: String): URL =
+ URL("$baseURL/$entityId/blob/main/$relativeFilePath")
}
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt
index 767b6ac36521..58323191f8fc 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt
@@ -1,15 +1,16 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.python.community.impl.huggingFace.documentation
+import com.intellij.json.JsonLanguage
import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTION
import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTIONS
-import com.intellij.openapi.util.NlsSafe
-import org.jetbrains.annotations.Nls
import com.intellij.markdown.utils.convertMarkdownToHtml
import com.intellij.markdown.utils.lang.HtmlSyntaxHighlighter
import com.intellij.openapi.project.Project
+import com.intellij.openapi.util.NlsSafe
import com.intellij.openapi.util.text.StringUtil
import com.jetbrains.python.PythonLanguage
+import org.jetbrains.annotations.Nls
/**
@@ -75,8 +76,10 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) {
rawCode = decodeHtmlEntities(rawCode)
val language = when {
+ // PY-70540 - if additional languages are needed - to be added here
"language-python" in codeAttributes -> PythonLanguage.INSTANCE
- else -> null // it seems like only python is crucial
+ "language-json" in codeAttributes -> JsonLanguage.INSTANCE
+ else -> null
}
val highlightedHtmlChunk = if (language != null) {
diff --git a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
index e9e541114813..318c8034f39e 100644
--- a/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
+++ b/python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
@@ -4,14 +4,7 @@ import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants
import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind
import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider
import java.net.URL
-
-
-private const val HF_MD_HEADER_SEPARATOR = "---\n"
-private const val ERR_PY_CODE_FENCE_HEADER = "```py\n"
-private const val PY_CODE_FENCE_HEADER = "```python\n"
-private const val CODE_FENCE_MARKER = "```"
-private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)"""
-private const val HTML_IMG_PATTERN = """
]+)?>"""
+import java.util.*
class HuggingFaceReadmeCleaner(
@@ -22,10 +15,15 @@ class HuggingFaceReadmeCleaner(
private val cardUrl: URL = HuggingFaceURLProvider.getEntityCardLink(entityId, entityKind)
fun doCleanUp(): HuggingFaceReadmeCleaner {
+ // todo: some optimisation is needed:
+ // headers are collected twice - in the increaseHeaderLevels and fixContentTables
removeMetaData()
increaseHeaderLevels()
fixCodeFences()
+ cleanupNotSupportedElements()
cleanupImages()
+ convertRelativeFileLinksToAbsolute()
+ fixContentTables()
processMarkdownTables()
removeMarkdownSeparators()
// trimLongMd()
@@ -48,15 +46,51 @@ class HuggingFaceReadmeCleaner(
}
}
- private fun fixCodeFences() {
- markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER)
- .replace("", "")
- .replace(" ", "")
- .replace(Regex(".*"), "")
+ private fun fixContentTables() {
+ val internalLinksRegex = INTERNAL_LINK_PATTERN.toRegex()
+ val headersRegex = MARKDOWN_HEADER_PATTERN.toRegex()
+
+ val internalLinks = internalLinksRegex.findAll(markdown).map { it.groupValues[2] }.toList()
+ val headers = headersRegex.findAll(markdown).map { it.value.trim() }.toList()
+
+ internalLinks.forEach { link ->
+ val anchor = ""
+ if (markdown.contains(anchor)) {
+ return@forEach
+ }
+
+ val normalizedLink = link.replace("-", "").lowercase(Locale.getDefault())
+
+ headers.forEach { header ->
+ val normalizedHeader = header
+ .replace(Regex("^#{1,6}\\s"), "")
+ .replace(" ", "").lowercase(Locale.getDefault())
+ if (normalizedLink == normalizedHeader) {
+ // Find the position of the header and insert the anchor above it
+ val headerIndex = markdown.indexOf(header)
+ if (headerIndex != -1) {
+ markdown = markdown.substring(0, headerIndex) + "$anchor\n" + markdown.substring(headerIndex)
+ }
+ }
+ }
+ }
}
- private fun cleanupImages() { // See PY-70539
- // Pattern to match 
+ private fun fixCodeFences() {
+ markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER)
+ }
+
+ private fun cleanupNotSupportedElements() {
+ markdown = markdown
+ .replace("", "")
+ .replace(" ", "")
+ .replace(Regex(SUMMARY_TAGS_PATTERN)) { matchResult ->
+ matchResult.groupValues[1] // Return only the content captured between tags
+ }
+ }
+
+ private fun cleanupImages() {
+ // See PY-70539 -> potentially we could keep svgs
val markdownImgPattern = Regex(MD_IMG_PATTERN)
markdown = markdownImgPattern.replace(markdown) { matchResult ->
val altText = matchResult.groupValues[1].ifBlank { matchResult.groupValues[2].split("/").last() }
@@ -77,6 +111,16 @@ class HuggingFaceReadmeCleaner(
}
}
+ private fun convertRelativeFileLinksToAbsolute() {
+ // Catch relative links to files excluding internal markdown links (like in tables of content)
+ val regex = RELATIVE_LINK_PATTERN.toRegex()
+ markdown = regex.replace(markdown) { matchResult ->
+ val (linkText, relativePath) = matchResult.destructured
+ val absoluteUrl = HuggingFaceURLProvider.makeAbsoluteFileLink(entityId, relativePath).toString()
+ "[$linkText]($absoluteUrl)"
+ }
+ }
+
private fun processMarkdownTables() {
val lines = markdown.split("\n")
val processedLines = mutableListOf()
@@ -107,7 +151,7 @@ class HuggingFaceReadmeCleaner(
val header = table.first()
val columnCount = header.split("|").filter { it.isNotBlank() }.size
- if (columnCount <= 3) return table
+ if (columnCount <= 4) return table
val truncatedTable = mutableListOf()
truncatedTable.add(header.split("|").take(4).joinToString("|") + "|...|")
@@ -165,4 +209,17 @@ class HuggingFaceReadmeCleaner(
"${trimmedMd.trimEnd()}\n\n${placeholder}"
}
}
+
+ companion object {
+ private const val HF_MD_HEADER_SEPARATOR = "---\n"
+ private const val ERR_PY_CODE_FENCE_HEADER = "```py\n"
+ private const val PY_CODE_FENCE_HEADER = "```python\n"
+ private const val CODE_FENCE_MARKER = "```"
+ private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)"""
+ private const val HTML_IMG_PATTERN = """
]+)?>"""
+ private const val MARKDOWN_HEADER_PATTERN = """(?m)^#{1,6}\s(.*?)$"""
+ private const val INTERNAL_LINK_PATTERN = """\[(.*?)\]\(#(.*?)\)"""
+ private const val RELATIVE_LINK_PATTERN = """\[(.*?)\]\((?!http|#)(.*?)(?(.*?)"
+ }
}
\ No newline at end of file