MDS/openide
1
0
Fork 0
mirror of https://gitflic.ru/project/openide/openide.git synced 2026-09-27 10:03:11 +07:00
Code Issues Packages Projects Releases Wiki Activity

[pycharm] PY-70541 HF cards layout and formatting fixes (see ticket comments)

Browse Source 
GitOrigin-RevId: e53005ef230830a5e9cfa42fa69e62b5c56584e8
This commit is contained in:
Bogdan Kirilenko
2024-03-07 00:34:50 +00:00
committed by intellij-monorepo-bot
parent f0b627a002
commit 0b7d585ec3
5 changed files with 86 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/huggingFace/intellij.python.community.impl.huggingFace.iml
+1
View File
@@ -28,5 +28,6 @@
<orderEntry type="module" module-name="intellij.platform.ide" />
<orderEntry type="module" module-name="intellij.platform.markdown.utils" />
<orderEntry type="module" module-name="intellij.platform.statistics" />
<orderEntry type="module" module-name="intellij.json" />
</component>
</module>
python/huggingFace/resources/messages/PyHuggingFaceBundle.properties
+1 -1
View File
@@ -12,6 +12,6 @@ python.hugging.face.dictionary.name=Hugging Face model and dataset dictionary
python.hugging.face.placeholder.gated.model=<font color='gray'>Sorry, this {0} is gated.<br>Please visit the [HuggingFace website]({1}) <br>for more details about the {2} card.</font>
python.hugging.face.placeholder.no.readme=This model has no description available. For more details, please follow the [link]({0})
python.hugging.face.placeholder.trimmed=*Please find the rest of the {0} card [here]({1})*
python.hugging.face.placeholder.no.internet="Failed to fetch data for {0}. Please check your internet connection."
python.hugging.face.placeholder.no.internet=Failed to fetch data for {0}. Please check your internet connection.
python.hugging.face.dataset=Dataset
python/huggingFace/src/com/intellij/python/community/impl/huggingFace/api/HuggingFaceURLProvider.kt
+5 -2
View File
@@ -73,6 +73,9 @@ object HuggingFaceURLProvider {
return url
}
//fun makeAbsoluteImageLink(modelName: String, relativeImagePath: String): URL =
// URL("$baseURL/$modelName/resolve/main/$relativeImagePath")
fun makeAbsoluteImageLink(entityId: String, relativeImagePath: String): URL =
URL("$baseURL/$entityId/resolve/main/$relativeImagePath")
fun makeAbsoluteFileLink(entityId: String, relativeFilePath: String): URL =
URL("$baseURL/$entityId/blob/main/$relativeFilePath")
}
python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceMarkdownToHtmlConverter.kt
+6 -3
View File
@@ -1,15 +1,16 @@
// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license.
package com.intellij.python.community.impl.huggingFace.documentation
import com.intellij.json.JsonLanguage
import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTION
import com.intellij.lang.documentation.DocumentationMarkup.CLASS_SECTIONS
import com.intellij.openapi.util.NlsSafe
import org.jetbrains.annotations.Nls
import com.intellij.markdown.utils.convertMarkdownToHtml
import com.intellij.markdown.utils.lang.HtmlSyntaxHighlighter
import com.intellij.openapi.project.Project
import com.intellij.openapi.util.NlsSafe
import com.intellij.openapi.util.text.StringUtil
import com.jetbrains.python.PythonLanguage
import org.jetbrains.annotations.Nls
/**
@@ -75,8 +76,10 @@ class HuggingFaceMarkdownToHtmlConverter(private val project: Project) {
rawCode = decodeHtmlEntities(rawCode)
val language = when {
// PY-70540 - if additional languages are needed - to be added here
"language-python" in codeAttributes -> PythonLanguage.INSTANCE
else -> null // it seems like only python is crucial
"language-json" in codeAttributes -> JsonLanguage.INSTANCE
else -> null
}
val highlightedHtmlChunk = if (language != null) {
python/huggingFace/src/com/intellij/python/community/impl/huggingFace/documentation/HuggingFaceReadmeCleaner.kt
+73 -16
View File
@@ -4,14 +4,7 @@ import com.intellij.python.community.impl.huggingFace.HuggingFaceConstants
import com.intellij.python.community.impl.huggingFace.HuggingFaceEntityKind
import com.intellij.python.community.impl.huggingFace.api.HuggingFaceURLProvider
import java.net.URL
private const val HF_MD_HEADER_SEPARATOR = "---\n"
private const val ERR_PY_CODE_FENCE_HEADER = "```py\n"
private const val PY_CODE_FENCE_HEADER = "```python\n"
private const val CODE_FENCE_MARKER = "```"
private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)"""
private const val HTML_IMG_PATTERN = """<img([^>]+)?>"""
import java.util.*
class HuggingFaceReadmeCleaner(
@@ -22,10 +15,15 @@ class HuggingFaceReadmeCleaner(
private val cardUrl: URL = HuggingFaceURLProvider.getEntityCardLink(entityId, entityKind)
fun doCleanUp(): HuggingFaceReadmeCleaner {
// todo: some optimisation is needed:
// headers are collected twice - in the increaseHeaderLevels and fixContentTables
removeMetaData()
increaseHeaderLevels()
fixCodeFences()
cleanupNotSupportedElements()
cleanupImages()
convertRelativeFileLinksToAbsolute()
fixContentTables()
processMarkdownTables()
removeMarkdownSeparators()
// trimLongMd()
@@ -48,15 +46,51 @@ class HuggingFaceReadmeCleaner(
}
}
private fun fixCodeFences() {
markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER)
.replace("<details>", "")
.replace("</details>", "")
.replace(Regex("<summary>.*</summary>"), "")
private fun fixContentTables() {
val internalLinksRegex = INTERNAL_LINK_PATTERN.toRegex()
val headersRegex = MARKDOWN_HEADER_PATTERN.toRegex()
val internalLinks = internalLinksRegex.findAll(markdown).map { it.groupValues[2] }.toList()
val headers = headersRegex.findAll(markdown).map { it.value.trim() }.toList()
internalLinks.forEach { link ->
val anchor = "<a name=\"$link\"></a>"
if (markdown.contains(anchor)) {
return@forEach
}
val normalizedLink = link.replace("-", "").lowercase(Locale.getDefault())
headers.forEach { header ->
val normalizedHeader = header
.replace(Regex("^#{1,6}\\s"), "")
.replace(" ", "").lowercase(Locale.getDefault())
if (normalizedLink == normalizedHeader) {
// Find the position of the header and insert the anchor above it
val headerIndex = markdown.indexOf(header)
if (headerIndex != -1) {
markdown = markdown.substring(0, headerIndex) + "$anchor\n" + markdown.substring(headerIndex)
}
}
}
}
}
private fun cleanupImages() { // See PY-70539
// Pattern to match ![alt text](url)
private fun fixCodeFences() {
markdown = markdown.replace(ERR_PY_CODE_FENCE_HEADER, PY_CODE_FENCE_HEADER)
}
private fun cleanupNotSupportedElements() {
markdown = markdown
.replace("<details>", "")
.replace("</details>", "")
.replace(Regex(SUMMARY_TAGS_PATTERN)) { matchResult ->
matchResult.groupValues[1] // Return only the content captured between <summary> tags
}
}
private fun cleanupImages() {
// See PY-70539 -> potentially we could keep svgs
val markdownImgPattern = Regex(MD_IMG_PATTERN)
markdown = markdownImgPattern.replace(markdown) { matchResult ->
val altText = matchResult.groupValues[1].ifBlank { matchResult.groupValues[2].split("/").last() }
@@ -77,6 +111,16 @@ class HuggingFaceReadmeCleaner(
}
}
private fun convertRelativeFileLinksToAbsolute() {
// Catch relative links to files excluding internal markdown links (like in tables of content)
val regex = RELATIVE_LINK_PATTERN.toRegex()
markdown = regex.replace(markdown) { matchResult ->
val (linkText, relativePath) = matchResult.destructured
val absoluteUrl = HuggingFaceURLProvider.makeAbsoluteFileLink(entityId, relativePath).toString()
"[$linkText]($absoluteUrl)"
}
}
private fun processMarkdownTables() {
val lines = markdown.split("\n")
val processedLines = mutableListOf<String>()
@@ -107,7 +151,7 @@ class HuggingFaceReadmeCleaner(
val header = table.first()
val columnCount = header.split("|").filter { it.isNotBlank() }.size
if (columnCount <= 3) return table
if (columnCount <= 4) return table
val truncatedTable = mutableListOf<String>()
truncatedTable.add(header.split("|").take(4).joinToString("|") + "|...|")
@@ -165,4 +209,17 @@ class HuggingFaceReadmeCleaner(
"${trimmedMd.trimEnd()}\n\n${placeholder}"
}
}
companion object {
private const val HF_MD_HEADER_SEPARATOR = "---\n"
private const val ERR_PY_CODE_FENCE_HEADER = "```py\n"
private const val PY_CODE_FENCE_HEADER = "```python\n"
private const val CODE_FENCE_MARKER = "```"
private const val MD_IMG_PATTERN = """!\[(.*?)]\((.*?)\)"""
private const val HTML_IMG_PATTERN = """<img([^>]+)?>"""
private const val MARKDOWN_HEADER_PATTERN = """(?m)^#{1,6}\s(.*?)$"""
private const val INTERNAL_LINK_PATTERN = """\[(.*?)\]\(#(.*?)\)"""
private const val RELATIVE_LINK_PATTERN = """\[(.*?)\]\((?!http|#)(.*?)(?<!\.(jpg|jpeg|png|gif))\)"""
private const val SUMMARY_TAGS_PATTERN = "<summary>(.*?)</summary>"
}
}