From e4e1e56b64defb5cf74bba3c7853afa6cf27ae40 Mon Sep 17 00:00:00 2001 From: Chris Lemaire Date: Thu, 7 Nov 2024 10:58:18 +0100 Subject: [PATCH] [github] Add reporting for rates used during requests GitOrigin-RevId: 70db6715002f37965663a6960f23315eb96fd168 --- .../resources/graphql/fragment/rates.graphql | 3 + .../findOpenPullRequestsByBranches.graphql | 12 - .../query/findOrganizationTeams.graphql | 22 +- .../graphql/query/findPendingReview.graphql | 18 +- .../graphql/query/findPullRequest.graphql | 4 + .../graphql/query/findPullRequestId.graphql | 4 + .../query/findPullRequestMergeability.graphql | 12 +- .../graphql/query/findRepository.graphql | 10 +- .../resources/graphql/query/findUser.graphql | 4 + .../graphql/query/getProtectionRules.graphql | 24 +- .../query/getPullRequestTemplates.graphql | 4 + .../getRepositoryCommitStatusContexts.graphql | 4 + .../getRepositoryCommitStatusShort.graphql | 4 + .../graphql/query/issueSearch.graphql | 4 + .../graphql/query/pullRequestCommits.graphql | 32 +- .../graphql/query/pullRequestFiles.graphql | 26 +- .../query/pullRequestReviewThreads.graphql | 24 +- .../graphql/query/pullRequestTimeline.graphql | 410 +++++++++--------- .../plugins/github/api/GHGQLQueries.kt | 1 - .../github/api/GithubApiContentHelper.kt | 3 + .../plugins/github/api/GithubApiRequest.kt | 53 ++- .../github/api/GithubApiRequestExecutor.kt | 23 +- .../github/api/data/graphql/GHGQLRateLimit.kt | 7 + .../pullrequest/GHPRStatisticsCollector.kt | 113 ++++- 24 files changed, 513 insertions(+), 308 deletions(-) create mode 100644 plugins/github/resources/graphql/fragment/rates.graphql delete mode 100644 plugins/github/resources/graphql/query/findOpenPullRequestsByBranches.graphql create mode 100644 plugins/github/src/org/jetbrains/plugins/github/api/data/graphql/GHGQLRateLimit.kt diff --git a/plugins/github/resources/graphql/fragment/rates.graphql b/plugins/github/resources/graphql/fragment/rates.graphql new file mode 100644 index 000000000000..b2234fa1e567 --- /dev/null +++ b/plugins/github/resources/graphql/fragment/rates.graphql @@ -0,0 +1,3 @@ +fragment rates on RateLimit { + cost +} \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findOpenPullRequestsByBranches.graphql b/plugins/github/resources/graphql/query/findOpenPullRequestsByBranches.graphql deleted file mode 100644 index b3b249d87198..000000000000 --- a/plugins/github/resources/graphql/query/findOpenPullRequestsByBranches.graphql +++ /dev/null @@ -1,12 +0,0 @@ -query($repoOwner: String!, $repoName: String!, $baseBranch: String, $headBranch: String) { - repository(owner: $repoOwner, name: $repoName) { - pullRequests(states: [OPEN], baseRefName: $baseBranch, headRefName: $headBranch, first: 1) { - pageInfo { - ...pageInfo - } - nodes { - ...pullRequestInfo - } - } - } -} \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findOrganizationTeams.graphql b/plugins/github/resources/graphql/query/findOrganizationTeams.graphql index 503e526dbb99..ad463249af9a 100644 --- a/plugins/github/resources/graphql/query/findOrganizationTeams.graphql +++ b/plugins/github/resources/graphql/query/findOrganizationTeams.graphql @@ -1,12 +1,16 @@ query($organization: String!, $pageSize: Int = 100, $cursor: String) { - organization(login: $organization) { - teams(first: $pageSize, after: $cursor) { - pageInfo { - ...pageInfo - } - nodes { - ...teamInfo - } - } + organization(login: $organization) { + teams(first: $pageSize, after: $cursor) { + pageInfo { + ...pageInfo + } + nodes { + ...teamInfo + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findPendingReview.graphql b/plugins/github/resources/graphql/query/findPendingReview.graphql index 521b6784984e..71d46243bff5 100644 --- a/plugins/github/resources/graphql/query/findPendingReview.graphql +++ b/plugins/github/resources/graphql/query/findPendingReview.graphql @@ -1,11 +1,15 @@ query($pullRequestId: ID!) { - node(id: $pullRequestId) { - ... on PullRequest { - reviews(states: [PENDING], first: 1) { - nodes { - ...pullRequestPendingReview - } - } + node(id: $pullRequestId) { + ... on PullRequest { + reviews(states: [PENDING], first: 1) { + nodes { + ...pullRequestPendingReview } + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findPullRequest.graphql b/plugins/github/resources/graphql/query/findPullRequest.graphql index df53ee25b5c1..fe9e3569cddd 100644 --- a/plugins/github/resources/graphql/query/findPullRequest.graphql +++ b/plugins/github/resources/graphql/query/findPullRequest.graphql @@ -4,4 +4,8 @@ query($repoOwner: String!, $repoName: String!, $number: Int!) { ...pullRequestInfo } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findPullRequestId.graphql b/plugins/github/resources/graphql/query/findPullRequestId.graphql index b4d77644b630..d1f778e12b33 100644 --- a/plugins/github/resources/graphql/query/findPullRequestId.graphql +++ b/plugins/github/resources/graphql/query/findPullRequestId.graphql @@ -5,4 +5,8 @@ query($repoOwner: String!, $repoName: String!, $number: Int!) { number } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findPullRequestMergeability.graphql b/plugins/github/resources/graphql/query/findPullRequestMergeability.graphql index 35688c81c4d0..f094704a2263 100644 --- a/plugins/github/resources/graphql/query/findPullRequestMergeability.graphql +++ b/plugins/github/resources/graphql/query/findPullRequestMergeability.graphql @@ -1,7 +1,11 @@ query($repoOwner: String!, $repoName: String!, $number: Int!) { - repository(owner: $repoOwner, name: $repoName) { - pullRequest(number: $number) { - ...pullRequestMergeability - } + repository(owner: $repoOwner, name: $repoName) { + pullRequest(number: $number) { + ...pullRequestMergeability } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findRepository.graphql b/plugins/github/resources/graphql/query/findRepository.graphql index 880b2ae38363..83f4adc2dfe2 100644 --- a/plugins/github/resources/graphql/query/findRepository.graphql +++ b/plugins/github/resources/graphql/query/findRepository.graphql @@ -1,5 +1,9 @@ query($repoOwner: String!, $repoName: String!) { - repository(owner: $repoOwner, name: $repoName) { - ...repository - } + repository(owner: $repoOwner, name: $repoName) { + ...repository + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/findUser.graphql b/plugins/github/resources/graphql/query/findUser.graphql index 458c1ebfbe2b..687b7e452092 100644 --- a/plugins/github/resources/graphql/query/findUser.graphql +++ b/plugins/github/resources/graphql/query/findUser.graphql @@ -3,4 +3,8 @@ query($login: String!) { __typename ...userInfo } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/getProtectionRules.graphql b/plugins/github/resources/graphql/query/getProtectionRules.graphql index f945710522fb..b164f6115adc 100644 --- a/plugins/github/resources/graphql/query/getProtectionRules.graphql +++ b/plugins/github/resources/graphql/query/getProtectionRules.graphql @@ -1,13 +1,17 @@ query($repoOwner: String!, $repoName: String!, $pageSize: Int = 100, $cursor: String) { - repository(owner: $repoOwner, name: $repoName) { - branchProtectionRules(first: $pageSize, after: $cursor) { - pageInfo { - ...pageInfo - } - nodes { - ...nodeInfo - pattern - } - } + repository(owner: $repoOwner, name: $repoName) { + branchProtectionRules(first: $pageSize, after: $cursor) { + pageInfo { + ...pageInfo + } + nodes { + ...nodeInfo + pattern + } } + } + + rateLimit { + ...rates + } } diff --git a/plugins/github/resources/graphql/query/getPullRequestTemplates.graphql b/plugins/github/resources/graphql/query/getPullRequestTemplates.graphql index df9d92922e51..6024e053ad94 100644 --- a/plugins/github/resources/graphql/query/getPullRequestTemplates.graphql +++ b/plugins/github/resources/graphql/query/getPullRequestTemplates.graphql @@ -4,4 +4,8 @@ query($repoOwner: String!, $repoName: String!) { body } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/getRepositoryCommitStatusContexts.graphql b/plugins/github/resources/graphql/query/getRepositoryCommitStatusContexts.graphql index 519556cbc90f..11c0cb82256f 100644 --- a/plugins/github/resources/graphql/query/getRepositoryCommitStatusContexts.graphql +++ b/plugins/github/resources/graphql/query/getRepositoryCommitStatusContexts.graphql @@ -35,4 +35,8 @@ query($repoOwner: String!, $repoName: String!, $oid: GitObjectID!, $pageSize: In } } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/getRepositoryCommitStatusShort.graphql b/plugins/github/resources/graphql/query/getRepositoryCommitStatusShort.graphql index 0ecf5f467bff..f1c920803a93 100644 --- a/plugins/github/resources/graphql/query/getRepositoryCommitStatusShort.graphql +++ b/plugins/github/resources/graphql/query/getRepositoryCommitStatusShort.graphql @@ -8,4 +8,8 @@ query($repoOwner: String!, $repoName: String!, $oid: GitObjectID!) { } } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/issueSearch.graphql b/plugins/github/resources/graphql/query/issueSearch.graphql index 7493682f1a9b..a0516c38f462 100644 --- a/plugins/github/resources/graphql/query/issueSearch.graphql +++ b/plugins/github/resources/graphql/query/issueSearch.graphql @@ -7,4 +7,8 @@ query issueSearch($query: String!, $pageSize: Int = 100, $cursor: String) { ...pullRequestInfoShort } } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/pullRequestCommits.graphql b/plugins/github/resources/graphql/query/pullRequestCommits.graphql index a460a2af9f25..adcdc63506d8 100644 --- a/plugins/github/resources/graphql/query/pullRequestCommits.graphql +++ b/plugins/github/resources/graphql/query/pullRequestCommits.graphql @@ -1,18 +1,22 @@ query($repoOwner: String!, $repoName: String!, $number: Int!, $pageSize: Int = 100, $cursor: String) { - repository(owner: $repoOwner, name: $repoName) { - pullRequest(number: $number) { - commits(first: $pageSize, after: $cursor) { - pageInfo { - ...pageInfo - } - nodes { - ...nodeInfo - commit { - ...commit - } - url - } - } + repository(owner: $repoOwner, name: $repoName) { + pullRequest(number: $number) { + commits(first: $pageSize, after: $cursor) { + pageInfo { + ...pageInfo } + nodes { + ...nodeInfo + commit { + ...commit + } + url + } + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/pullRequestFiles.graphql b/plugins/github/resources/graphql/query/pullRequestFiles.graphql index ab43b04a53a9..04af0f1928ee 100644 --- a/plugins/github/resources/graphql/query/pullRequestFiles.graphql +++ b/plugins/github/resources/graphql/query/pullRequestFiles.graphql @@ -1,15 +1,19 @@ query($repoOwner: String!, $repoName: String!, $number: Int!, $pageSize: Int = 100, $cursor: String) { - repository(owner: $repoOwner, name: $repoName) { - pullRequest(number: $number) { - files(first: $pageSize, after: $cursor) { - pageInfo { - ...pageInfo - } - nodes { - path - viewerViewedState - } - } + repository(owner: $repoOwner, name: $repoName) { + pullRequest(number: $number) { + files(first: $pageSize, after: $cursor) { + pageInfo { + ...pageInfo } + nodes { + path + viewerViewedState + } + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/pullRequestReviewThreads.graphql b/plugins/github/resources/graphql/query/pullRequestReviewThreads.graphql index 8d1b2a425038..6cc16c20d50d 100644 --- a/plugins/github/resources/graphql/query/pullRequestReviewThreads.graphql +++ b/plugins/github/resources/graphql/query/pullRequestReviewThreads.graphql @@ -1,14 +1,18 @@ query($repoOwner: String!, $repoName: String!, $number: Int!, $pageSize: Int = 100, $cursor: String) { - repository(owner: $repoOwner, name: $repoName) { - pullRequest(number: $number) { - reviewThreads(first: $pageSize, after: $cursor) { - pageInfo { - ...pageInfo - } - nodes { - ...pullRequestReviewThread - } - } + repository(owner: $repoOwner, name: $repoName) { + pullRequest(number: $number) { + reviewThreads(first: $pageSize, after: $cursor) { + pageInfo { + ...pageInfo } + nodes { + ...pullRequestReviewThread + } + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/resources/graphql/query/pullRequestTimeline.graphql b/plugins/github/resources/graphql/query/pullRequestTimeline.graphql index 1495be38531c..bfdeab83835d 100644 --- a/plugins/github/resources/graphql/query/pullRequestTimeline.graphql +++ b/plugins/github/resources/graphql/query/pullRequestTimeline.graphql @@ -1,283 +1,287 @@ fragment issueCommentInfo on IssueComment { - ...comment - viewerCanUpdate - viewerCanDelete - viewerCanReact + ...comment + viewerCanUpdate + viewerCanDelete + viewerCanReact } fragment prCommitInfo on PullRequestCommit { - ...nodeInfo - commit { - ...commitShort - } - url + ...nodeInfo + commit { + ...commitShort + } + url } fragment reviewDismissedEvent on ReviewDismissedEvent { - actor { - ...actorInfo - } - createdAt - dismissalMessageHTML - review { - author { - ...actorInfo - } + actor { + ...actorInfo + } + createdAt + dismissalMessageHTML + review { + author { + ...actorInfo } + } } fragment readyForReviewEvent on ReadyForReviewEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment convertToDraftEvent on ConvertToDraftEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment renamedEvent on RenamedTitleEvent { - actor { - ...actorInfo - } - previousTitle - currentTitle - createdAt + actor { + ...actorInfo + } + previousTitle + currentTitle + createdAt } fragment closedEvent on ClosedEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment reopenedEvent on ReopenedEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment mergedEvent on MergedEvent { - actor { - ...actorInfo - } - createdAt - commit { - ...commitShort - } - mergeRefName + actor { + ...actorInfo + } + createdAt + commit { + ...commitShort + } + mergeRefName } fragment assignedEvent on AssignedEvent { - actor { - ...actorInfo - } - createdAt - user { - ...userInfo - } + actor { + ...actorInfo + } + createdAt + user { + ...userInfo + } } fragment unassignedEvent on UnassignedEvent { - actor { - ...actorInfo - } - createdAt - user { - ...userInfo - } + actor { + ...actorInfo + } + createdAt + user { + ...userInfo + } } fragment labeledEvent on LabeledEvent { - actor { - ...actorInfo - } - createdAt - label { - ...labelInfo - } + actor { + ...actorInfo + } + createdAt + label { + ...labelInfo + } } fragment unlabeledEvent on UnlabeledEvent { - actor { - ...actorInfo - } - createdAt - label { - ...labelInfo - } + actor { + ...actorInfo + } + createdAt + label { + ...labelInfo + } } fragment reviewRequestedEvent on ReviewRequestedEvent { - actor { - ...actorInfo - } - createdAt - requestedReviewer { - ...pullRequestReviewerInfo - } + actor { + ...actorInfo + } + createdAt + requestedReviewer { + ...pullRequestReviewerInfo + } } fragment reviewRequestRemovedEvent on ReviewRequestRemovedEvent { - actor { - ...actorInfo - } - createdAt - requestedReviewer { - ...pullRequestReviewerInfo - } + actor { + ...actorInfo + } + createdAt + requestedReviewer { + ...pullRequestReviewerInfo + } } fragment gitRefName on Ref { - name + name } fragment baseRefForcePushedEvent on BaseRefForcePushedEvent { - actor { - ...actorInfo - } - createdAt - ref { - ...gitRefName - } + actor { + ...actorInfo + } + createdAt + ref { + ...gitRefName + } } fragment headRefForcePushedEvent on HeadRefForcePushedEvent { - actor { - ...actorInfo - } - createdAt - ref { - ...gitRefName - } - beforeCommit { - ...commitHash - } - afterCommit { - ...commitHash - } + actor { + ...actorInfo + } + createdAt + ref { + ...gitRefName + } + beforeCommit { + ...commitHash + } + afterCommit { + ...commitHash + } } fragment headRefDeletedEvent on HeadRefDeletedEvent { - actor { - ...actorInfo - } - createdAt - headRefName + actor { + ...actorInfo + } + createdAt + headRefName } fragment headRefRestoredEvent on HeadRefRestoredEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment baseRefChangedEvent on BaseRefChangedEvent { - actor { - ...actorInfo - } - createdAt + actor { + ...actorInfo + } + createdAt } fragment referenceSubject on ReferencedSubject { - __typename - ... on PullRequest { - title - number - state - isDraft - url - } - ... on Issue { - title - number - state - url - } + __typename + ... on PullRequest { + title + number + state + isDraft + url + } + ... on Issue { + title + number + state + url + } } fragment crossReferencedEvent on CrossReferencedEvent { - actor { - ...actorInfo - } - createdAt - source { - ...referenceSubject - } + actor { + ...actorInfo + } + createdAt + source { + ...referenceSubject + } } fragment connectedEvent on ConnectedEvent { - actor { - ...actorInfo - } - createdAt - subject { - ...referenceSubject - } + actor { + ...actorInfo + } + createdAt + subject { + ...referenceSubject + } } fragment disconnectedEvent on DisconnectedEvent { - actor { - ...actorInfo - } - createdAt - subject { - ...referenceSubject - } + actor { + ...actorInfo + } + createdAt + subject { + ...referenceSubject + } } query($repoOwner: String!, $repoName: String!, $number: Int!, $pageSize: Int = 100, $cursor: String, $since: DateTime) { - repository(owner: $repoOwner, name: $repoName) { - pullRequest(number: $number) { - timelineItems(first: $pageSize, after: $cursor, since: $since) { - pageInfo { - ...pageInfo - } - nodes { - __typename - ...prCommitInfo - ...pullRequestReview - ...issueCommentInfo - - ...renamedEvent - - ...closedEvent - ...reopenedEvent - ...mergedEvent - - ...assignedEvent - ...unassignedEvent - - ...labeledEvent - ...unlabeledEvent - - ...reviewRequestedEvent - ...reviewRequestRemovedEvent - - ...reviewDismissedEvent - - ...readyForReviewEvent - ...convertToDraftEvent - - ...baseRefChangedEvent - ...baseRefForcePushedEvent - - ...headRefForcePushedEvent - ...headRefDeletedEvent - ...headRefRestoredEvent - - ...crossReferencedEvent - ...connectedEvent - ...disconnectedEvent - } - } + repository(owner: $repoOwner, name: $repoName) { + pullRequest(number: $number) { + timelineItems(first: $pageSize, after: $cursor, since: $since) { + pageInfo { + ...pageInfo } + nodes { + __typename + ...prCommitInfo + ...pullRequestReview + ...issueCommentInfo + + ...renamedEvent + + ...closedEvent + ...reopenedEvent + ...mergedEvent + + ...assignedEvent + ...unassignedEvent + + ...labeledEvent + ...unlabeledEvent + + ...reviewRequestedEvent + ...reviewRequestRemovedEvent + + ...reviewDismissedEvent + + ...readyForReviewEvent + ...convertToDraftEvent + + ...baseRefChangedEvent + ...baseRefForcePushedEvent + + ...headRefForcePushedEvent + ...headRefDeletedEvent + ...headRefRestoredEvent + + ...crossReferencedEvent + ...connectedEvent + ...disconnectedEvent + } + } } + } + + rateLimit { + ...rates + } } \ No newline at end of file diff --git a/plugins/github/src/org/jetbrains/plugins/github/api/GHGQLQueries.kt b/plugins/github/src/org/jetbrains/plugins/github/api/GHGQLQueries.kt index 5d6b0f9525f8..190e6bef9a89 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/api/GHGQLQueries.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/api/GHGQLQueries.kt @@ -13,7 +13,6 @@ object GHGQLQueries { const val createPullRequest = "graphql/query/createPullRequest.graphql" const val findPullRequest = "graphql/query/findPullRequest.graphql" const val findPullRequestId = "graphql/query/findPullRequestId.graphql" - const val findOpenPullRequestsByBranches = "graphql/query/findOpenPullRequestsByBranches.graphql" const val updatePullRequest = "graphql/query/updatePullRequest.graphql" const val markPullRequestReadyForReview = "graphql/query/markPullRequestReadyForReview.graphql" const val pullRequestTimeline = "graphql/query/pullRequestTimeline.graphql" diff --git a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiContentHelper.kt b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiContentHelper.kt index d45b71e8970e..1478cb5b2230 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiContentHelper.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiContentHelper.kt @@ -46,6 +46,9 @@ object GithubApiContentHelper { @Throws(GithubJsonException::class) inline fun fromJson(string: String): T = fromJson(string, T::class.java) + @Throws(GithubJsonException::class) + internal inline fun fromJson(string: String, gqlNaming: Boolean = false): T = fromJson(string, T::class.java, gqlNaming) + @JvmStatic @Throws(GithubJsonException::class) fun fromJson(string: String, clazz: Class, gqlNaming: Boolean = false): T { diff --git a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequest.kt b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequest.kt index 866f0abb4067..8978835827e2 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequest.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequest.kt @@ -9,6 +9,7 @@ import com.intellij.util.ThrowableConvertor import org.jetbrains.plugins.github.api.data.GithubResponsePage import org.jetbrains.plugins.github.api.data.GithubSearchResult import org.jetbrains.plugins.github.api.data.graphql.GHGQLError +import org.jetbrains.plugins.github.api.data.graphql.GHGQLRateLimit import org.jetbrains.plugins.github.exceptions.GithubAuthenticationException import org.jetbrains.plugins.github.exceptions.GithubConfusingException import org.jetbrains.plugins.github.exceptions.GithubJsonException @@ -98,8 +99,7 @@ sealed class GithubApiRequest(val url: String) { url: String, private val clazz: Class, acceptMimeType: String? = GithubApiContentHelper.V3_JSON_MIME_TYPE, - ) - : Get>(url, acceptMimeType) { + ) : Get>(url, acceptMimeType) { override fun extractResult(response: GithubApiResponse): GithubResponsePage { val page = parseJsonSearchPage(response, clazz) @@ -161,20 +161,20 @@ sealed class GithubApiRequest(val url: String) { throw GithubConfusingException(errors.toString()) } + abstract fun extractResultWithCost(response: GithubApiResponse): Pair + + override fun extractResult(response: GithubApiResponse): T = + extractResultWithCost(response).first + class Parsed( url: String, requestFilePath: String, variablesObject: Any, private val clazz: Class, ) : GQLQuery(url, requestFilePath, variablesObject) { - override fun extractResult(response: GithubApiResponse): T { - val result: GraphQLResponseDTO = parseGQLResponse(response, clazz) - val data = result.data - if (data != null) return data - - val errors = result.errors - if (errors == null) error("Undefined request state - both result and errors are null") - else throwException(errors) + override fun extractResultWithCost(response: GithubApiResponse): Pair { + return parseResponse(response, clazz) + ?: throw GithubJsonException("Non-nullable entity is null or entity path is invalid") } } @@ -185,8 +185,7 @@ sealed class GithubApiRequest(val url: String) { private val clazz: Class, private vararg val pathFromData: String, ) : GQLQuery(url, requestFilePath, variablesObject) { - - override fun extractResult(response: GithubApiResponse): T { + override fun extractResultWithCost(response: GithubApiResponse): Pair { return parseResponse(response, clazz, pathFromData) ?: throw GithubJsonException("Non-nullable entity is null or entity path is invalid") } @@ -199,8 +198,8 @@ sealed class GithubApiRequest(val url: String) { private val clazz: Class, private vararg val pathFromData: String, ) : GQLQuery(url, requestFilePath, variablesObject) { - override fun extractResult(response: GithubApiResponse): T? { - return parseResponse(response, clazz, pathFromData) + override fun extractResultWithCost(response: GithubApiResponse): Pair { + return parseResponse(response, clazz, pathFromData) ?: (null to null) } } @@ -211,17 +210,25 @@ sealed class GithubApiRequest(val url: String) { private val clazz: Class, private vararg val pathFromData: String, ) : GQLQuery?>(url, requestFilePath, variablesObject) { - override fun extractResult(response: GithubApiResponse): List? = + override fun extractResultWithCost(response: GithubApiResponse): Pair?, GHGQLRateLimit?> = parseResponse(response, pathFromData) { GithubApiContentHelper.readJsonList(it.toString().reader(), clazz) - } + } ?: (null to null) } + protected fun parseResponse( + response: GithubApiResponse, + clazz: Class, + ): Pair? = + parseResponse(response, arrayOf()) { + GithubApiContentHelper.fromJson(it.toString(), clazz, true) + } + protected fun parseResponse( response: GithubApiResponse, clazz: Class, pathFromData: Array, - ): T? = + ): Pair? = parseResponse(response, pathFromData) { GithubApiContentHelper.fromJson(it.toString(), clazz, true) } @@ -230,15 +237,21 @@ sealed class GithubApiRequest(val url: String) { response: GithubApiResponse, pathFromData: Array, deserialize: (JsonNode) -> T, - ): T? { - val result: GraphQLResponseDTO = parseGQLResponse(response, JsonNode::class.java) + ): Pair? { + val result = parseGQLResponse(response, JsonNode::class.java) val data = result.data if (data != null && !data.isNull) { var node: JsonNode = data for (path in pathFromData) { node = node[path] ?: break } - if (!node.isNull) return deserialize(node) + if (!node.isNull) { + val result = deserialize(node) + val rates = data["rateLimit"] + ?.takeIf { !it.isNull } + ?.let { GithubApiContentHelper.fromJson(it.toString(), gqlNaming = true) } + return result to rates + } } val errors = result.errors if (errors == null) return null diff --git a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequestExecutor.kt b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequestExecutor.kt index e1468698c09f..134a6026e459 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequestExecutor.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/api/GithubApiRequestExecutor.kt @@ -1,6 +1,7 @@ // Copyright 2000-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. package org.jetbrains.plugins.github.api +import com.intellij.collaboration.api.dto.GraphQLRequestDTO import com.intellij.collaboration.api.httpclient.HttpClientUtil import com.intellij.collaboration.ui.SimpleEventListener import com.intellij.openapi.Disposable @@ -22,6 +23,7 @@ import org.jetbrains.annotations.TestOnly import org.jetbrains.plugins.github.api.data.GithubErrorMessage import org.jetbrains.plugins.github.exceptions.* import org.jetbrains.plugins.github.i18n.GithubBundle +import org.jetbrains.plugins.github.pullrequest.GHPRStatisticsCollector import org.jetbrains.plugins.github.util.GithubSettings import java.io.IOException import java.io.InputStream @@ -95,6 +97,7 @@ sealed class GithubApiRequestExecutor { indicator.checkCanceled() try { LOG.debug("Request: ${request.url} ${request.operationName} : Connecting") + val activity = GHPRStatisticsCollector.logApiRequestStart(request.operation) return connect { val connection = it.connection as HttpURLConnection if (request is GithubApiRequest.WithBody) { @@ -104,10 +107,28 @@ sealed class GithubApiRequestExecutor { else { LOG.debug("Request: ${connection.requestMethod} ${connection.url} : Connected") } + + GHPRStatisticsCollector.logApiResponseReceived( + activity = activity, + remaining = connection.getHeaderFieldInt("x-ratelimit-remaining", -1), + resourceName = connection.getHeaderField("x-ratelimit-resource"), + statusCode = connection.responseCode, + ) + checkResponseCode(connection) checkServerVersion(connection) + indicator.checkCanceled() - val result = request.extractResult(createResponse(it, indicator)) + + val (result, rates) = if (request is GithubApiRequest.Post.GQLQuery) { + request.extractResultWithCost(createResponse(it, indicator)) + } else { + request.extractResult(createResponse(it, indicator)) to null + } + val cost = rates?.cost + + GHPRStatisticsCollector.logApiResponseRates(request.operation, cost ?: 1, isGuessed = cost == null) + LOG.debug("Request: ${connection.requestMethod} ${connection.url} : Result extracted") result } diff --git a/plugins/github/src/org/jetbrains/plugins/github/api/data/graphql/GHGQLRateLimit.kt b/plugins/github/src/org/jetbrains/plugins/github/api/data/graphql/GHGQLRateLimit.kt new file mode 100644 index 000000000000..256f5af35cc5 --- /dev/null +++ b/plugins/github/src/org/jetbrains/plugins/github/api/data/graphql/GHGQLRateLimit.kt @@ -0,0 +1,7 @@ +// Copyright 2000-2024 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. +package org.jetbrains.plugins.github.api.data.graphql + +import com.intellij.collaboration.api.dto.GraphQLFragment + +@GraphQLFragment("/graphql/fragment/rates.graphql") +data class GHGQLRateLimit(val cost: Int) \ No newline at end of file diff --git a/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt b/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt index 470a6b751561..40dcd1919661 100644 --- a/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt +++ b/plugins/github/src/org/jetbrains/plugins/github/pullrequest/GHPRStatisticsCollector.kt @@ -1,18 +1,21 @@ // Copyright 2000-2023 JetBrains s.r.o. and contributors. Use of this source code is governed by the Apache 2.0 license. package org.jetbrains.plugins.github.pullrequest +import com.intellij.internal.statistic.StructuredIdeActivity import com.intellij.internal.statistic.eventLog.EventLogGroup import com.intellij.internal.statistic.eventLog.events.EventFields import com.intellij.internal.statistic.eventLog.events.EventPair import com.intellij.internal.statistic.service.fus.collectors.CounterUsagesCollector import com.intellij.openapi.components.Service import com.intellij.openapi.components.service +import com.intellij.openapi.diagnostic.logger import com.intellij.openapi.fileEditor.FileEditorManager import com.intellij.openapi.project.Project import com.intellij.openapi.startup.ProjectActivity import com.intellij.platform.util.coroutines.childScope import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import org.jetbrains.plugins.github.api.GithubApiRequestOperation import org.jetbrains.plugins.github.api.GithubServerPath import org.jetbrains.plugins.github.api.data.GHEnterpriseServerMeta import org.jetbrains.plugins.github.api.data.GithubPullRequestMergeMethod @@ -22,8 +25,10 @@ import org.jetbrains.plugins.github.util.GHEnterpriseServerMetadataLoader import java.util.* // TODO: Fix or replace a whole bunch of these statistics as they're no longer being collected since generalizing to Collab Tools -internal object GHPRStatisticsCollector: CounterUsagesCollector() { - private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 7) +internal object GHPRStatisticsCollector : CounterUsagesCollector() { + private val COUNTERS_GROUP = EventLogGroup("vcs.github.pullrequest.counters", 8) + + private val LOG = logger() override fun getGroup() = COUNTERS_GROUP @@ -159,6 +164,108 @@ internal object GHPRStatisticsCollector: CounterUsagesCollector() { SERVER_META_EVENT.log(project, server.toUrl(), meta.installedVersion) } //endregion + + //region: API + private enum class RateLimitResource { + Core, Search, CodeSearch, GraphQL, + IntegrationManifest, DependencySnapshots, CodeScanningUpload, ActionsRunnerRegistration, + SourceImport, + Collaborators, + Unknown; + + companion object { + fun fromString(name: String): RateLimitResource = when (name) { + "core", "rate" -> Core + "search" -> Search + "code_search" -> CodeSearch + "graphql" -> GraphQL + "integration_manifest" -> IntegrationManifest + "dependency_snapshots" -> DependencySnapshots + "code_scanning_upload" -> CodeScanningUpload + "actions_runner_registration" -> ActionsRunnerRegistration + "collaborators" -> Collaborators + "source_import" -> SourceImport + else -> Unknown + } + } + } + + private val API_REQUEST_OPERATION_FIELD = EventFields.Enum( + "operation", description = "The type of operation executed." + ) + + private val API_REQUEST_RATELIMIT_REMAINING_FIELD = EventFields.Int( + "rates_remaining", + description = "The rate limit remaining for the resource as returned by GitHub." + + "This is not a good measure of rates-used-per-request, instead it's a measure of total usage for a session/user." + ) + + private val API_REQUEST_RATELIMIT_USED_FIELD = EventFields.Int( + "rates_used", + description = "The rate limit used during the request. " + + "In the case of GraphQL rates, this is taken directly from the request. " + + "In the case of REST rates, this defaults to 1, but may be changed in the future if GitHub changes their rate limit policy." + ) + + private val API_REQUEST_RATELIMIT_GUESSED_FIELD = EventFields.Boolean( + "rates_used_guessed", + description = "Whether the rates used during the request are guessed. " + + "If `false`, the rates are pulled directly from the request." + ) + + private val API_REQUEST_RATELIMIT_RESOURCE_FIELD = EventFields.Enum( + "rates_resource", description = "The resource from which rates are taken as returned by GitHub." + ) + + private val API_REQUEST_STATUS_CODE_FIELD = EventFields.Int( + "status", description = "The status code of the response (200 = OK, 404 = Not Found, etc.)." + ) + + private val API_REQUEST_EVENT = COUNTERS_GROUP.registerIdeActivity( + "api.request", + startEventAdditionalFields = arrayOf(API_REQUEST_OPERATION_FIELD), + finishEventAdditionalFields = arrayOf(API_REQUEST_RATELIMIT_REMAINING_FIELD, API_REQUEST_RATELIMIT_RESOURCE_FIELD, API_REQUEST_STATUS_CODE_FIELD) + ) + + private val API_REQUEST_RATES_EVENT = COUNTERS_GROUP.registerEvent( + "api.rates", + API_REQUEST_OPERATION_FIELD, + API_REQUEST_RATELIMIT_USED_FIELD, + API_REQUEST_RATELIMIT_GUESSED_FIELD, + description = "Event that happens internally after we have tried to determine the rates used for a request." + ) + + fun logApiRequestStart(operation: GithubApiRequestOperation): StructuredIdeActivity = + API_REQUEST_EVENT.started(null) { + listOf(API_REQUEST_OPERATION_FIELD.with(operation)) + } + + fun logApiResponseReceived( + activity: StructuredIdeActivity, + remaining: Int, resourceName: String, + statusCode: Int, + ) { + val resource = RateLimitResource.fromString(resourceName) + + if (resource == RateLimitResource.Unknown) { + LOG.warn("Unknown rate limit resource: ${resourceName}") + } + + activity.finished { + listOf( + API_REQUEST_RATELIMIT_REMAINING_FIELD.with(remaining), + API_REQUEST_RATELIMIT_RESOURCE_FIELD.with(resource), + API_REQUEST_STATUS_CODE_FIELD.with(statusCode) + ) + } + } + + fun logApiResponseRates(operation: GithubApiRequestOperation, used: Int, isGuessed: Boolean) { + LOG.debug("Rates { operation: $operation, used: $used, isGuessed: $isGuessed }") + + API_REQUEST_RATES_EVENT.log(operation, used, isGuessed) + } + //endregion } enum class GHPRAction { @@ -176,7 +283,7 @@ enum class GHPRAction { @Service(Service.Level.PROJECT) private class GHServerVersionsCollector( private val project: Project, - parentCs: CoroutineScope + parentCs: CoroutineScope, ) { private val scope = parentCs.childScope(javaClass.name)