feat(cmd): pagination unification + fj api --paginate
Before this, only `release list` walked pages. `repo list`, `pr list` (the
non-filter branch), and `issue list` all passed `PageSize: limit` directly
to the gitea SDK — which silently caps PageSize at 50, so any request for
more than 50 results was truncated to 50 with no warning. `--limit` was
effectively a per-page hint, not a real limit.
## Changes
- New `cmd/paginate.go` — generic `paginateGitea[T any]` that walks pages
until the response is short or the limit is reached. Uses Go 1.20
generics so each list command keeps its existing typed slice without
conversion overhead.
- `repo list` — paginates ListUserRepos.
- `pr list` — paginates ListRepoPullRequests in both branches:
- With client-side filters (assignee, author, labels, search, draft,
head, base): pull all pages then filter+limit.
- Without filters: paginate up to limit.
- `issue list` — paginates ListRepoIssues. Overshoots 2x because the API
returns both issues AND PRs and we filter PRs out client-side; the
overshoot keeps us bounded but reduces the chance of returning fewer
results than `--limit`.
## `fj api --paginate`
Mirrors `gh api --paginate`:
- Follows RFC 5988 `Link: rel="next"` headers (Forgejo emits these on
list endpoints).
- Concatenates each page's JSON array into a single array via
`concatPaginatedJSON`. If a page is not a JSON array, errors with a
clear message — `--paginate` only makes sense for paginatable endpoints.
- GET-only (errors on POST/PUT/DELETE).
- Reuses the same auth and custom headers across pages; the body-size
limit applies per-page.
Refactored the request execution into a `doOnce` closure so the loop body
isn't a copy of the single-request path.
Verified live:
$ fj api 'repos/public/claude-code-proxy/commits?limit=2' \
--paginate --jq '. | length'
44
(44 = total commits in the repo, walked via Link headers from a 2-per-page
starting query.)
Out of scope for this commit, deferred:
- De-duplicating cmd/aliases.go ↔ cmd/actions.go subtrees (the type
mismatch they caused is already fixed in the prior commit; the
duplication itself is polish).
This commit is contained in:
parent
0c181df1d1
commit
133fb2fea4
5 changed files with 199 additions and 66 deletions
136
cmd/api.go
136
cmd/api.go
|
|
@ -64,7 +64,40 @@ func init() {
|
||||||
apiCmd.Flags().StringArrayP("header", "H", nil, "Add an HTTP request header (key:value)")
|
apiCmd.Flags().StringArrayP("header", "H", nil, "Add an HTTP request header (key:value)")
|
||||||
apiCmd.Flags().Bool("silent", false, "Do not print the response body")
|
apiCmd.Flags().Bool("silent", false, "Do not print the response body")
|
||||||
apiCmd.Flags().BoolP("include", "i", false, "Include HTTP response headers in the output")
|
apiCmd.Flags().BoolP("include", "i", false, "Include HTTP response headers in the output")
|
||||||
addJSONFlags(apiCmd, "Output the response as JSON; pass a comma-separated field list to project specific keys")
|
apiCmd.Flags().Bool("paginate", false, "Follow rel=\"next\" Link headers and concatenate JSON array pages (gh-compatible)")
|
||||||
|
addJSONFlags(apiCmd, "Output the response as JSON")
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseLinkHeaderNext extracts the URL with rel="next" from an RFC 5988
|
||||||
|
// Link header. Returns "" if not present.
|
||||||
|
func parseLinkHeaderNext(link string) string {
|
||||||
|
for _, segment := range strings.Split(link, ",") {
|
||||||
|
segment = strings.TrimSpace(segment)
|
||||||
|
if !strings.Contains(segment, `rel="next"`) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
start := strings.Index(segment, "<")
|
||||||
|
end := strings.Index(segment, ">")
|
||||||
|
if start >= 0 && end > start {
|
||||||
|
return segment[start+1 : end]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// concatPaginatedJSON parses each body as a JSON array and merges them.
|
||||||
|
// Errors if any body isn't an array (e.g. an object response means the
|
||||||
|
// endpoint isn't paginated and --paginate doesn't apply).
|
||||||
|
func concatPaginatedJSON(bodies [][]byte) ([]byte, error) {
|
||||||
|
merged := make([]json.RawMessage, 0)
|
||||||
|
for i, b := range bodies {
|
||||||
|
var page []json.RawMessage
|
||||||
|
if err := json.Unmarshal(b, &page); err != nil {
|
||||||
|
return nil, fmt.Errorf("--paginate requires JSON array responses; page %d wasn't an array: %w", i+1, err)
|
||||||
|
}
|
||||||
|
merged = append(merged, page...)
|
||||||
|
}
|
||||||
|
return json.Marshal(merged)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runAPI(cmd *cobra.Command, args []string) error {
|
func runAPI(cmd *cobra.Command, args []string) error {
|
||||||
|
|
@ -198,21 +231,42 @@ func runAPI(cmd *cobra.Command, args []string) error {
|
||||||
req.Header.Set(strings.TrimSpace(key), strings.TrimSpace(value))
|
req.Header.Set(strings.TrimSpace(key), strings.TrimSpace(value))
|
||||||
}
|
}
|
||||||
|
|
||||||
// Execute request via the shared client (30 s timeout, pooled
|
paginate, _ := cmd.Flags().GetBool("paginate")
|
||||||
// connections). Previous zero-value http.Client{} had no timeout, which
|
if paginate && method != http.MethodGet {
|
||||||
// pinned the CLI on a hung Forgejo indefinitely.
|
return fmt.Errorf("--paginate only supports GET requests")
|
||||||
ios.StartSpinner("Requesting...")
|
}
|
||||||
resp, err := api.SharedHTTPClient.Do(req)
|
|
||||||
ios.StopSpinner()
|
// doOnce executes a single request via the shared client (30 s timeout,
|
||||||
if err != nil {
|
// pooled connections), reads the body bounded by maxAPIResponseBytes,
|
||||||
return fmt.Errorf("failed to perform request: %w", err)
|
// and closes the body before returning. Previous zero-value http.Client{}
|
||||||
|
// had no timeout, pinning the CLI on a hung Forgejo indefinitely.
|
||||||
|
doOnce := func(r *http.Request) (body []byte, header http.Header, status int, proto string, statusText string, retErr error) {
|
||||||
|
ios.StartSpinner("Requesting...")
|
||||||
|
resp, err := api.SharedHTTPClient.Do(r)
|
||||||
|
ios.StopSpinner()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, "", "", fmt.Errorf("failed to perform request: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
|
||||||
|
body, err = io.ReadAll(io.LimitReader(resp.Body, maxAPIResponseBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, 0, "", "", fmt.Errorf("failed to read response body: %w", err)
|
||||||
|
}
|
||||||
|
if int64(len(body)) > maxAPIResponseBytes {
|
||||||
|
return nil, nil, 0, "", "", fmt.Errorf("response body exceeded %d bytes (use a different tool for bulk transfers)", maxAPIResponseBytes)
|
||||||
|
}
|
||||||
|
return body, resp.Header, resp.StatusCode, resp.Proto, resp.Status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
respBody, respHeader, statusCode, proto, status, err := doOnce(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
// Print response headers if requested
|
|
||||||
if include {
|
if include {
|
||||||
fmt.Fprintf(ios.Out, "%s %s\n", resp.Proto, resp.Status)
|
fmt.Fprintf(ios.Out, "%s %s\n", proto, status)
|
||||||
for key, values := range resp.Header {
|
for key, values := range respHeader {
|
||||||
for _, v := range values {
|
for _, v := range values {
|
||||||
fmt.Fprintf(ios.Out, "%s: %s\n", key, v)
|
fmt.Fprintf(ios.Out, "%s: %s\n", key, v)
|
||||||
}
|
}
|
||||||
|
|
@ -220,32 +274,60 @@ func runAPI(cmd *cobra.Command, args []string) error {
|
||||||
fmt.Fprintln(ios.Out)
|
fmt.Fprintln(ios.Out)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read response body with a hard ceiling so a runaway upstream can't OOM
|
if statusCode < 200 || statusCode >= 300 {
|
||||||
// the CLI. Read maxAPIResponseBytes+1 to detect overflow.
|
|
||||||
respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxAPIResponseBytes+1))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to read response body: %w", err)
|
|
||||||
}
|
|
||||||
if int64(len(respBody)) > maxAPIResponseBytes {
|
|
||||||
return fmt.Errorf("response body exceeded %d bytes (use a different tool for bulk transfers)", maxAPIResponseBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Handle non-2xx status codes
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
||||||
if !silent {
|
if !silent {
|
||||||
fmt.Fprint(ios.ErrOut, string(respBody))
|
fmt.Fprint(ios.ErrOut, string(respBody))
|
||||||
if len(respBody) > 0 && respBody[len(respBody)-1] != '\n' {
|
if len(respBody) > 0 && respBody[len(respBody)-1] != '\n' {
|
||||||
fmt.Fprintln(ios.ErrOut)
|
fmt.Fprintln(ios.ErrOut)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return fmt.Errorf("API request failed with status %d", resp.StatusCode)
|
return fmt.Errorf("API request failed with status %d", statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Follow `Link: rel="next"` headers when --paginate is set, accumulating
|
||||||
|
// each page's body. After the loop, concatPaginatedJSON merges them into
|
||||||
|
// a single JSON array. Endpoint must be paginatable (returns an array).
|
||||||
|
if paginate {
|
||||||
|
bodies := [][]byte{respBody}
|
||||||
|
nextURL := parseLinkHeaderNext(respHeader.Get("Link"))
|
||||||
|
for nextURL != "" {
|
||||||
|
nextReq, err := http.NewRequest(http.MethodGet, nextURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to build paginated request: %w", err)
|
||||||
|
}
|
||||||
|
if host.Token != "" {
|
||||||
|
nextReq.Header.Set("Authorization", "token "+host.Token)
|
||||||
|
}
|
||||||
|
nextReq.Header.Set("Accept", "application/json")
|
||||||
|
for _, h := range headers {
|
||||||
|
key, value, found := strings.Cut(h, ":")
|
||||||
|
if !found {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nextReq.Header.Set(strings.TrimSpace(key), strings.TrimSpace(value))
|
||||||
|
}
|
||||||
|
pageBody, pageHeader, pageStatus, _, _, err := doOnce(nextReq)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if pageStatus < 200 || pageStatus >= 300 {
|
||||||
|
return fmt.Errorf("paginated request to %s failed with status %d", nextURL, pageStatus)
|
||||||
|
}
|
||||||
|
bodies = append(bodies, pageBody)
|
||||||
|
nextURL = parseLinkHeaderNext(pageHeader.Get("Link"))
|
||||||
|
}
|
||||||
|
merged, err := concatPaginatedJSON(bodies)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
respBody = merged
|
||||||
}
|
}
|
||||||
|
|
||||||
if silent || len(respBody) == 0 {
|
if silent || len(respBody) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
contentType := resp.Header.Get("Content-Type")
|
contentType := respHeader.Get("Content-Type")
|
||||||
isJSON := strings.Contains(contentType, "json") || json.Valid(respBody)
|
isJSON := strings.Contains(contentType, "json") || json.Valid(respBody)
|
||||||
|
|
||||||
// If the user asked for JSON projection or jq filtering, route through
|
// If the user asked for JSON projection or jq filtering, route through
|
||||||
|
|
|
||||||
28
cmd/issue.go
28
cmd/issue.go
|
|
@ -221,13 +221,24 @@ func runIssueList(cmd *cobra.Command, args []string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
ios.StartSpinner("Fetching issues...")
|
ios.StartSpinner("Fetching issues...")
|
||||||
issues, _, err := client.ListRepoIssues(owner, name, gitea.ListIssueOption{
|
// ListRepoIssues returns both issues AND PRs (we filter PRs out below).
|
||||||
State: stateType,
|
// Pull more than `limit` so post-filter we still have `limit` real issues
|
||||||
Labels: labels,
|
// — overshoot 2x as a heuristic. paginateGitea(0, ...) would be safer
|
||||||
KeyWord: search,
|
// but spends extra round-trips; keep it bounded.
|
||||||
CreatedBy: author,
|
fetchLimit := limit * 2
|
||||||
AssignedBy: assignee,
|
if fetchLimit < 50 {
|
||||||
ListOptions: gitea.ListOptions{PageSize: limit},
|
fetchLimit = 50
|
||||||
|
}
|
||||||
|
issues, err := paginateGitea(fetchLimit, func(page, pageSize int) ([]*gitea.Issue, error) {
|
||||||
|
batch, _, err := client.ListRepoIssues(owner, name, gitea.ListIssueOption{
|
||||||
|
State: stateType,
|
||||||
|
Labels: labels,
|
||||||
|
KeyWord: search,
|
||||||
|
CreatedBy: author,
|
||||||
|
AssignedBy: assignee,
|
||||||
|
ListOptions: gitea.ListOptions{Page: page, PageSize: pageSize},
|
||||||
|
})
|
||||||
|
return batch, err
|
||||||
})
|
})
|
||||||
ios.StopSpinner()
|
ios.StopSpinner()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -240,6 +251,9 @@ func runIssueList(cmd *cobra.Command, args []string) error {
|
||||||
nonPRIssues = append(nonPRIssues, issue)
|
nonPRIssues = append(nonPRIssues, issue)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if limit > 0 && len(nonPRIssues) > limit {
|
||||||
|
nonPRIssues = nonPRIssues[:limit]
|
||||||
|
}
|
||||||
|
|
||||||
if wantJSON(cmd) {
|
if wantJSON(cmd) {
|
||||||
return outputJSON(cmd, nonPRIssues)
|
return outputJSON(cmd, nonPRIssues)
|
||||||
|
|
|
||||||
43
cmd/paginate.go
Normal file
43
cmd/paginate.go
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
// paginateGitea walks pages of a gitea SDK list method until the response
|
||||||
|
// is short (last page) or we hit limit. limit=0 means unlimited.
|
||||||
|
//
|
||||||
|
// Forgejo/Gitea caps PageSize at 50, so naive `PageSize: limit` for limit > 50
|
||||||
|
// silently truncated results across most `fj * list` commands. This helper
|
||||||
|
// centralizes the loop so every list command paginates consistently.
|
||||||
|
//
|
||||||
|
// fetch is called with (page, pageSize) and returns the items for that page.
|
||||||
|
// The 1-based `page` matches the gitea SDK convention.
|
||||||
|
func paginateGitea[T any](limit int, fetch func(page, pageSize int) ([]T, error)) ([]T, error) {
|
||||||
|
const maxPageSize = 50
|
||||||
|
pageSize := maxPageSize
|
||||||
|
if limit > 0 && limit < pageSize {
|
||||||
|
pageSize = limit
|
||||||
|
}
|
||||||
|
|
||||||
|
var all []T
|
||||||
|
for page := 1; ; page++ {
|
||||||
|
if limit > 0 && len(all) >= limit {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
batch, err := fetch(page, pageSize)
|
||||||
|
if err != nil {
|
||||||
|
return all, err
|
||||||
|
}
|
||||||
|
if len(batch) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
all = append(all, batch...)
|
||||||
|
// A short page (less than the requested size) is the conventional
|
||||||
|
// "you've reached the end" signal — saves one extra round-trip.
|
||||||
|
if len(batch) < pageSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit > 0 && len(all) > limit {
|
||||||
|
all = all[:limit]
|
||||||
|
}
|
||||||
|
return all, nil
|
||||||
|
}
|
||||||
45
cmd/pr.go
45
cmd/pr.go
|
|
@ -252,39 +252,32 @@ func runPRList(cmd *cobra.Command, args []string) error {
|
||||||
needsClientFilter := assignee != "" || author != "" || len(labels) > 0 || search != "" || draft || head != "" || base != ""
|
needsClientFilter := assignee != "" || author != "" || len(labels) > 0 || search != "" || draft || head != "" || base != ""
|
||||||
|
|
||||||
ios.StartSpinner("Fetching pull requests...")
|
ios.StartSpinner("Fetching pull requests...")
|
||||||
|
// When client-side filtering is needed, pull pages until exhausted (no
|
||||||
|
// limit) so we can apply filters; otherwise paginate up to the user's
|
||||||
|
// limit. Either way, paginate — `PageSize: limit` capped at 50 silently.
|
||||||
|
fetchPage := func(page, pageSize int) ([]*gitea.PullRequest, error) {
|
||||||
|
batch, _, err := client.ListRepoPullRequests(owner, name, gitea.ListPullRequestsOptions{
|
||||||
|
State: stateType,
|
||||||
|
ListOptions: gitea.ListOptions{Page: page, PageSize: pageSize},
|
||||||
|
})
|
||||||
|
return batch, err
|
||||||
|
}
|
||||||
var prs []*gitea.PullRequest
|
var prs []*gitea.PullRequest
|
||||||
if needsClientFilter {
|
if needsClientFilter {
|
||||||
page := 1
|
prs, err = paginateGitea(0, fetchPage) // pull all, then filter + limit
|
||||||
for {
|
if err == nil {
|
||||||
batch, _, err := client.ListRepoPullRequests(owner, name, gitea.ListPullRequestsOptions{
|
prs = filterPRs(prs, author, assignee, labels, search, draft, head, base)
|
||||||
State: stateType,
|
if limit > 0 && len(prs) > limit {
|
||||||
ListOptions: gitea.ListOptions{Page: page, PageSize: 50},
|
prs = prs[:limit]
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
ios.StopSpinner()
|
|
||||||
return fmt.Errorf("failed to list pull requests: %w", err)
|
|
||||||
}
|
}
|
||||||
prs = append(prs, batch...)
|
|
||||||
if len(batch) < 50 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
page++
|
|
||||||
}
|
|
||||||
prs = filterPRs(prs, author, assignee, labels, search, draft, head, base)
|
|
||||||
if len(prs) > limit {
|
|
||||||
prs = prs[:limit]
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
prs, _, err = client.ListRepoPullRequests(owner, name, gitea.ListPullRequestsOptions{
|
prs, err = paginateGitea(limit, fetchPage)
|
||||||
State: stateType,
|
|
||||||
ListOptions: gitea.ListOptions{PageSize: limit},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
ios.StopSpinner()
|
|
||||||
return fmt.Errorf("failed to list pull requests: %w", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
ios.StopSpinner()
|
ios.StopSpinner()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to list pull requests: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
if wantJSON(cmd) {
|
if wantJSON(cmd) {
|
||||||
return outputJSON(cmd, prs)
|
return outputJSON(cmd, prs)
|
||||||
|
|
|
||||||
13
cmd/repo.go
13
cmd/repo.go
|
|
@ -216,17 +216,18 @@ func runRepoList(cmd *cobra.Command, args []string) error {
|
||||||
return fmt.Errorf("failed to get user info: %w", err)
|
return fmt.Errorf("failed to get user info: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
repos, _, err := client.ListUserRepos(user.UserName, gitea.ListReposOptions{})
|
limit, _ := cmd.Flags().GetInt("limit")
|
||||||
|
repos, err := paginateGitea(limit, func(page, pageSize int) ([]*gitea.Repository, error) {
|
||||||
|
batch, _, err := client.ListUserRepos(user.UserName, gitea.ListReposOptions{
|
||||||
|
ListOptions: gitea.ListOptions{Page: page, PageSize: pageSize},
|
||||||
|
})
|
||||||
|
return batch, err
|
||||||
|
})
|
||||||
ios.StopSpinner()
|
ios.StopSpinner()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to list repositories: %w", err)
|
return fmt.Errorf("failed to list repositories: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
limit, _ := cmd.Flags().GetInt("limit")
|
|
||||||
if limit > 0 && len(repos) > limit {
|
|
||||||
repos = repos[:limit]
|
|
||||||
}
|
|
||||||
|
|
||||||
if wantJSON(cmd) {
|
if wantJSON(cmd) {
|
||||||
return outputJSON(cmd, repos)
|
return outputJSON(cmd, repos)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue