Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,7 @@ multi-arch images to GHCR with SLSA provenance attestation on push to
permissions section to exercise end-to-end; there's no mock/fixture
server in this repo, so manual testing against `/metrics` needs
`GITHUB_ORG`/`GITHUB_TOKEN` pointed at an actual org.

## Conditional requests

`Client.request` keeps each GET URL's last 200 (body, `ETag`, `Link`; bodies over 2 MiB and beyond 4096 URLs are not kept) and sends `If-None-Match` next time. GitHub answers 304 without counting it against the rate limit, and the cached 200 is handed back to the caller, so slow-changing reads (repo list, workflows, alerts, PR counts) are nearly free once seen, and the same URL asked by two collectors (the org stats and the adoption stats both read PR counts) is free the second time. `github_api_requests_total{result}` shows fresh vs not_modified.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ scrape.
| `github_rate_limit_remaining` | | Remaining core API budget in the scarcest active window: the one that runs out first |
| `github_rate_limit_limit` | | Total core API budget of that window |
| `github_rate_limit_window_remaining` | `reset` | Remaining budget per active core window, labelled with its reset time. GitHub counts per region, so one token can have several windows at once, and which one a call counts against depends on the endpoint. A window's series disappears once it resets |
| `github_api_requests_total` | `result` | GitHub API requests: `fresh` (a full 200, counted against the rate limit) and `not_modified` (a 304 answered from the in-memory conditional-request cache, not counted). Rate of `not_modified` over the total is the saving |
| `github_repo_open_prs` | `repo` | Number of open pull requests |
| `github_repo_ci_last_run_conclusion` | `repo`, `workflow`, `url`, `conclusion` | Always 1 - an "info" metric. `conclusion` is GitHub's own string verbatim (`success`, `failure`, `cancelled`, `skipped`, `neutral`, `timed_out`, `action_required`, `stale`), not collapsed to pass/fail here - what counts as "actually broken" is a dashboard-level call. `url` links to the run on github.com. Absent if the workflow has never run |
| `github_repo_ci_last_run_timestamp_seconds` | `repo`, `workflow` | Unix timestamp of that workflow's latest completed run |
Expand Down
29 changes: 29 additions & 0 deletions internal/collector/apirequests.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package collector

import (
"github.com/prometheus/client_golang/prometheus"
)

// APIRequests exposes how many GitHub API requests were full responses (they count against the
// rate limit) and how many were 304s answered from the conditional-request cache (they do not).
type APIRequests struct {
counts func() (fresh, notModified int64)
desc *prometheus.Desc
}

func NewAPIRequests(counts func() (fresh, notModified int64)) *APIRequests {
return &APIRequests{
counts: counts,
desc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "api", "requests_total"),
"GitHub API requests by result: fresh (a 200, counted against the rate limit) or not_modified (a 304 answered from the cache, not counted).",
[]string{"result"}, nil),
}
}

func (a *APIRequests) Describe(ch chan<- *prometheus.Desc) { ch <- a.desc }

func (a *APIRequests) Collect(ch chan<- prometheus.Metric) {
fresh, notModified := a.counts()
ch <- prometheus.MustNewConstMetric(a.desc, prometheus.CounterValue, float64(fresh), "fresh")
ch <- prometheus.MustNewConstMetric(a.desc, prometheus.CounterValue, float64(notModified), "not_modified")
}
78 changes: 78 additions & 0 deletions internal/github/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,17 @@
package github

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"sync"
"sync/atomic"
"time"
)

Expand All @@ -26,8 +29,32 @@ type Client struct {
// core budgets by reset time: GitHub counts per region, so one token has
// more than one window at once, depending on which region serves an endpoint
budgets map[int64]RateLimit

// each URL's last 200 response, so the next request can be conditional (see request)
cache map[string]cachedResponse

fresh, notModified atomic.Int64 // requests that cost rate limit vs answered 304
}

// RequestCounts is how many requests came back as a full 200 (counted against the rate limit) and
// how many as a 304 answered from the cache (not counted).
func (c *Client) RequestCounts() (fresh, notModified int64) {
return c.fresh.Load(), c.notModified.Load()
}

// cachedResponse is what a conditional request needs to answer 304 locally.
type cachedResponse struct {
etag string
link string
body []byte
}

const (
// a page this large is passed through uncached rather than held in memory
maxCachedBody = 2 << 20
maxCachedEntries = 4096
)

func NewClient(org, token string, timeout time.Duration) *Client {
return &Client{
baseURL: apiBase,
Expand All @@ -38,6 +65,10 @@ func NewClient(org, token string, timeout time.Duration) *Client {
}
}

// request is a GET with conditional caching: GitHub does not count a 304 against the rate limit, so
// the slow-changing lists (repos, workflows, alerts, PR counts) cost almost nothing once seen. The
// same URL asked twice by two collectors is also free the second time. A 304 comes back to callers
// as the cached 200, Link header included, so nothing above this function knows the difference.
func (c *Client) request(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
Expand All @@ -47,11 +78,58 @@ func (c *Client) request(ctx context.Context, url string) (*http.Response, error
req.Header.Set("Accept", "application/vnd.github+json")
req.Header.Set("X-GitHub-Api-Version", "2022-11-28")

c.mu.Lock()
cached, have := c.cache[url]
c.mu.Unlock()
if have {
req.Header.Set("If-None-Match", cached.etag)
}

resp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("requesting %s: %w", url, err)
}
c.recordRateLimit(resp.Header)
if resp.StatusCode == http.StatusNotModified {
c.notModified.Add(1)
} else {
c.fresh.Add(1)
}

switch {
case resp.StatusCode == http.StatusNotModified && have:
_ = resp.Body.Close()
resp.StatusCode, resp.Status = http.StatusOK, "200 OK"
resp.Body = io.NopCloser(bytes.NewReader(cached.body))
resp.Header.Set("Link", cached.link)
case resp.StatusCode == http.StatusOK && resp.Header.Get("ETag") != "":
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCachedBody+1))
if err != nil {
_ = resp.Body.Close()
return nil, fmt.Errorf("reading %s: %w", url, err)
}
if len(body) > maxCachedBody {
resp.Body = struct {
io.Reader
io.Closer
}{io.MultiReader(bytes.NewReader(body), resp.Body), resp.Body}
break
}
_ = resp.Body.Close()
resp.Body = io.NopCloser(bytes.NewReader(body))
c.mu.Lock()
if c.cache == nil {
c.cache = map[string]cachedResponse{}
}
if _, known := c.cache[url]; !known && len(c.cache) >= maxCachedEntries {
for k := range c.cache { // full: drop one, any one
delete(c.cache, k)
break
}
}
c.cache[url] = cachedResponse{etag: resp.Header.Get("ETag"), link: resp.Header.Get("Link"), body: body}
c.mu.Unlock()
}
return resp, nil
}

Expand Down
79 changes: 79 additions & 0 deletions internal/github/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@ package github
import (
"context"
"fmt"
"io"
"maps"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
Expand Down Expand Up @@ -175,3 +177,80 @@ func TestWorkflowsPaginates(t *testing.T) {
t.Fatalf("workflows = %+v, %v", wfs, err)
}
}

// etagServer answers 200 with an ETag and a Link header, then 304 to a matching If-None-Match.
func etagServer(hits *[]string) *httptest.Server {
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
*hits = append(*hits, r.Header.Get("If-None-Match"))
w.Header().Set("X-RateLimit-Limit", "5000")
w.Header().Set("X-RateLimit-Remaining", "4999")
w.Header().Set("X-RateLimit-Reset", "9999999999")
if r.Header.Get("If-None-Match") == `"v1"` {
w.WriteHeader(http.StatusNotModified)
return
}
w.Header().Set("ETag", `"v1"`)
w.Header().Set("Link", fmt.Sprintf(`<http://%s%s?page=2>; rel="last"`, r.Host, r.URL.Path))
_, _ = w.Write([]byte(`[{"name":"a"},{"name":"b"}]`))
}))
}

func TestRepeatedRequestsAreConditionalAndAnswerFromTheCacheOn304(t *testing.T) {
var hits []string
srv := etagServer(&hits)
defer srv.Close()
c := NewClient("org", "token", time.Second)
c.baseURL = srv.URL

for i := 0; i < 3; i++ {
repos, err := c.Repos(context.Background())
if err != nil || len(repos) != 2 || repos[1].Name != "b" {
t.Fatalf("call %d: repos = %+v, %v", i, repos, err)
}
}
// the Link header (here "last", read by OpenPRCount) survives a 304 too
n, err := c.OpenPRCount(context.Background(), "repo")
if err != nil || n != 2 {
t.Fatalf("OpenPRCount = %d, %v", n, err)
}
n, err = c.OpenPRCount(context.Background(), "repo")
if err != nil || n != 2 {
t.Fatalf("OpenPRCount (304) = %d, %v", n, err)
}
// first request of each URL is unconditional, every later one carries the ETag
want := []string{"", `"v1"`, `"v1"`, "", `"v1"`}
if len(hits) != len(want) {
t.Fatalf("If-None-Match sent = %q, want %q", hits, want)
}
for i := range want {
if hits[i] != want[i] {
t.Fatalf("request %d If-None-Match = %q, want %q (all: %q)", i, hits[i], want[i], hits)
}
}
if got := c.RateLimits(); len(got) != 1 {
t.Fatalf("a 304 must still record the rate-limit headers: %+v", got)
}
if fresh, notModified := c.RequestCounts(); fresh != 2 || notModified != 3 {
t.Fatalf("counts = %d fresh, %d not modified; want 2 and 3", fresh, notModified)
}
}

func TestOversizedBodiesAreNotCachedButStillRead(t *testing.T) {
big := strings.Repeat("x", maxCachedBody+10)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("ETag", `"big"`)
_, _ = w.Write([]byte(`["` + big + `"]`))
}))
defer srv.Close()
c := NewClient("org", "token", time.Second)
c.baseURL = srv.URL
resp, err := c.request(context.Background(), srv.URL+"/x")
if err != nil {
t.Fatal(err)
}
body, _ := io.ReadAll(resp.Body)
_ = resp.Body.Close()
if len(body) != maxCachedBody+14 || len(c.cache) != 0 {
t.Fatalf("read %d bytes, cached %d entries", len(body), len(c.cache))
}
}
1 change: 1 addition & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func main() {
}, cfg.OrgCacheTTL, cfg.OrgCacheMaxStale, log)

registry := prometheus.NewRegistry()
registry.MustRegister(collector.NewAPIRequests(client.RequestCounts))
registry.MustRegister(collector.New(cfg.GitHubOrg, runnerFetcher, orgFetcher, client.RateLimits, cfg.RunnerCacheTTL, cfg.OrgCacheTTL), feed)

if len(cfg.AdoptionRepos) > 0 {
Expand Down