diff --git a/CURATOR-REPORT.md b/CURATOR-REPORT.md index 7b84e229..3573d12b 100644 --- a/CURATOR-REPORT.md +++ b/CURATOR-REPORT.md @@ -193,8 +193,17 @@ Not bugs, so as not to waste your time: ## Waiting on someone else - **The render service runs in a container now** (`restart: unless-stopped`), so a - reboot no longer stops GIF and PPTX. Still outstanding before this fronts - reactome.org: rate limiting in front of it. + reboot no longer stops GIF and PPTX. **Rate limiting is now in front of it**, so + the condition this listed is met: nginx gives `/RenderService/` its own budget + of 2 requests a second with a burst of 8 — the service's own queue depth, past + which it is rejecting anyway. The same budget now covers the Java exporters + that draw (`/ContentService/exporter/diagram|fireworks|document|event`), which + had none; the reaction-diagram JSON on the same prefix is deliberately outside + it, being 2.9 KB in 0.17s and on every reaction page's critical path. The site-wide limit it previously fell under is + 100 a second, which is sized for page assets, not for a headless browser + drawing a 12000-pixel canvas. Crawlers on the old + `/ContentService/exporter/*` URLs are what exhausted Tomcat's heap and took the + origin down; this is the bound that stops that repeating. - **Cloudflare cache purge** — one-off, for figures cached before 2026-08-20. Nothing new is cached now. - **[#139](https://github.com/reactome/WebsiteAngular/issues/139) native cytoscape diff --git a/deploy/nginx/common/routes.conf b/deploy/nginx/common/routes.conf index 0d339f5b..acca254d 100644 --- a/deploy/nginx/common/routes.conf +++ b/deploy/nginx/common/routes.conf @@ -64,6 +64,36 @@ location ~ ^/ContentService/data/person/[^/]+/(authored|reviewed)(Pathways|React proxy_pass http://content_node; } +# The exporters that actually draw something, which are the expensive part of +# ContentService and the only part with a history of taking the origin down: +# crawlers on these URLs exhausted Tomcat's heap. They have had no budget of +# their own until now -- only the site-wide 100r/s, which is sized for page +# assets. +# +# Named individually rather than as `/exporter/`, because that prefix is not all +# one thing. `/exporter/reaction/{id}/diagram` is a 2.9 KB JSON document served +# in 0.17s and it is on the critical path for every reaction page; a render of +# R-HSA-109606 is 454 KB in 0.55s, and at full quality 7 MB. Measured, after +# writing the prefix version and realising it would throttle reaction pages to +# protect something they do not use. +# +# A regex location, so it is tried before the `/ContentService/` prefix below and +# anything not listed here -- the reaction JSON included -- falls through to it +# unbudgeted. +# +# Shares `render_rate` with /RenderService/ deliberately. The budget belongs to +# the work -- drawing a diagram -- rather than to which implementation happens +# to serve it, so a caller cannot spend it twice by alternating between the Java +# exporter and the node renderer. That also means it is already on the right +# path for #273, which moves these exports to node under these same URLs. +location ~ ^/ContentService/exporter/(diagram|fireworks|document|event)/ { + limit_req zone=render_rate burst=8 nodelay; + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://content; + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + location /ContentService/ { include /etc/nginx/common/upstream-proxy.conf; proxy_pass http://content/ContentService/; @@ -186,6 +216,25 @@ location /mcp { proxy_read_timeout 300s; } +# The render service. Routed here rather than left to fall through to the site +# server -- which proxies it via proxy.conf.js and would still work -- because +# a budget can only be applied where the path is named. +# +# `/RenderService/` with a trailing slash on both sides, so the prefix is +# stripped: the service serves `/render/{id}.{ext}` at its root. That matches +# what proxy.conf.js already does with `pathRewrite`. +location /RenderService/ { + limit_req zone=render_rate burst=8 nodelay; + include /etc/nginx/common/upstream-proxy.conf; + proxy_pass http://render/; + # A large diagram takes tens of seconds and returns megabytes. The site's + # 300s matches what the exporters needed; buffering off so a slow render + # streams rather than being held whole in nginx first. + proxy_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; +} + # The site itself. location / { include /etc/nginx/common/upstream-proxy.conf; diff --git a/deploy/nginx/dev.conf b/deploy/nginx/dev.conf index b8d57c03..a2aea2e3 100644 --- a/deploy/nginx/dev.conf +++ b/deploy/nginx/dev.conf @@ -93,6 +93,18 @@ upstream mcp { keepalive 16; } +# The headless render service, which draws diagram figures (PNG, SVG, GIF, +# PPTX, PDF) with the site's own renderer. +# +# Reached today through the site server, which proxies /RenderService from +# proxy.conf.js. Named here so nginx can put a budget in front of it before it +# ever fronts reactome.org -- which CURATOR-REPORT.md lists as the outstanding +# condition on that. +upstream render { + server 127.0.0.1:4310; + keepalive 16; +} + include /etc/nginx/common/websocket.conf; include /etc/nginx/common/cloudflare-real-ip.conf; include /etc/nginx/common/block-ai-crawlers.conf; @@ -113,6 +125,20 @@ limit_req_zone $binary_remote_addr zone=dev_rate:10m rate=100r/s; # page assets and would let one client queue analyses as fast as it can ask. limit_req_zone $binary_remote_addr zone=mcp_rate:10m rate=2r/s; +# A budget for renders, two orders of magnitude tighter than the site's. +# +# A render is a headless browser drawing a canvas -- R-HSA-109606 comes back as +# a 7 MB, 11952x6700 PNG -- so the cost per request is seconds of CPU rather +# than a file read. The site's 100r/s is sized for page assets, and crawlers on +# the old /ContentService/exporter/* URLs are what exhausted Tomcat's heap and +# took the origin down. That is the failure this exists to prevent repeating. +# +# 2r/s with a burst of 8, which is the service's own queue depth +# (RENDER_CONCURRENCY=2, RENDER_QUEUE=8): past that it is rejecting anyway, so +# a burst larger than the queue only buys a deeper pile of work to throw away. +limit_req_zone $binary_remote_addr zone=render_rate:10m rate=2r/s; + + limit_conn_zone $binary_remote_addr zone=dev_conn:10m; # The retired hostnames stay retired, and so does anything else pointed here. diff --git a/deploy/nginx/production.conf b/deploy/nginx/production.conf index 527cf3fd..42225b36 100644 --- a/deploy/nginx/production.conf +++ b/deploy/nginx/production.conf @@ -102,6 +102,18 @@ upstream mcp { keepalive 16; } +# The headless render service, which draws diagram figures (PNG, SVG, GIF, +# PPTX, PDF) with the site's own renderer. +# +# Reached today through the site server, which proxies /RenderService from +# proxy.conf.js. Named here so nginx can put a budget in front of it before it +# ever fronts reactome.org -- which CURATOR-REPORT.md lists as the outstanding +# condition on that. +upstream render { + server 127.0.0.1:4310; + keepalive 16; +} + include /etc/nginx/common/websocket.conf; include /etc/nginx/common/cloudflare-real-ip.conf; include /etc/nginx/common/block-ai-crawlers.conf; @@ -119,6 +131,20 @@ limit_req_zone $binary_remote_addr zone=prod_rate:20m rate=600r/s; # page assets and would let one client queue analyses as fast as it can ask. limit_req_zone $binary_remote_addr zone=mcp_rate:10m rate=2r/s; +# A budget for renders, two orders of magnitude tighter than the site's. +# +# A render is a headless browser drawing a canvas -- R-HSA-109606 comes back as +# a 7 MB, 11952x6700 PNG -- so the cost per request is seconds of CPU rather +# than a file read. The site's 100r/s is sized for page assets, and crawlers on +# the old /ContentService/exporter/* URLs are what exhausted Tomcat's heap and +# took the origin down. That is the failure this exists to prevent repeating. +# +# 2r/s with a burst of 8, which is the service's own queue depth +# (RENDER_CONCURRENCY=2, RENDER_QUEUE=8): past that it is rejecting anyway, so +# a burst larger than the queue only buys a deeper pile of work to throw away. +limit_req_zone $binary_remote_addr zone=render_rate:10m rate=2r/s; + + limit_conn_zone $binary_remote_addr zone=prod_conn:20m; server { diff --git a/deploy/nginx/release.conf b/deploy/nginx/release.conf index 05b7190a..8644d10b 100644 --- a/deploy/nginx/release.conf +++ b/deploy/nginx/release.conf @@ -96,6 +96,18 @@ upstream mcp { keepalive 16; } +# The headless render service, which draws diagram figures (PNG, SVG, GIF, +# PPTX, PDF) with the site's own renderer. +# +# Reached today through the site server, which proxies /RenderService from +# proxy.conf.js. Named here so nginx can put a budget in front of it before it +# ever fronts reactome.org -- which CURATOR-REPORT.md lists as the outstanding +# condition on that. +upstream render { + server 127.0.0.1:4310; + keepalive 16; +} + include /etc/nginx/common/websocket.conf; include /etc/nginx/common/cloudflare-real-ip.conf; include /etc/nginx/common/block-ai-crawlers.conf; @@ -112,6 +124,20 @@ limit_req_zone $binary_remote_addr zone=rel_rate:10m rate=100r/s; # page assets and would let one client queue analyses as fast as it can ask. limit_req_zone $binary_remote_addr zone=mcp_rate:10m rate=2r/s; +# A budget for renders, two orders of magnitude tighter than the site's. +# +# A render is a headless browser drawing a canvas -- R-HSA-109606 comes back as +# a 7 MB, 11952x6700 PNG -- so the cost per request is seconds of CPU rather +# than a file read. The site's 100r/s is sized for page assets, and crawlers on +# the old /ContentService/exporter/* URLs are what exhausted Tomcat's heap and +# took the origin down. That is the failure this exists to prevent repeating. +# +# 2r/s with a burst of 8, which is the service's own queue depth +# (RENDER_CONCURRENCY=2, RENDER_QUEUE=8): past that it is rejecting anyway, so +# a burst larger than the queue only buys a deeper pile of work to throw away. +limit_req_zone $binary_remote_addr zone=render_rate:10m rate=2r/s; + + limit_conn_zone $binary_remote_addr zone=rel_conn:10m; server {