From c622064a254858d2e1ec77235d0f7faa5ddb91cb Mon Sep 17 00:00:00 2001
From: Noah
Date: Thu, 12 Nov 2026 15:45:16 +0100
Subject: [PATCH 1/2] Make the system's memory reserve a setting the agent can
move
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The cap of all groups together was a fixed formula. The reserve for the
system is now a setting (Settings → Memory, `genco memory reserve
`), 512 MB to half the RAM, the formula stays the default;
it applies at once and is refused when a group's own limit would be
above the new cap. `genco memory` shows the split and what the groups
(now and their peak) and the system use. genco's agent may move it in
small steps when the system runs short while the groups have room, or
groups hit the cap while the reserve sits unused, and the daily health
check looks at it.
Co-Authored-By: Claude Opus 5.5
---
docs/OPERATIONS.md | 12 +-
server/src/activity.rs | 1 +
server/src/agent/instructions.md | 10 +-
server/src/agent/routines/health.md | 3 +
server/src/apply/memory.rs | 16 +-
server/src/apply/mod.rs | 10 +-
server/src/hosting.rs | 2 +-
server/src/limits.rs | 82 ++++---
server/src/main.rs | 20 ++
server/src/memory.rs | 215 ++++++++++++++++++
server/src/serve.rs | 1 +
webui/src/components/GroupLimits.svelte | 14 +-
.../components/dialogs/MemorySettings.svelte | 86 +++++++
.../components/dialogs/SettingsDialog.svelte | 3 +
webui/src/lib/api.ts | 25 +-
15 files changed, 438 insertions(+), 62 deletions(-)
create mode 100644 server/src/memory.rs
create mode 100644 webui/src/components/dialogs/MemorySettings.svelte
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index 15d75ff..09d12f8 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -438,9 +438,15 @@ now; `genco tree` has each group's `limits` (`null` is the default, its
`defaults` what an empty one means).
**All groups together.** The groups' slices are in `genco.slice`, which
-genco caps at the RAM without a reserve for the system: a quarter of the
-RAM, at least 768 MB, at most half (3.9 GB of RAM: 2.9 GB for the
-groups). So the groups' own limits may add up to far more than the RAM,
+genco caps at the RAM without the reserve for the system: by default a
+quarter of the RAM, at least 768 MB, at most half (3.9 GB of RAM: 2.9 GB
+for the groups). Settings → Memory and `genco memory` show the split and
+what the groups and the system use; the reserve is a setting there
+(`genco memory reserve 1.5G`, `default`), 512 MB to half the RAM, and
+applies at once. genco's agent moves it too, in the daily health check
+or on an alert, when the system runs short while the groups have room or
+the other way round, and says why. A new group limit is at most the cap;
+one a group already has stays, the cap holds it anyway. So the groups' own limits may add up to far more than the RAM,
groups rarely peak together; when they do, the kernel takes memory back
from them and kills within the groups, never the databases, nginx, mail
or genco. The shared services (not genco, whose children such as imports
diff --git a/server/src/activity.rs b/server/src/activity.rs
index 0cdd5f3..d6f1256 100644
--- a/server/src/activity.rs
+++ b/server/src/activity.rs
@@ -165,6 +165,7 @@ fn describe(method: &Method, path: &str) -> Option {
("POST", "/settings/mail/hosting", "Change mail hosting"),
("POST", "/settings/notify", "Change notifications"),
("POST", "/settings/model", "Change the agent's model"),
+ ("POST", "/settings/memory", "Change the memory reserve"),
("POST", "/domains/*/env", "Change environment variables"),
("POST", "/groups/*/workers", "Add worker"),
("POST", "/workers/*", "Change worker"),
diff --git a/server/src/agent/instructions.md b/server/src/agent/instructions.md
index 913ba13..c09ae5f 100644
--- a/server/src/agent/instructions.md
+++ b/server/src/agent/instructions.md
@@ -269,10 +269,18 @@ field in the panel.
`genco-*.service.d/`) or `systemctl set-property`: they override genco and are reported as a
problem; `genco group adopt-limits ` moves a slice drop-in's
limits into the settings. All groups together are capped at the RAM
- without a reserve for the system (`genco.slice`), a group's limit at
+ without the reserve for the system (`genco.slice`), a group's limit at
most that; their limits may add up to more. Base a limit on what the
group really uses (Usage, `memory.peak` of its slice), alerts come on
memory pressure and OOM kills.
+- The reserve for the system: `genco memory` shows the split and what
+ the groups and the system use, `genco memory reserve `
+ changes it within its bounds and applies. Move it yourself when the
+ split doesn't fit: raise it when the system (databases, mail, genco)
+ runs short or swaps while the groups have room, lower it when groups
+ get OOM kills at the cap while the system leaves much of its reserve
+ unused. Small steps (256-512 MB), and say what you changed and why
+ with the numbers.
- Craft with Imagick: transforms of large originals take 500-700 MB.
genco limits ImageMagick per process for groups with PHP
(MAGICK_MEMORY_LIMIT a quarter of the soft limit, at most 256 MiB, then
diff --git a/server/src/agent/routines/health.md b/server/src/agent/routines/health.md
index b533203..5449178 100644
--- a/server/src/agent/routines/health.md
+++ b/server/src/agent/routines/health.md
@@ -13,6 +13,9 @@ admin, and write a short report:
- Certificates: any domain without one, or expiring in under 30 days
(`certbot certificates`)?
- Mail: anything stuck in the queue (`postqueue -p`)?
+- Memory: `genco memory`. If the split between the groups and the
+ system doesn't fit what they use, move the reserve (see your
+ instructions) and say so.
- Updates are the weekly update round's; only say if the box needs a
reboot (`/var/run/reboot-required`).
- Open warnings: `genco attention list` shows the routines and alerts
diff --git a/server/src/apply/memory.rs b/server/src/apply/memory.rs
index 583392f..a51d55b 100644
--- a/server/src/apply/memory.rs
+++ b/server/src/apply/memory.rs
@@ -1,5 +1,5 @@
//! Memory of the whole server: all groups share `genco.slice`, capped at
-//! the RAM without a reserve for the system (limits.rs), so groups whose
+//! the RAM without the reserve for the system (memory.rs), so groups whose
//! own limits add up to more than the RAM run out among themselves. The
//! shared services get a low OOM score: when the kernel has to kill
//! outside the groups after all, it picks something else than the
@@ -28,11 +28,10 @@ const SYSTEM_SLICE: &str = "/sys/fs/cgroup/system.slice";
/// genco.slice and a drop-in with the OOM score for each protected service
/// that runs here.
-pub(super) fn memory_files() -> Vec {
- let (ram, _) = crate::limits::server_memory();
+pub(super) fn memory_files(cap_mb: u32) -> Vec {
let mut files = vec![File {
path: render::unit_path(GROUPS_SLICE),
- content: crate::limits::groups_slice_unit(ram),
+ content: crate::limits::groups_slice_unit(cap_mb),
private_to: None,
}];
for (unit, _) in protected_units() {
@@ -86,10 +85,13 @@ fn protected(unit: &str) -> bool {
/// The cap on the running slice, and the OOM score of the protected
/// services' processes right away (the drop-in counts from their next
/// start).
-pub(super) async fn memory(r: &mut Report, changed: &HashSet) {
+pub(super) async fn memory(
+ r: &mut Report,
+ changed: &HashSet,
+ cap_mb: u32,
+) {
if changed.contains(&render::unit_path(GROUPS_SLICE)) {
- let (ram, _) = crate::limits::server_memory();
- let cap = format!("MemoryMax={}M", crate::limits::groups_cap(ram));
+ let cap = format!("MemoryMax={cap_mb}M");
if r.run(&[
"systemctl",
"set-property",
diff --git a/server/src/apply/mod.rs b/server/src/apply/mod.rs
index 80b286f..8b61212 100644
--- a/server/src/apply/mod.rs
+++ b/server/src/apply/mod.rs
@@ -151,6 +151,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report {
return r;
};
let blocked = blocked(&mut r, db);
+ let cap = crate::memory::groups_cap(&db.conn());
let old_names = crate::panel::old_names(&db.conn());
relay_sasl(&mut r, db).await;
@@ -161,7 +162,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report {
deploy_key(&mut r, g).await;
}
}
- sync(&mut r, cfg, &groups, &blocked, &old_names).await;
+ sync(&mut r, cfg, &groups, &blocked, &old_names, cap).await;
databases(&mut r, &groups).await;
// with a new certificate the sites get https, the apps the mail host
@@ -170,7 +171,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report {
for g in &mut groups {
g.mail_host = host.clone();
}
- sync(&mut r, cfg, &groups, &blocked, &old_names).await;
+ sync(&mut r, cfg, &groups, &blocked, &old_names, cap).await;
}
// genco doesn't touch them, an admin moves them into the group's
@@ -192,6 +193,7 @@ async fn sync(
groups: &[Group],
blocked: &[String],
old_names: &[String],
+ cap_mb: u32,
) {
let certs = host::certs(groups);
let mut files: Vec = groups
@@ -199,7 +201,7 @@ async fn sync(
.flat_map(|g| render::group_files(g, &certs))
.collect();
files.extend(panel_files(cfg, old_names));
- files.extend(memory_files());
+ files.extend(memory_files(cap_mb));
files.push(firewall_file(cfg, groups, blocked));
files.push(File {
path: render::RESERVED_PORTS.into(),
@@ -225,7 +227,7 @@ async fn sync(
r.run(&["systemctl", "daemon-reload"]).await;
}
- memory(r, &changed).await;
+ memory(r, &changed, cap_mb).await;
for g in groups {
set_limits(r, g, &changed).await;
start_units(r, g, &changed).await;
diff --git a/server/src/hosting.rs b/server/src/hosting.rs
index a6d5570..f1bf4b4 100644
--- a/server/src/hosting.rs
+++ b/server/src/hosting.rs
@@ -411,7 +411,7 @@ pub fn set_limits(conn: &Connection, id: i64, limits: &Limits) -> Result<()> {
)
.optional()?
.flatten();
- limits::check_here(limits, kept)?;
+ limits::check_here(limits, crate::memory::reserve(conn), kept)?;
conn.execute(
"UPDATE domain_groups SET memory_max_mb = ?1, memory_high_mb = ?2,
cpu_percent = ?3, tasks_max = ?4 WHERE id = ?5",
diff --git a/server/src/limits.rs b/server/src/limits.rs
index de59a32..a285c41 100644
--- a/server/src/limits.rs
+++ b/server/src/limits.rs
@@ -77,27 +77,24 @@ impl Limits {
}
}
- /// Within what the server has: `ram_mb` and `cores`. A group may
- /// have at most what all groups together may use (`groups_cap`); a
- /// memory limit it already has (`kept_mb`) stays allowed, the cap of
- /// all groups holds it anyway.
+ /// Within what the server has: `cap_mb`, what all groups may use
+ /// together (`groups_cap`), and `cores`. A memory limit the group
+ /// already has (`kept_mb`) stays allowed, the cap of all groups holds
+ /// it anyway.
pub fn check(
&self,
- ram_mb: u32,
+ cap_mb: u32,
cores: u32,
kept_mb: Option,
) -> Result<()> {
let max = self.memory_max();
- let cap = groups_cap(ram_mb);
if self.memory_max_mb.is_some_and(|m| Some(m) != kept_mb)
- && !(128..=cap).contains(&max)
+ && !(128..=cap_mb).contains(&max)
{
return Err(Error::Invalid(format!(
"the memory limit is 128 MB to {}, what all groups may use \
- together (the server's {} without {} for the system)",
- size(cap),
- size(ram_mb),
- size(reserve(ram_mb))
+ together (`genco memory`)",
+ size(cap_mb)
)));
}
if !(64..=max).contains(&self.memory_high()) {
@@ -154,18 +151,14 @@ pub fn with_defaults(
s: S,
) -> std::result::Result {
#[derive(Serialize)]
- #[serde(rename_all = "camelCase")]
struct Shown<'a> {
#[serde(flatten)]
limits: &'a Limits,
defaults: Defaults,
- /// what all groups may use together, the most one may have
- groups_cap_mb: u32,
}
Shown {
limits,
defaults: limits.defaults(),
- groups_cap_mb: groups_cap(server_memory().0),
}
.serialize(s)
}
@@ -252,29 +245,42 @@ fn cores() -> u32 {
std::thread::available_parallelism().map_or(1, |n| n.get() as u32)
}
-/// Checks the limits against this server, `kept_mb` is the memory limit
-/// the group has now.
-pub fn check_here(limits: &Limits, kept_mb: Option) -> Result<()> {
- limits.check(server_memory().0, cores(), kept_mb)
+/// Checks the limits against this server with the system's `reserve_mb`,
+/// `kept_mb` is the memory limit the group has now.
+pub fn check_here(
+ limits: &Limits,
+ reserve_mb: u32,
+ kept_mb: Option,
+) -> Result<()> {
+ limits.check(server_memory().0 - reserve_mb, cores(), kept_mb)
}
+/// The least an admin or the agent may keep for the system.
+pub const MIN_RESERVE_MB: u32 = 512;
+
/// The memory genco keeps for the system (the databases, nginx, mail,
-/// genco itself): a quarter of the RAM, at least 768 MB, at most half.
-pub fn reserve(ram_mb: u32) -> u32 {
+/// genco itself) unless set (memory.rs): a quarter of the RAM, at least
+/// 768 MB, at most half.
+pub fn default_reserve(ram_mb: u32) -> u32 {
(ram_mb / 4).max(768).min(ram_mb / 2)
}
+/// What a reserve may be: `MIN_RESERVE_MB` to half the RAM.
+pub fn reserve_bounds(ram_mb: u32) -> (u32, u32) {
+ (MIN_RESERVE_MB.min(ram_mb / 2), ram_mb / 2)
+}
+
/// What all groups together may use (`genco.slice`): the RAM without the
-/// `reserve`. Their own limits may add up to much more, they rarely peak
+/// reserve. Their own limits may add up to much more, they rarely peak
/// together; when they do, a group runs out, never the system.
-pub fn groups_cap(ram_mb: u32) -> u32 {
- ram_mb - reserve(ram_mb)
+pub fn groups_cap(ram_mb: u32, reserve_mb: u32) -> u32 {
+ ram_mb.saturating_sub(reserve_mb)
}
/// The unit of the slice all groups' slices are in.
-pub fn groups_slice_unit(ram_mb: u32) -> String {
+pub fn groups_slice_unit(cap_mb: u32) -> String {
format!(
- "{}: genco sets it from the RAM
+ "{}: genco memory reserve changes it
[Unit]
Description=All groups of genco
@@ -282,7 +288,7 @@ Description=All groups of genco
MemoryMax={}
",
crate::render::HEADER,
- unit_size(groups_cap(ram_mb))
+ unit_size(cap_mb)
)
}
@@ -533,7 +539,7 @@ mod tests {
#[test]
fn bounds_follow_the_server() {
- let ok = |l: Limits| l.check(4000, 2, None).is_ok();
+ let ok = |l: Limits| l.check(3000, 2, None).is_ok();
assert!(ok(Limits::default()));
let mem = |mb| Limits {
memory_max_mb: Some(mb),
@@ -545,8 +551,8 @@ mod tests {
assert!(ok(mem(3000)));
assert!(!ok(mem(3001)));
// one it had before the cap stays
- assert!(mem(3500).check(4000, 2, Some(3500)).is_ok());
- assert!(mem(3600).check(4000, 2, Some(3500)).is_err());
+ assert!(mem(3500).check(3000, 2, Some(3500)).is_ok());
+ assert!(mem(3600).check(3000, 2, Some(3500)).is_err());
// a soft limit above the max
assert!(!ok(Limits {
memory_high_mb: Some(2048),
@@ -570,13 +576,15 @@ mod tests {
#[test]
fn the_system_keeps_a_reserve() {
- assert_eq!(reserve(4000), 1000);
- assert_eq!(groups_cap(4000), 3000);
- assert_eq!(reserve(2048), 768);
- assert_eq!(reserve(1024), 512);
- assert_eq!(reserve(16384), 4096);
- assert!(groups_slice_unit(4000).contains("MemoryMax=3000M\n"));
- assert!(groups_slice_unit(4096).contains("MemoryMax=3G\n"));
+ assert_eq!(default_reserve(4000), 1000);
+ assert_eq!(groups_cap(4000, 1000), 3000);
+ assert_eq!(default_reserve(2048), 768);
+ assert_eq!(default_reserve(1024), 512);
+ assert_eq!(default_reserve(16384), 4096);
+ assert_eq!(reserve_bounds(4000), (512, 2000));
+ assert_eq!(reserve_bounds(800), (400, 400));
+ assert!(groups_slice_unit(3000).contains("MemoryMax=3000M\n"));
+ assert!(groups_slice_unit(3072).contains("MemoryMax=3G\n"));
}
#[test]
diff --git a/server/src/main.rs b/server/src/main.rs
index d4d6841..354f9c7 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -31,6 +31,7 @@ mod llm;
mod mail;
mod mailbox;
mod mailfilter;
+mod memory;
mod notify;
mod panel;
mod peers;
@@ -167,6 +168,10 @@ pub(crate) enum Cmd {
/// Block addresses from the sites, mail and workers (never ssh)
#[clap(subcommand)]
Firewall(firewall::FirewallCmd),
+ /// How the RAM is split between the groups and the system; change the
+ /// reserve for the system
+ #[clap(subcommand)]
+ Memory(memory::MemoryCmd),
/// The routines and alerts that still need attention: list them,
/// resolve one that is taken care of
#[clap(subcommand)]
@@ -248,6 +253,7 @@ impl Cmd {
| Cmd::Firewall(firewall::FirewallCmd::List)
| Cmd::Activity { .. }
| Cmd::Attention(conversations::AttentionCmd::List)
+ | Cmd::Memory(memory::MemoryCmd::Show)
| Cmd::Update { .. }
| Cmd::Runners(RunnersCmd::List)
| Cmd::Group(cli::GroupCmd::DeployKey { .. })
@@ -403,6 +409,20 @@ async fn run(args: Args) -> Result<(), String> {
}
Cmd::Docs { .. } => unreachable!("read before the config"),
Cmd::User(cmd) => user(cfg, cmd),
+ Cmd::Memory(memory::MemoryCmd::Show) => {
+ let db = Db::open(&cfg.db_path())?;
+ let s = memory::status(&db.conn()).map_err(|e| e.to_string())?;
+ println!("{}", s.text());
+ Ok(())
+ }
+ Cmd::Memory(memory::MemoryCmd::Reserve { size }) => {
+ let reserve =
+ limits::parse_size(&size).map_err(|e| e.to_string())?;
+ let db = Db::open(&cfg.db_path())?;
+ memory::set_reserve(&db.conn(), reserve)
+ .map_err(|e| e.to_string())?;
+ apply_cmd(cfg).await
+ }
Cmd::Attention(cmd) => {
edit(cfg, |_, conn| conversations::attention(conn, cmd))
}
diff --git a/server/src/memory.rs b/server/src/memory.rs
new file mode 100644
index 0000000..80e576e
--- /dev/null
+++ b/server/src/memory.rs
@@ -0,0 +1,215 @@
+//! How the RAM is split between the groups and the system: all groups
+//! share `genco.slice`, capped at the RAM without the reserve for the
+//! system (apply/memory.rs). The reserve is a setting (Settings → Memory,
+//! `genco memory reserve`), a quarter of the RAM by default; admins and
+//! genco's agent move it within `limits::reserve_bounds` when the split
+//! no longer fits what the groups and the system really use.
+
+use std::sync::Arc;
+
+use axum::extract::State;
+use axum::routing::get;
+use axum::{Json, Router};
+use rusqlite::Connection;
+use serde::{Deserialize, Serialize};
+
+use crate::AppState;
+use crate::apply;
+use crate::config::Config;
+use crate::db::{Db, set_setting};
+use crate::error::{Error, Result};
+use crate::events::{Event, Events};
+use crate::limits;
+use crate::users::Authed;
+
+const GROUPS_CGROUP: &str = "/sys/fs/cgroup/genco.slice";
+
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Setting {
+ /// `None` is the default
+ reserve_mb: Option,
+}
+
+/// The reserve for the system on this server, set or the default.
+pub fn reserve(conn: &Connection) -> u32 {
+ let (ram, _) = limits::server_memory();
+ let set: Setting = crate::db::get_json(conn, "memory").unwrap_or_default();
+ let (min, max) = limits::reserve_bounds(ram);
+ set.reserve_mb
+ .map(|r| r.clamp(min, max))
+ .unwrap_or_else(|| limits::default_reserve(ram))
+}
+
+/// What all groups together may use on this server.
+pub fn groups_cap(conn: &Connection) -> u32 {
+ limits::groups_cap(limits::server_memory().0, reserve(conn))
+}
+
+/// Sets the reserve, `None` for the default; apply makes it hold.
+pub fn set_reserve(conn: &Connection, reserve_mb: Option) -> Result<()> {
+ let (ram, _) = limits::server_memory();
+ let (min, max) = limits::reserve_bounds(ram);
+ if let Some(r) = reserve_mb
+ && !(min..=max).contains(&r)
+ {
+ return Err(Error::Invalid(format!(
+ "the reserve is {min} MB to {max} MB, half the RAM"
+ )));
+ }
+ let set = Setting { reserve_mb };
+ set_setting(conn, "memory", &serde_json::to_string(&set).unwrap())?;
+ Ok(())
+}
+
+/// The split and what the groups and the system use, for the panel, the
+/// CLI and the agent's decisions.
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Status {
+ pub ram_mb: u32,
+ pub swap_mb: u32,
+ pub reserve_mb: u32,
+ /// the reserve is the default, not set
+ pub default: bool,
+ pub default_reserve_mb: u32,
+ pub min_reserve_mb: u32,
+ pub max_reserve_mb: u32,
+ /// what all groups may use together
+ pub cap_mb: u32,
+ /// the groups' own limits added up (the default for those without)
+ pub limits_mb: u32,
+ pub groups_now_mb: Option,
+ /// since the groups' slice started (the last boot, mostly)
+ pub groups_peak_mb: Option,
+ /// everything outside the groups: the RAM in use without them
+ pub system_now_mb: Option,
+}
+
+pub fn status(conn: &Connection) -> Result {
+ let (ram, swap) = limits::server_memory();
+ let set: Setting = crate::db::get_json(conn, "memory").unwrap_or_default();
+ let reserve_mb = reserve(conn);
+ let (min, max) = limits::reserve_bounds(ram);
+ let read = |file: &str| -> Option {
+ let v =
+ std::fs::read_to_string(format!("{GROUPS_CGROUP}/{file}")).ok()?;
+ Some((v.trim().parse::().ok()? >> 20) as u32)
+ };
+ let groups_now = read("memory.current");
+ let info = std::fs::read_to_string("/proc/meminfo").unwrap_or_default();
+ let used = crate::stats::meminfo(&info, "MemTotal:")
+ .zip(crate::stats::meminfo(&info, "MemAvailable:"))
+ .map(|(t, a)| ((t - a) >> 20) as u32);
+ Ok(Status {
+ ram_mb: ram,
+ swap_mb: swap,
+ reserve_mb,
+ default: set.reserve_mb.is_none(),
+ default_reserve_mb: limits::default_reserve(ram),
+ min_reserve_mb: min,
+ max_reserve_mb: max,
+ cap_mb: limits::groups_cap(ram, reserve_mb),
+ limits_mb: crate::hosting::groups(conn)?
+ .iter()
+ .map(|g| g.limits.memory_max())
+ .sum(),
+ groups_now_mb: groups_now,
+ groups_peak_mb: read("memory.peak"),
+ system_now_mb: used.zip(groups_now).map(|(u, g)| u.saturating_sub(g)),
+ })
+}
+
+impl Status {
+ /// For `genco memory`.
+ pub fn text(&self) -> String {
+ let mb = |v: Option| v.map_or("?".into(), |v| format!("{v} MB"));
+ format!(
+ "RAM {} MB, swap {} MB\n\
+ reserve for the system {} MB ({}; {} to {} MB)\n\
+ all groups together at most {} MB, their own limits add up to \
+ {} MB\n\
+ groups now {}, at most {} since their slice started\n\
+ system now {}",
+ self.ram_mb,
+ self.swap_mb,
+ self.reserve_mb,
+ if self.default {
+ "the default".to_string()
+ } else {
+ format!("set, the default is {} MB", self.default_reserve_mb)
+ },
+ self.min_reserve_mb,
+ self.max_reserve_mb,
+ self.cap_mb,
+ self.limits_mb,
+ mb(self.groups_now_mb),
+ mb(self.groups_peak_mb),
+ mb(self.system_now_mb),
+ )
+ }
+}
+
+/// `genco memory`: the split, or a new reserve.
+#[derive(Debug, clap::Parser)]
+pub enum MemoryCmd {
+ /// Show the split between the groups and the system and what each uses
+ Show,
+ /// Keep this much for the system (2G, 1536M, 1536) or `default`;
+ /// applies at once
+ Reserve { size: String },
+}
+
+async fn get_route(
+ _authed: Authed,
+ State(db): State,
+) -> Result> {
+ Ok(Json(status(&db.conn())?))
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct SetReq {
+ /// `None` for the default
+ reserve_mb: Option,
+}
+
+/// Sets the reserve and applies, genco.slice gets the new cap at once.
+async fn set_route(
+ _authed: Authed,
+ State(cfg): State>,
+ State(db): State,
+ State(events): State,
+ Json(req): Json,
+) -> Result> {
+ set_reserve(&db.conn(), req.reserve_mb)?;
+ let report = apply::apply(&cfg, &db).await;
+ events.send(Event::Tree);
+ Ok(Json(report))
+}
+
+pub fn routes() -> Router {
+ Router::new().route("/settings/memory", get(get_route).post(set_route))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn the_reserve_is_a_setting_within_bounds() {
+ let db = Db::memory();
+ let conn = db.conn();
+ let (ram, _) = limits::server_memory();
+ assert_eq!(reserve(&conn), limits::default_reserve(ram));
+
+ let (min, max) = limits::reserve_bounds(ram);
+ assert!(set_reserve(&conn, Some(min - 1)).is_err());
+ assert!(set_reserve(&conn, Some(max + 1)).is_err());
+ set_reserve(&conn, Some(min)).unwrap();
+ assert_eq!(reserve(&conn), min);
+ assert!(!status(&conn).unwrap().default);
+ set_reserve(&conn, None).unwrap();
+ assert!(status(&conn).unwrap().default);
+ }
+}
diff --git a/server/src/serve.rs b/server/src/serve.rs
index f7a3a7f..542ba53 100644
--- a/server/src/serve.rs
+++ b/server/src/serve.rs
@@ -222,6 +222,7 @@ pub(crate) async fn serve(cfg: Config) -> Result<(), String> {
.merge(peers::routes())
.merge(proposals::routes())
.merge(quarantine::routes())
+ .merge(memory::routes())
.merge(approvals::routes())
.merge(update::routes())
.merge(appenv::routes())
diff --git a/webui/src/components/GroupLimits.svelte b/webui/src/components/GroupLimits.svelte
index ac65501..8fcd46a 100644
--- a/webui/src/components/GroupLimits.svelte
+++ b/webui/src/components/GroupLimits.svelte
@@ -7,6 +7,7 @@
type Group,
type Limits,
type MemoryUsage,
+ type MemoryStatus,
type Stats,
} from '@/lib/api';
import Form from '@/components/win/Form.svelte';
@@ -28,6 +29,11 @@
let usage = $state({ memory: null, peak: null, limit: null });
const defaults = $derived(group.limits.defaults);
+ // what all groups may use together, Settings → Memory
+ let capMb = $state(null);
+ get('/settings/memory')
+ .then((m: MemoryStatus) => (capMb = m.capMb))
+ .catch(() => {});
$effect(() => {
get(`/stats?scope=group:${group.id}&range=24h`)
@@ -64,10 +70,10 @@
? ` of ${formatBytes(usage.limit)}`
: ''}, at most {formatBytes(usage.peak ?? 0)} in the last 24 hours.
{/if}
- All groups together may use {formatBytes(
- group.limits.groupsCapMb * 1024 * 1024,
- )}, the rest of the RAM stays for the databases, mail and genco; their
- limits may add up to more.
+ {#if capMb !== null}
+ All groups together may use {formatBytes(capMb * 1024 * 1024)} (Settings
+ → Memory), their limits may add up to more.
+ {/if}
+
+ genco's agent moves it too when the split doesn't fit what the groups
+ and the system use, and says why.
+
+{/if}
+
+
diff --git a/webui/src/components/dialogs/SettingsDialog.svelte b/webui/src/components/dialogs/SettingsDialog.svelte
index 3c53e90..6193c8b 100644
--- a/webui/src/components/dialogs/SettingsDialog.svelte
+++ b/webui/src/components/dialogs/SettingsDialog.svelte
@@ -5,6 +5,7 @@
import BackupSettings from './BackupSettings.svelte';
import FirewallSettings from './FirewallSettings.svelte';
import MailSettings from './MailSettings.svelte';
+ import MemorySettings from './MemorySettings.svelte';
import NotifySettings from './NotifySettings.svelte';
import RunnerSettings from './RunnerSettings.svelte';
import UpdateSettings from './UpdateSettings.svelte';
@@ -19,6 +20,7 @@
{ id: 'update', label: 'Updates' },
{ id: 'runners', label: 'Runners' },
{ id: 'firewall', label: 'Firewall' },
+ { id: 'memory', label: 'Memory' },
];
let tab = $state('mail');
@@ -34,6 +36,7 @@
+
diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts
index 6c5fe22..17a913b 100644
--- a/webui/src/lib/api.ts
+++ b/webui/src/lib/api.ts
@@ -185,11 +185,7 @@ export type Group = {
workers: Worker[];
schedules: Schedule[];
php: PhpSettings;
- limits: Limits & {
- defaults: LimitDefaults;
- /** what all groups may use together, the most one may have */
- groupsCapMb: number;
- };
+ limits: Limits & { defaults: LimitDefaults };
/** stopped because it looks compromised */
quarantine: Quarantine | null;
};
@@ -203,6 +199,25 @@ export type Quarantine = {
at: number;
};
+/** how the RAM is split between the groups and the system (memory.rs) */
+export type MemoryStatus = {
+ ramMb: number;
+ swapMb: number;
+ reserveMb: number;
+ /** the reserve is the default, not set */
+ default: boolean;
+ defaultReserveMb: number;
+ minReserveMb: number;
+ maxReserveMb: number;
+ /** what all groups may use together */
+ capMb: number;
+ /** the groups' own limits added up */
+ limitsMb: number;
+ groupsNowMb: number | null;
+ groupsPeakMb: number | null;
+ systemNowMb: number | null;
+};
+
/** what a group's slice may use, null is the default */
export type Limits = {
memoryMaxMb: number | null;
From 714cf034ad49ecca5fd4e0f69027866dbd03bf9c Mon Sep 17 00:00:00 2001
From: Noah
Date: Thu, 12 Nov 2026 15:56:35 +0100
Subject: [PATCH 2/2] Fix review findings of the memory reserve
- `genco memory` works without a subcommand, as the docs, the agent's
instructions and the health check use it; `reserve` is long-running
like apply, so a closed pipe doesn't kill it halfway.
- A reserve that leaves the groups less than they use now without their
page cache is refused: the kernel would kill in the groups at once.
- "groups now" leaves out their page cache like the system's number, so
the system's use is no longer undercounted.
- Limit checks take the cap from memory::groups_cap instead of a second
unchecked subtraction.
- Offline tests for the bounds and the clamping instead of the host's
RAM.
- Webui: requests in lib/api.ts, applyErrors shared with the quarantine,
newest() for the reload, the tab reloads after a failed apply, the
input takes digits only.
- PLAN.md entry, OPERATIONS.md wrapped.
Co-Authored-By: Claude Opus 5.5
---
docs/OPERATIONS.md | 8 +-
docs/PLAN.md | 4 +
server/src/hosting.rs | 2 +-
server/src/limits.rs | 8 +-
server/src/main.rs | 15 ++-
server/src/memory.rs | 124 +++++++++++++-----
webui/src/components/GroupLimits.svelte | 6 +-
.../components/dialogs/MemorySettings.svelte | 43 +++---
webui/src/lib/api.ts | 19 +++
webui/src/pages/Group.svelte | 6 +-
10 files changed, 160 insertions(+), 75 deletions(-)
diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md
index 09d12f8..6043c1d 100644
--- a/docs/OPERATIONS.md
+++ b/docs/OPERATIONS.md
@@ -445,9 +445,11 @@ what the groups and the system use; the reserve is a setting there
(`genco memory reserve 1.5G`, `default`), 512 MB to half the RAM, and
applies at once. genco's agent moves it too, in the daily health check
or on an alert, when the system runs short while the groups have room or
-the other way round, and says why. A new group limit is at most the cap;
-one a group already has stays, the cap holds it anyway. So the groups' own limits may add up to far more than the RAM,
-groups rarely peak together; when they do, the kernel takes memory back
+the other way round, and says why. A reserve that leaves the groups
+less than they use now without their page cache is refused. A new group
+limit is at most the cap; one a group already has stays, the cap holds
+it anyway. So the groups' own limits may add up to far more than the
+RAM, groups rarely peak together; when they do, the kernel takes memory back
from them and kills within the groups, never the databases, nginx, mail
or genco. The shared services (not genco, whose children such as imports
and the agent's commands would inherit it) also get a low OOM score
diff --git a/docs/PLAN.md b/docs/PLAN.md
index 181e584..2ce660d 100644
--- a/docs/PLAN.md
+++ b/docs/PLAN.md
@@ -307,6 +307,10 @@ terminal.
the RAM without a reserve (genco.slice), shared services with a
low OOM score, alerts on memory pressure and OOM kills instead of
the sum of limits, the group's usage and 24 h peak beside its limit
+- [x] the reserve for the system as a setting (operator 2026-11-12):
+ Settings → Memory, `genco memory [reserve ]`,
+ 512 MB to half the RAM, moved by the agent when the split doesn't
+ fit what the groups and the system use
- [x] drop-ins that override genco's units reported by apply, the minute
check and the config view; `genco group adopt-limits`
- [x] ImageMagick limits for groups with PHP, from the group's memory
diff --git a/server/src/hosting.rs b/server/src/hosting.rs
index f1bf4b4..04779a9 100644
--- a/server/src/hosting.rs
+++ b/server/src/hosting.rs
@@ -411,7 +411,7 @@ pub fn set_limits(conn: &Connection, id: i64, limits: &Limits) -> Result<()> {
)
.optional()?
.flatten();
- limits::check_here(limits, crate::memory::reserve(conn), kept)?;
+ limits::check_here(limits, crate::memory::groups_cap(conn), kept)?;
conn.execute(
"UPDATE domain_groups SET memory_max_mb = ?1, memory_high_mb = ?2,
cpu_percent = ?3, tasks_max = ?4 WHERE id = ?5",
diff --git a/server/src/limits.rs b/server/src/limits.rs
index a285c41..da05b13 100644
--- a/server/src/limits.rs
+++ b/server/src/limits.rs
@@ -245,14 +245,14 @@ fn cores() -> u32 {
std::thread::available_parallelism().map_or(1, |n| n.get() as u32)
}
-/// Checks the limits against this server with the system's `reserve_mb`,
-/// `kept_mb` is the memory limit the group has now.
+/// Checks the limits against this server with what all groups may use
+/// (`cap_mb`), `kept_mb` is the memory limit the group has now.
pub fn check_here(
limits: &Limits,
- reserve_mb: u32,
+ cap_mb: u32,
kept_mb: Option,
) -> Result<()> {
- limits.check(server_memory().0 - reserve_mb, cores(), kept_mb)
+ limits.check(cap_mb, cores(), kept_mb)
}
/// The least an admin or the agent may keep for the system.
diff --git a/server/src/main.rs b/server/src/main.rs
index 354f9c7..7d13f88 100644
--- a/server/src/main.rs
+++ b/server/src/main.rs
@@ -170,8 +170,10 @@ pub(crate) enum Cmd {
Firewall(firewall::FirewallCmd),
/// How the RAM is split between the groups and the system; change the
/// reserve for the system
- #[clap(subcommand)]
- Memory(memory::MemoryCmd),
+ Memory {
+ #[clap(subcommand)]
+ cmd: Option,
+ },
/// The routines and alerts that still need attention: list them,
/// resolve one that is taken care of
#[clap(subcommand)]
@@ -224,6 +226,7 @@ impl Cmd {
| Cmd::Setup { .. }
| Cmd::Apply | Cmd::Deploy { .. }
| Cmd::Backup { .. }
+ | Cmd::Memory { cmd: Some(_) }
| Cmd::Restic { .. }
| Cmd::Update { .. }
| Cmd::Panel(_)
@@ -253,7 +256,7 @@ impl Cmd {
| Cmd::Firewall(firewall::FirewallCmd::List)
| Cmd::Activity { .. }
| Cmd::Attention(conversations::AttentionCmd::List)
- | Cmd::Memory(memory::MemoryCmd::Show)
+ | Cmd::Memory { cmd: None }
| Cmd::Update { .. }
| Cmd::Runners(RunnersCmd::List)
| Cmd::Group(cli::GroupCmd::DeployKey { .. })
@@ -409,13 +412,15 @@ async fn run(args: Args) -> Result<(), String> {
}
Cmd::Docs { .. } => unreachable!("read before the config"),
Cmd::User(cmd) => user(cfg, cmd),
- Cmd::Memory(memory::MemoryCmd::Show) => {
+ Cmd::Memory { cmd: None } => {
let db = Db::open(&cfg.db_path())?;
let s = memory::status(&db.conn()).map_err(|e| e.to_string())?;
println!("{}", s.text());
Ok(())
}
- Cmd::Memory(memory::MemoryCmd::Reserve { size }) => {
+ Cmd::Memory {
+ cmd: Some(memory::MemoryCmd::Reserve { size }),
+ } => {
let reserve =
limits::parse_size(&size).map_err(|e| e.to_string())?;
let db = Db::open(&cfg.db_path())?;
diff --git a/server/src/memory.rs b/server/src/memory.rs
index 80e576e..4a2ec6b 100644
--- a/server/src/memory.rs
+++ b/server/src/memory.rs
@@ -32,13 +32,20 @@ struct Setting {
}
/// The reserve for the system on this server, set or the default.
-pub fn reserve(conn: &Connection) -> u32 {
- let (ram, _) = limits::server_memory();
- let set: Setting = crate::db::get_json(conn, "memory").unwrap_or_default();
- let (min, max) = limits::reserve_bounds(ram);
+fn reserve(conn: &Connection) -> u32 {
+ reserve_of(limits::server_memory().0, &setting(conn))
+}
+
+fn setting(conn: &Connection) -> Setting {
+ crate::db::get_json(conn, "memory").unwrap_or_default()
+}
+
+/// A reserve that no longer fits the RAM (a resized VM) is clamped.
+fn reserve_of(ram_mb: u32, set: &Setting) -> u32 {
+ let (min, max) = limits::reserve_bounds(ram_mb);
set.reserve_mb
.map(|r| r.clamp(min, max))
- .unwrap_or_else(|| limits::default_reserve(ram))
+ .unwrap_or_else(|| limits::default_reserve(ram_mb))
}
/// What all groups together may use on this server.
@@ -49,19 +56,60 @@ pub fn groups_cap(conn: &Connection) -> u32 {
/// Sets the reserve, `None` for the default; apply makes it hold.
pub fn set_reserve(conn: &Connection, reserve_mb: Option) -> Result<()> {
let (ram, _) = limits::server_memory();
- let (min, max) = limits::reserve_bounds(ram);
- if let Some(r) = reserve_mb
- && !(min..=max).contains(&r)
- {
+ check_reserve(ram, reserve_mb, groups_used())?;
+ let set = Setting { reserve_mb };
+ set_setting(conn, "memory", &serde_json::to_string(&set).unwrap())?;
+ Ok(())
+}
+
+/// Within the bounds, and the new cap leaves the groups what they use now
+/// without their page cache: below that the kernel kills in the groups
+/// right away.
+fn check_reserve(
+ ram_mb: u32,
+ reserve_mb: Option,
+ groups_used_mb: Option,
+) -> Result<()> {
+ let Some(r) = reserve_mb else {
+ return Ok(());
+ };
+ let (min, max) = limits::reserve_bounds(ram_mb);
+ if !(min..=max).contains(&r) {
return Err(Error::Invalid(format!(
"the reserve is {min} MB to {max} MB, half the RAM"
)));
}
- let set = Setting { reserve_mb };
- set_setting(conn, "memory", &serde_json::to_string(&set).unwrap())?;
+ let cap = limits::groups_cap(ram_mb, r);
+ if let Some(used) = groups_used_mb
+ && cap < used
+ {
+ return Err(Error::Invalid(format!(
+ "the groups use {used} MB now, a reserve of {r} MB leaves them \
+ {cap} MB; at most {} MB",
+ (ram_mb - used).min(max)
+ )));
+ }
Ok(())
}
+fn read_groups(file: &str) -> Option {
+ std::fs::read_to_string(format!("{GROUPS_CGROUP}/{file}")).ok()
+}
+
+/// What the groups use now without their page cache, which the kernel
+/// takes back first, in MB.
+fn groups_used() -> Option {
+ let current: u64 = read_groups("memory.current")?.trim().parse().ok()?;
+ let stat = read_groups("memory.stat")?;
+ let file: u64 = stat
+ .lines()
+ .find_map(|l| l.strip_prefix("file "))?
+ .trim()
+ .parse()
+ .ok()?;
+ Some((current.saturating_sub(file) >> 20) as u32)
+}
+
/// The split and what the groups and the system use, for the panel, the
/// CLI and the agent's decisions.
#[derive(Debug, Serialize)]
@@ -79,6 +127,7 @@ pub struct Status {
pub cap_mb: u32,
/// the groups' own limits added up (the default for those without)
pub limits_mb: u32,
+ /// without their page cache, like the system's number
pub groups_now_mb: Option,
/// since the groups' slice started (the last boot, mostly)
pub groups_peak_mb: Option,
@@ -88,15 +137,10 @@ pub struct Status {
pub fn status(conn: &Connection) -> Result {
let (ram, swap) = limits::server_memory();
- let set: Setting = crate::db::get_json(conn, "memory").unwrap_or_default();
- let reserve_mb = reserve(conn);
+ let set = setting(conn);
+ let reserve_mb = reserve_of(ram, &set);
let (min, max) = limits::reserve_bounds(ram);
- let read = |file: &str| -> Option {
- let v =
- std::fs::read_to_string(format!("{GROUPS_CGROUP}/{file}")).ok()?;
- Some((v.trim().parse::().ok()? >> 20) as u32)
- };
- let groups_now = read("memory.current");
+ let groups_now = groups_used();
let info = std::fs::read_to_string("/proc/meminfo").unwrap_or_default();
let used = crate::stats::meminfo(&info, "MemTotal:")
.zip(crate::stats::meminfo(&info, "MemAvailable:"))
@@ -115,7 +159,9 @@ pub fn status(conn: &Connection) -> Result {
.map(|g| g.limits.memory_max())
.sum(),
groups_now_mb: groups_now,
- groups_peak_mb: read("memory.peak"),
+ groups_peak_mb: read_groups("memory.peak")
+ .and_then(|v| v.trim().parse::().ok())
+ .map(|v| (v >> 20) as u32),
system_now_mb: used.zip(groups_now).map(|(u, g)| u.saturating_sub(g)),
})
}
@@ -150,11 +196,9 @@ impl Status {
}
}
-/// `genco memory`: the split, or a new reserve.
+/// `genco memory reserve`: a new reserve.
#[derive(Debug, clap::Parser)]
pub enum MemoryCmd {
- /// Show the split between the groups and the system and what each uses
- Show,
/// Keep this much for the system (2G, 1536M, 1536) or `default`;
/// applies at once
Reserve { size: String },
@@ -197,19 +241,27 @@ mod tests {
use super::*;
#[test]
- fn the_reserve_is_a_setting_within_bounds() {
+ fn the_reserve_stays_within_bounds() {
+ assert!(check_reserve(4000, None, Some(3900)).is_ok());
+ assert!(check_reserve(4000, Some(511), None).is_err());
+ assert!(check_reserve(4000, Some(2001), None).is_err());
+ assert!(check_reserve(4000, Some(512), None).is_ok());
+ assert!(check_reserve(4000, Some(2000), None).is_ok());
+ // the groups use 2500 MB without their cache
+ assert!(check_reserve(4000, Some(1500), Some(2500)).is_ok());
+ assert!(check_reserve(4000, Some(1600), Some(2500)).is_err());
+ }
+
+ #[test]
+ fn a_set_reserve_is_clamped_to_the_ram() {
+ let set = |r| Setting { reserve_mb: r };
+ assert_eq!(reserve_of(4000, &set(None)), 1000);
+ assert_eq!(reserve_of(4000, &set(Some(1500))), 1500);
+ // the VM shrank to 2 GB
+ assert_eq!(reserve_of(2000, &set(Some(1500))), 1000);
+
let db = Db::memory();
- let conn = db.conn();
- let (ram, _) = limits::server_memory();
- assert_eq!(reserve(&conn), limits::default_reserve(ram));
-
- let (min, max) = limits::reserve_bounds(ram);
- assert!(set_reserve(&conn, Some(min - 1)).is_err());
- assert!(set_reserve(&conn, Some(max + 1)).is_err());
- set_reserve(&conn, Some(min)).unwrap();
- assert_eq!(reserve(&conn), min);
- assert!(!status(&conn).unwrap().default);
- set_reserve(&conn, None).unwrap();
- assert!(status(&conn).unwrap().default);
+ set_setting(&db.conn(), "memory", r#"{"reserveMb":700}"#).unwrap();
+ assert_eq!(setting(&db.conn()).reserve_mb, Some(700));
}
}
diff --git a/webui/src/components/GroupLimits.svelte b/webui/src/components/GroupLimits.svelte
index 8fcd46a..471a53e 100644
--- a/webui/src/components/GroupLimits.svelte
+++ b/webui/src/components/GroupLimits.svelte
@@ -2,12 +2,12 @@
import {
formatBytes,
get,
+ memoryStatus,
memoryUsage,
saveGroup,
type Group,
type Limits,
type MemoryUsage,
- type MemoryStatus,
type Stats,
} from '@/lib/api';
import Form from '@/components/win/Form.svelte';
@@ -31,8 +31,8 @@
const defaults = $derived(group.limits.defaults);
// what all groups may use together, Settings → Memory
let capMb = $state(null);
- get('/settings/memory')
- .then((m: MemoryStatus) => (capMb = m.capMb))
+ memoryStatus()
+ .then(m => (capMb = m.capMb))
.catch(() => {});
$effect(() => {
diff --git a/webui/src/components/dialogs/MemorySettings.svelte b/webui/src/components/dialogs/MemorySettings.svelte
index 9d08861..63ec85d 100644
--- a/webui/src/components/dialogs/MemorySettings.svelte
+++ b/webui/src/components/dialogs/MemorySettings.svelte
@@ -1,5 +1,10 @@