diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 15d75ff..6043c1d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -438,10 +438,18 @@ now; `genco tree` has each group's `limits` (`null` is the default, its `defaults` what an empty one means). **All groups together.** The groups' slices are in `genco.slice`, which -genco caps at the RAM without a reserve for the system: a quarter of the -RAM, at least 768 MB, at most half (3.9 GB of RAM: 2.9 GB for the -groups). So the groups' own limits may add up to far more than the RAM, -groups rarely peak together; when they do, the kernel takes memory back +genco caps at the RAM without the reserve for the system: by default a +quarter of the RAM, at least 768 MB, at most half (3.9 GB of RAM: 2.9 GB +for the groups). Settings → Memory and `genco memory` show the split and +what the groups and the system use; the reserve is a setting there +(`genco memory reserve 1.5G`, `default`), 512 MB to half the RAM, and +applies at once. genco's agent moves it too, in the daily health check +or on an alert, when the system runs short while the groups have room or +the other way round, and says why. A reserve that leaves the groups +less than they use now without their page cache is refused. A new group +limit is at most the cap; one a group already has stays, the cap holds +it anyway. So the groups' own limits may add up to far more than the +RAM, groups rarely peak together; when they do, the kernel takes memory back from them and kills within the groups, never the databases, nginx, mail or genco. The shared services (not genco, whose children such as imports and the agent's commands would inherit it) also get a low OOM score diff --git a/docs/PLAN.md b/docs/PLAN.md index 181e584..2ce660d 100644 --- a/docs/PLAN.md +++ b/docs/PLAN.md @@ -307,6 +307,10 @@ terminal. the RAM without a reserve (genco.slice), shared services with a low OOM score, alerts on memory pressure and OOM kills instead of the sum of limits, the group's usage and 24 h peak beside its limit +- [x] the reserve for the system as a setting (operator 2026-11-12): + Settings → Memory, `genco memory [reserve ]`, + 512 MB to half the RAM, moved by the agent when the split doesn't + fit what the groups and the system use - [x] drop-ins that override genco's units reported by apply, the minute check and the config view; `genco group adopt-limits` - [x] ImageMagick limits for groups with PHP, from the group's memory diff --git a/server/src/activity.rs b/server/src/activity.rs index 0cdd5f3..d6f1256 100644 --- a/server/src/activity.rs +++ b/server/src/activity.rs @@ -165,6 +165,7 @@ fn describe(method: &Method, path: &str) -> Option { ("POST", "/settings/mail/hosting", "Change mail hosting"), ("POST", "/settings/notify", "Change notifications"), ("POST", "/settings/model", "Change the agent's model"), + ("POST", "/settings/memory", "Change the memory reserve"), ("POST", "/domains/*/env", "Change environment variables"), ("POST", "/groups/*/workers", "Add worker"), ("POST", "/workers/*", "Change worker"), diff --git a/server/src/agent/instructions.md b/server/src/agent/instructions.md index 913ba13..c09ae5f 100644 --- a/server/src/agent/instructions.md +++ b/server/src/agent/instructions.md @@ -269,10 +269,18 @@ field in the panel. `genco-*.service.d/`) or `systemctl set-property`: they override genco and are reported as a problem; `genco group adopt-limits ` moves a slice drop-in's limits into the settings. All groups together are capped at the RAM - without a reserve for the system (`genco.slice`), a group's limit at + without the reserve for the system (`genco.slice`), a group's limit at most that; their limits may add up to more. Base a limit on what the group really uses (Usage, `memory.peak` of its slice), alerts come on memory pressure and OOM kills. +- The reserve for the system: `genco memory` shows the split and what + the groups and the system use, `genco memory reserve ` + changes it within its bounds and applies. Move it yourself when the + split doesn't fit: raise it when the system (databases, mail, genco) + runs short or swaps while the groups have room, lower it when groups + get OOM kills at the cap while the system leaves much of its reserve + unused. Small steps (256-512 MB), and say what you changed and why + with the numbers. - Craft with Imagick: transforms of large originals take 500-700 MB. genco limits ImageMagick per process for groups with PHP (MAGICK_MEMORY_LIMIT a quarter of the soft limit, at most 256 MiB, then diff --git a/server/src/agent/routines/health.md b/server/src/agent/routines/health.md index b533203..5449178 100644 --- a/server/src/agent/routines/health.md +++ b/server/src/agent/routines/health.md @@ -13,6 +13,9 @@ admin, and write a short report: - Certificates: any domain without one, or expiring in under 30 days (`certbot certificates`)? - Mail: anything stuck in the queue (`postqueue -p`)? +- Memory: `genco memory`. If the split between the groups and the + system doesn't fit what they use, move the reserve (see your + instructions) and say so. - Updates are the weekly update round's; only say if the box needs a reboot (`/var/run/reboot-required`). - Open warnings: `genco attention list` shows the routines and alerts diff --git a/server/src/apply/memory.rs b/server/src/apply/memory.rs index 583392f..a51d55b 100644 --- a/server/src/apply/memory.rs +++ b/server/src/apply/memory.rs @@ -1,5 +1,5 @@ //! Memory of the whole server: all groups share `genco.slice`, capped at -//! the RAM without a reserve for the system (limits.rs), so groups whose +//! the RAM without the reserve for the system (memory.rs), so groups whose //! own limits add up to more than the RAM run out among themselves. The //! shared services get a low OOM score: when the kernel has to kill //! outside the groups after all, it picks something else than the @@ -28,11 +28,10 @@ const SYSTEM_SLICE: &str = "/sys/fs/cgroup/system.slice"; /// genco.slice and a drop-in with the OOM score for each protected service /// that runs here. -pub(super) fn memory_files() -> Vec { - let (ram, _) = crate::limits::server_memory(); +pub(super) fn memory_files(cap_mb: u32) -> Vec { let mut files = vec![File { path: render::unit_path(GROUPS_SLICE), - content: crate::limits::groups_slice_unit(ram), + content: crate::limits::groups_slice_unit(cap_mb), private_to: None, }]; for (unit, _) in protected_units() { @@ -86,10 +85,13 @@ fn protected(unit: &str) -> bool { /// The cap on the running slice, and the OOM score of the protected /// services' processes right away (the drop-in counts from their next /// start). -pub(super) async fn memory(r: &mut Report, changed: &HashSet) { +pub(super) async fn memory( + r: &mut Report, + changed: &HashSet, + cap_mb: u32, +) { if changed.contains(&render::unit_path(GROUPS_SLICE)) { - let (ram, _) = crate::limits::server_memory(); - let cap = format!("MemoryMax={}M", crate::limits::groups_cap(ram)); + let cap = format!("MemoryMax={cap_mb}M"); if r.run(&[ "systemctl", "set-property", diff --git a/server/src/apply/mod.rs b/server/src/apply/mod.rs index 80b286f..8b61212 100644 --- a/server/src/apply/mod.rs +++ b/server/src/apply/mod.rs @@ -151,6 +151,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report { return r; }; let blocked = blocked(&mut r, db); + let cap = crate::memory::groups_cap(&db.conn()); let old_names = crate::panel::old_names(&db.conn()); relay_sasl(&mut r, db).await; @@ -161,7 +162,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report { deploy_key(&mut r, g).await; } } - sync(&mut r, cfg, &groups, &blocked, &old_names).await; + sync(&mut r, cfg, &groups, &blocked, &old_names, cap).await; databases(&mut r, &groups).await; // with a new certificate the sites get https, the apps the mail host @@ -170,7 +171,7 @@ pub async fn apply(cfg: &Config, db: &Db) -> Report { for g in &mut groups { g.mail_host = host.clone(); } - sync(&mut r, cfg, &groups, &blocked, &old_names).await; + sync(&mut r, cfg, &groups, &blocked, &old_names, cap).await; } // genco doesn't touch them, an admin moves them into the group's @@ -192,6 +193,7 @@ async fn sync( groups: &[Group], blocked: &[String], old_names: &[String], + cap_mb: u32, ) { let certs = host::certs(groups); let mut files: Vec = groups @@ -199,7 +201,7 @@ async fn sync( .flat_map(|g| render::group_files(g, &certs)) .collect(); files.extend(panel_files(cfg, old_names)); - files.extend(memory_files()); + files.extend(memory_files(cap_mb)); files.push(firewall_file(cfg, groups, blocked)); files.push(File { path: render::RESERVED_PORTS.into(), @@ -225,7 +227,7 @@ async fn sync( r.run(&["systemctl", "daemon-reload"]).await; } - memory(r, &changed).await; + memory(r, &changed, cap_mb).await; for g in groups { set_limits(r, g, &changed).await; start_units(r, g, &changed).await; diff --git a/server/src/hosting.rs b/server/src/hosting.rs index a6d5570..04779a9 100644 --- a/server/src/hosting.rs +++ b/server/src/hosting.rs @@ -411,7 +411,7 @@ pub fn set_limits(conn: &Connection, id: i64, limits: &Limits) -> Result<()> { ) .optional()? .flatten(); - limits::check_here(limits, kept)?; + limits::check_here(limits, crate::memory::groups_cap(conn), kept)?; conn.execute( "UPDATE domain_groups SET memory_max_mb = ?1, memory_high_mb = ?2, cpu_percent = ?3, tasks_max = ?4 WHERE id = ?5", diff --git a/server/src/limits.rs b/server/src/limits.rs index de59a32..da05b13 100644 --- a/server/src/limits.rs +++ b/server/src/limits.rs @@ -77,27 +77,24 @@ impl Limits { } } - /// Within what the server has: `ram_mb` and `cores`. A group may - /// have at most what all groups together may use (`groups_cap`); a - /// memory limit it already has (`kept_mb`) stays allowed, the cap of - /// all groups holds it anyway. + /// Within what the server has: `cap_mb`, what all groups may use + /// together (`groups_cap`), and `cores`. A memory limit the group + /// already has (`kept_mb`) stays allowed, the cap of all groups holds + /// it anyway. pub fn check( &self, - ram_mb: u32, + cap_mb: u32, cores: u32, kept_mb: Option, ) -> Result<()> { let max = self.memory_max(); - let cap = groups_cap(ram_mb); if self.memory_max_mb.is_some_and(|m| Some(m) != kept_mb) - && !(128..=cap).contains(&max) + && !(128..=cap_mb).contains(&max) { return Err(Error::Invalid(format!( "the memory limit is 128 MB to {}, what all groups may use \ - together (the server's {} without {} for the system)", - size(cap), - size(ram_mb), - size(reserve(ram_mb)) + together (`genco memory`)", + size(cap_mb) ))); } if !(64..=max).contains(&self.memory_high()) { @@ -154,18 +151,14 @@ pub fn with_defaults( s: S, ) -> std::result::Result { #[derive(Serialize)] - #[serde(rename_all = "camelCase")] struct Shown<'a> { #[serde(flatten)] limits: &'a Limits, defaults: Defaults, - /// what all groups may use together, the most one may have - groups_cap_mb: u32, } Shown { limits, defaults: limits.defaults(), - groups_cap_mb: groups_cap(server_memory().0), } .serialize(s) } @@ -252,29 +245,42 @@ fn cores() -> u32 { std::thread::available_parallelism().map_or(1, |n| n.get() as u32) } -/// Checks the limits against this server, `kept_mb` is the memory limit -/// the group has now. -pub fn check_here(limits: &Limits, kept_mb: Option) -> Result<()> { - limits.check(server_memory().0, cores(), kept_mb) +/// Checks the limits against this server with what all groups may use +/// (`cap_mb`), `kept_mb` is the memory limit the group has now. +pub fn check_here( + limits: &Limits, + cap_mb: u32, + kept_mb: Option, +) -> Result<()> { + limits.check(cap_mb, cores(), kept_mb) } +/// The least an admin or the agent may keep for the system. +pub const MIN_RESERVE_MB: u32 = 512; + /// The memory genco keeps for the system (the databases, nginx, mail, -/// genco itself): a quarter of the RAM, at least 768 MB, at most half. -pub fn reserve(ram_mb: u32) -> u32 { +/// genco itself) unless set (memory.rs): a quarter of the RAM, at least +/// 768 MB, at most half. +pub fn default_reserve(ram_mb: u32) -> u32 { (ram_mb / 4).max(768).min(ram_mb / 2) } +/// What a reserve may be: `MIN_RESERVE_MB` to half the RAM. +pub fn reserve_bounds(ram_mb: u32) -> (u32, u32) { + (MIN_RESERVE_MB.min(ram_mb / 2), ram_mb / 2) +} + /// What all groups together may use (`genco.slice`): the RAM without the -/// `reserve`. Their own limits may add up to much more, they rarely peak +/// reserve. Their own limits may add up to much more, they rarely peak /// together; when they do, a group runs out, never the system. -pub fn groups_cap(ram_mb: u32) -> u32 { - ram_mb - reserve(ram_mb) +pub fn groups_cap(ram_mb: u32, reserve_mb: u32) -> u32 { + ram_mb.saturating_sub(reserve_mb) } /// The unit of the slice all groups' slices are in. -pub fn groups_slice_unit(ram_mb: u32) -> String { +pub fn groups_slice_unit(cap_mb: u32) -> String { format!( - "{}: genco sets it from the RAM + "{}: genco memory reserve changes it [Unit] Description=All groups of genco @@ -282,7 +288,7 @@ Description=All groups of genco MemoryMax={} ", crate::render::HEADER, - unit_size(groups_cap(ram_mb)) + unit_size(cap_mb) ) } @@ -533,7 +539,7 @@ mod tests { #[test] fn bounds_follow_the_server() { - let ok = |l: Limits| l.check(4000, 2, None).is_ok(); + let ok = |l: Limits| l.check(3000, 2, None).is_ok(); assert!(ok(Limits::default())); let mem = |mb| Limits { memory_max_mb: Some(mb), @@ -545,8 +551,8 @@ mod tests { assert!(ok(mem(3000))); assert!(!ok(mem(3001))); // one it had before the cap stays - assert!(mem(3500).check(4000, 2, Some(3500)).is_ok()); - assert!(mem(3600).check(4000, 2, Some(3500)).is_err()); + assert!(mem(3500).check(3000, 2, Some(3500)).is_ok()); + assert!(mem(3600).check(3000, 2, Some(3500)).is_err()); // a soft limit above the max assert!(!ok(Limits { memory_high_mb: Some(2048), @@ -570,13 +576,15 @@ mod tests { #[test] fn the_system_keeps_a_reserve() { - assert_eq!(reserve(4000), 1000); - assert_eq!(groups_cap(4000), 3000); - assert_eq!(reserve(2048), 768); - assert_eq!(reserve(1024), 512); - assert_eq!(reserve(16384), 4096); - assert!(groups_slice_unit(4000).contains("MemoryMax=3000M\n")); - assert!(groups_slice_unit(4096).contains("MemoryMax=3G\n")); + assert_eq!(default_reserve(4000), 1000); + assert_eq!(groups_cap(4000, 1000), 3000); + assert_eq!(default_reserve(2048), 768); + assert_eq!(default_reserve(1024), 512); + assert_eq!(default_reserve(16384), 4096); + assert_eq!(reserve_bounds(4000), (512, 2000)); + assert_eq!(reserve_bounds(800), (400, 400)); + assert!(groups_slice_unit(3000).contains("MemoryMax=3000M\n")); + assert!(groups_slice_unit(3072).contains("MemoryMax=3G\n")); } #[test] diff --git a/server/src/main.rs b/server/src/main.rs index d4d6841..7d13f88 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -31,6 +31,7 @@ mod llm; mod mail; mod mailbox; mod mailfilter; +mod memory; mod notify; mod panel; mod peers; @@ -167,6 +168,12 @@ pub(crate) enum Cmd { /// Block addresses from the sites, mail and workers (never ssh) #[clap(subcommand)] Firewall(firewall::FirewallCmd), + /// How the RAM is split between the groups and the system; change the + /// reserve for the system + Memory { + #[clap(subcommand)] + cmd: Option, + }, /// The routines and alerts that still need attention: list them, /// resolve one that is taken care of #[clap(subcommand)] @@ -219,6 +226,7 @@ impl Cmd { | Cmd::Setup { .. } | Cmd::Apply | Cmd::Deploy { .. } | Cmd::Backup { .. } + | Cmd::Memory { cmd: Some(_) } | Cmd::Restic { .. } | Cmd::Update { .. } | Cmd::Panel(_) @@ -248,6 +256,7 @@ impl Cmd { | Cmd::Firewall(firewall::FirewallCmd::List) | Cmd::Activity { .. } | Cmd::Attention(conversations::AttentionCmd::List) + | Cmd::Memory { cmd: None } | Cmd::Update { .. } | Cmd::Runners(RunnersCmd::List) | Cmd::Group(cli::GroupCmd::DeployKey { .. }) @@ -403,6 +412,22 @@ async fn run(args: Args) -> Result<(), String> { } Cmd::Docs { .. } => unreachable!("read before the config"), Cmd::User(cmd) => user(cfg, cmd), + Cmd::Memory { cmd: None } => { + let db = Db::open(&cfg.db_path())?; + let s = memory::status(&db.conn()).map_err(|e| e.to_string())?; + println!("{}", s.text()); + Ok(()) + } + Cmd::Memory { + cmd: Some(memory::MemoryCmd::Reserve { size }), + } => { + let reserve = + limits::parse_size(&size).map_err(|e| e.to_string())?; + let db = Db::open(&cfg.db_path())?; + memory::set_reserve(&db.conn(), reserve) + .map_err(|e| e.to_string())?; + apply_cmd(cfg).await + } Cmd::Attention(cmd) => { edit(cfg, |_, conn| conversations::attention(conn, cmd)) } diff --git a/server/src/memory.rs b/server/src/memory.rs new file mode 100644 index 0000000..4a2ec6b --- /dev/null +++ b/server/src/memory.rs @@ -0,0 +1,267 @@ +//! How the RAM is split between the groups and the system: all groups +//! share `genco.slice`, capped at the RAM without the reserve for the +//! system (apply/memory.rs). The reserve is a setting (Settings → Memory, +//! `genco memory reserve`), a quarter of the RAM by default; admins and +//! genco's agent move it within `limits::reserve_bounds` when the split +//! no longer fits what the groups and the system really use. + +use std::sync::Arc; + +use axum::extract::State; +use axum::routing::get; +use axum::{Json, Router}; +use rusqlite::Connection; +use serde::{Deserialize, Serialize}; + +use crate::AppState; +use crate::apply; +use crate::config::Config; +use crate::db::{Db, set_setting}; +use crate::error::{Error, Result}; +use crate::events::{Event, Events}; +use crate::limits; +use crate::users::Authed; + +const GROUPS_CGROUP: &str = "/sys/fs/cgroup/genco.slice"; + +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct Setting { + /// `None` is the default + reserve_mb: Option, +} + +/// The reserve for the system on this server, set or the default. +fn reserve(conn: &Connection) -> u32 { + reserve_of(limits::server_memory().0, &setting(conn)) +} + +fn setting(conn: &Connection) -> Setting { + crate::db::get_json(conn, "memory").unwrap_or_default() +} + +/// A reserve that no longer fits the RAM (a resized VM) is clamped. +fn reserve_of(ram_mb: u32, set: &Setting) -> u32 { + let (min, max) = limits::reserve_bounds(ram_mb); + set.reserve_mb + .map(|r| r.clamp(min, max)) + .unwrap_or_else(|| limits::default_reserve(ram_mb)) +} + +/// What all groups together may use on this server. +pub fn groups_cap(conn: &Connection) -> u32 { + limits::groups_cap(limits::server_memory().0, reserve(conn)) +} + +/// Sets the reserve, `None` for the default; apply makes it hold. +pub fn set_reserve(conn: &Connection, reserve_mb: Option) -> Result<()> { + let (ram, _) = limits::server_memory(); + check_reserve(ram, reserve_mb, groups_used())?; + let set = Setting { reserve_mb }; + set_setting(conn, "memory", &serde_json::to_string(&set).unwrap())?; + Ok(()) +} + +/// Within the bounds, and the new cap leaves the groups what they use now +/// without their page cache: below that the kernel kills in the groups +/// right away. +fn check_reserve( + ram_mb: u32, + reserve_mb: Option, + groups_used_mb: Option, +) -> Result<()> { + let Some(r) = reserve_mb else { + return Ok(()); + }; + let (min, max) = limits::reserve_bounds(ram_mb); + if !(min..=max).contains(&r) { + return Err(Error::Invalid(format!( + "the reserve is {min} MB to {max} MB, half the RAM" + ))); + } + let cap = limits::groups_cap(ram_mb, r); + if let Some(used) = groups_used_mb + && cap < used + { + return Err(Error::Invalid(format!( + "the groups use {used} MB now, a reserve of {r} MB leaves them \ + {cap} MB; at most {} MB", + (ram_mb - used).min(max) + ))); + } + Ok(()) +} + +fn read_groups(file: &str) -> Option { + std::fs::read_to_string(format!("{GROUPS_CGROUP}/{file}")).ok() +} + +/// What the groups use now without their page cache, which the kernel +/// takes back first, in MB. +fn groups_used() -> Option { + let current: u64 = read_groups("memory.current")?.trim().parse().ok()?; + let stat = read_groups("memory.stat")?; + let file: u64 = stat + .lines() + .find_map(|l| l.strip_prefix("file "))? + .trim() + .parse() + .ok()?; + Some((current.saturating_sub(file) >> 20) as u32) +} + +/// The split and what the groups and the system use, for the panel, the +/// CLI and the agent's decisions. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Status { + pub ram_mb: u32, + pub swap_mb: u32, + pub reserve_mb: u32, + /// the reserve is the default, not set + pub default: bool, + pub default_reserve_mb: u32, + pub min_reserve_mb: u32, + pub max_reserve_mb: u32, + /// what all groups may use together + pub cap_mb: u32, + /// the groups' own limits added up (the default for those without) + pub limits_mb: u32, + /// without their page cache, like the system's number + pub groups_now_mb: Option, + /// since the groups' slice started (the last boot, mostly) + pub groups_peak_mb: Option, + /// everything outside the groups: the RAM in use without them + pub system_now_mb: Option, +} + +pub fn status(conn: &Connection) -> Result { + let (ram, swap) = limits::server_memory(); + let set = setting(conn); + let reserve_mb = reserve_of(ram, &set); + let (min, max) = limits::reserve_bounds(ram); + let groups_now = groups_used(); + let info = std::fs::read_to_string("/proc/meminfo").unwrap_or_default(); + let used = crate::stats::meminfo(&info, "MemTotal:") + .zip(crate::stats::meminfo(&info, "MemAvailable:")) + .map(|(t, a)| ((t - a) >> 20) as u32); + Ok(Status { + ram_mb: ram, + swap_mb: swap, + reserve_mb, + default: set.reserve_mb.is_none(), + default_reserve_mb: limits::default_reserve(ram), + min_reserve_mb: min, + max_reserve_mb: max, + cap_mb: limits::groups_cap(ram, reserve_mb), + limits_mb: crate::hosting::groups(conn)? + .iter() + .map(|g| g.limits.memory_max()) + .sum(), + groups_now_mb: groups_now, + groups_peak_mb: read_groups("memory.peak") + .and_then(|v| v.trim().parse::().ok()) + .map(|v| (v >> 20) as u32), + system_now_mb: used.zip(groups_now).map(|(u, g)| u.saturating_sub(g)), + }) +} + +impl Status { + /// For `genco memory`. + pub fn text(&self) -> String { + let mb = |v: Option| v.map_or("?".into(), |v| format!("{v} MB")); + format!( + "RAM {} MB, swap {} MB\n\ + reserve for the system {} MB ({}; {} to {} MB)\n\ + all groups together at most {} MB, their own limits add up to \ + {} MB\n\ + groups now {}, at most {} since their slice started\n\ + system now {}", + self.ram_mb, + self.swap_mb, + self.reserve_mb, + if self.default { + "the default".to_string() + } else { + format!("set, the default is {} MB", self.default_reserve_mb) + }, + self.min_reserve_mb, + self.max_reserve_mb, + self.cap_mb, + self.limits_mb, + mb(self.groups_now_mb), + mb(self.groups_peak_mb), + mb(self.system_now_mb), + ) + } +} + +/// `genco memory reserve`: a new reserve. +#[derive(Debug, clap::Parser)] +pub enum MemoryCmd { + /// Keep this much for the system (2G, 1536M, 1536) or `default`; + /// applies at once + Reserve { size: String }, +} + +async fn get_route( + _authed: Authed, + State(db): State, +) -> Result> { + Ok(Json(status(&db.conn())?)) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SetReq { + /// `None` for the default + reserve_mb: Option, +} + +/// Sets the reserve and applies, genco.slice gets the new cap at once. +async fn set_route( + _authed: Authed, + State(cfg): State>, + State(db): State, + State(events): State, + Json(req): Json, +) -> Result> { + set_reserve(&db.conn(), req.reserve_mb)?; + let report = apply::apply(&cfg, &db).await; + events.send(Event::Tree); + Ok(Json(report)) +} + +pub fn routes() -> Router { + Router::new().route("/settings/memory", get(get_route).post(set_route)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_reserve_stays_within_bounds() { + assert!(check_reserve(4000, None, Some(3900)).is_ok()); + assert!(check_reserve(4000, Some(511), None).is_err()); + assert!(check_reserve(4000, Some(2001), None).is_err()); + assert!(check_reserve(4000, Some(512), None).is_ok()); + assert!(check_reserve(4000, Some(2000), None).is_ok()); + // the groups use 2500 MB without their cache + assert!(check_reserve(4000, Some(1500), Some(2500)).is_ok()); + assert!(check_reserve(4000, Some(1600), Some(2500)).is_err()); + } + + #[test] + fn a_set_reserve_is_clamped_to_the_ram() { + let set = |r| Setting { reserve_mb: r }; + assert_eq!(reserve_of(4000, &set(None)), 1000); + assert_eq!(reserve_of(4000, &set(Some(1500))), 1500); + // the VM shrank to 2 GB + assert_eq!(reserve_of(2000, &set(Some(1500))), 1000); + + let db = Db::memory(); + set_setting(&db.conn(), "memory", r#"{"reserveMb":700}"#).unwrap(); + assert_eq!(setting(&db.conn()).reserve_mb, Some(700)); + } +} diff --git a/server/src/serve.rs b/server/src/serve.rs index f7a3a7f..542ba53 100644 --- a/server/src/serve.rs +++ b/server/src/serve.rs @@ -222,6 +222,7 @@ pub(crate) async fn serve(cfg: Config) -> Result<(), String> { .merge(peers::routes()) .merge(proposals::routes()) .merge(quarantine::routes()) + .merge(memory::routes()) .merge(approvals::routes()) .merge(update::routes()) .merge(appenv::routes()) diff --git a/webui/src/components/GroupLimits.svelte b/webui/src/components/GroupLimits.svelte index ac65501..471a53e 100644 --- a/webui/src/components/GroupLimits.svelte +++ b/webui/src/components/GroupLimits.svelte @@ -2,6 +2,7 @@ import { formatBytes, get, + memoryStatus, memoryUsage, saveGroup, type Group, @@ -28,6 +29,11 @@ let usage = $state({ memory: null, peak: null, limit: null }); const defaults = $derived(group.limits.defaults); + // what all groups may use together, Settings → Memory + let capMb = $state(null); + memoryStatus() + .then(m => (capMb = m.capMb)) + .catch(() => {}); $effect(() => { get(`/stats?scope=group:${group.id}&range=24h`) @@ -64,10 +70,10 @@ ? ` of ${formatBytes(usage.limit)}` : ''}, at most {formatBytes(usage.peak ?? 0)} in the last 24 hours. {/if} - All groups together may use {formatBytes( - group.limits.groupsCapMb * 1024 * 1024, - )}, the rest of the RAM stays for the databases, mail and genco; their - limits may add up to more. + {#if capMb !== null} + All groups together may use {formatBytes(capMb * 1024 * 1024)} (Settings + → Memory), their limits may add up to more. + {/if}

diff --git a/webui/src/components/dialogs/MemorySettings.svelte b/webui/src/components/dialogs/MemorySettings.svelte new file mode 100644 index 0000000..63ec85d --- /dev/null +++ b/webui/src/components/dialogs/MemorySettings.svelte @@ -0,0 +1,91 @@ + + +{#if error}

{error}

{/if} +{#if status} +

+ The server has {status.ramMb} MB of RAM. All groups together may use + {status.capMb} MB, {status.reserveMb} MB stay for the system (databases, nginx, + mail, genco). The groups' own limits add up to {status.limitsMb} MB, that + is fine, they rarely peak together. +

+

+ Now the groups use {mb(status.groupsNowMb)} (at most {mb( + status.groupsPeakMb, + )} since their slice started), the system {mb(status.systemNowMb)}. +

+ + + + + MB for the system, {status.minReserveMb} to {status.maxReserveMb}, + empty for the default + +
+

+ genco's agent moves it too when the split doesn't fit what the groups + and the system use, and says why. +

+{/if} + + diff --git a/webui/src/components/dialogs/SettingsDialog.svelte b/webui/src/components/dialogs/SettingsDialog.svelte index 3c53e90..6193c8b 100644 --- a/webui/src/components/dialogs/SettingsDialog.svelte +++ b/webui/src/components/dialogs/SettingsDialog.svelte @@ -5,6 +5,7 @@ import BackupSettings from './BackupSettings.svelte'; import FirewallSettings from './FirewallSettings.svelte'; import MailSettings from './MailSettings.svelte'; + import MemorySettings from './MemorySettings.svelte'; import NotifySettings from './NotifySettings.svelte'; import RunnerSettings from './RunnerSettings.svelte'; import UpdateSettings from './UpdateSettings.svelte'; @@ -19,6 +20,7 @@ { id: 'update', label: 'Updates' }, { id: 'runners', label: 'Runners' }, { id: 'firewall', label: 'Firewall' }, + { id: 'memory', label: 'Memory' }, ]; let tab = $state('mail'); @@ -34,6 +36,7 @@ + diff --git a/webui/src/lib/api.ts b/webui/src/lib/api.ts index 6c5fe22..60d15f3 100644 --- a/webui/src/lib/api.ts +++ b/webui/src/lib/api.ts @@ -185,11 +185,7 @@ export type Group = { workers: Worker[]; schedules: Schedule[]; php: PhpSettings; - limits: Limits & { - defaults: LimitDefaults; - /** what all groups may use together, the most one may have */ - groupsCapMb: number; - }; + limits: Limits & { defaults: LimitDefaults }; /** stopped because it looks compromised */ quarantine: Quarantine | null; }; @@ -203,6 +199,25 @@ export type Quarantine = { at: number; }; +/** how the RAM is split between the groups and the system (memory.rs) */ +export type MemoryStatus = { + ramMb: number; + swapMb: number; + reserveMb: number; + /** the reserve is the default, not set */ + default: boolean; + defaultReserveMb: number; + minReserveMb: number; + maxReserveMb: number; + /** what all groups may use together */ + capMb: number; + /** the groups' own limits added up */ + limitsMb: number; + groupsNowMb: number | null; + groupsPeakMb: number | null; + systemNowMb: number | null; +}; + /** what a group's slice may use, null is the default */ export type Limits = { memoryMaxMb: number | null; @@ -348,6 +363,11 @@ export function fileState( export type ApplyReport = { ok: boolean; log: string[] }; +/** The error lines of an apply. */ +export function applyErrors(report: ApplyReport): string { + return report.log.filter(l => l.startsWith('error')).join('\n'); +} + export type DatabaseInfo = { name: string; exists: boolean; @@ -707,6 +727,20 @@ export async function download( window.location.href = `${target}${sep}ticket=${encodeURIComponent(t)}`; } +/** How the RAM is split between the groups and the system. */ +export function memoryStatus(): Promise { + return get('/settings/memory'); +} + +/** + * Sets the reserve for the system, `null` for the default; apply runs + * right away and its errors throw. + */ +export async function setMemoryReserve(reserveMb: number | null) { + const report: ApplyReport = await post('/settings/memory', { reserveMb }); + if (!report.ok) throw new Error(applyErrors(report)); +} + /** Puts a group in quarantine with the reason, apply runs right away. */ export function quarantineGroup( group: Group, diff --git a/webui/src/pages/Group.svelte b/webui/src/pages/Group.svelte index 8612fa2..63ef141 100644 --- a/webui/src/pages/Group.svelte +++ b/webui/src/pages/Group.svelte @@ -10,6 +10,7 @@ import { GROUP_TABS } from '@/lib/groupTabs'; import { KIND_NAMES, + applyErrors, formatTime, get, liftQuarantine, @@ -123,10 +124,7 @@ if (!report.ok) { message.show( 'Quarantine', - 'Applied with errors:\n' + - report.log - .filter(l => l.startsWith('error')) - .join('\n'), + 'Applied with errors:\n' + applyErrors(report), ); } } catch (e: any) {