From f541829a26a7c73ab77915bab64df922e3f3007a Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 08:43:47 +1030 Subject: [PATCH 01/66] fix: console output via messages and vignette options reset (CRAN review) --- CITATION.cff | 2 +- DESCRIPTION | 2 +- NEWS.md | 6 ++++++ R/masque.R | 14 +++++++------- cran-comments.md | 12 ++++++++++-- vignettes/confidentiality.Rmd | 6 +++++- vignettes/design_preservation.Rmd | 6 +++++- vignettes/getting_started.Rmd | 6 +++++- vignettes/recipe_anatomy.Rmd | 6 +++++- 9 files changed, 45 insertions(+), 15 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 317f307..38db2f6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,7 +20,7 @@ authors: repository-code: "https://github.com/max578/masque" url: "https://max578.github.io/masque/" license: MIT -version: 0.15.0 +version: 0.15.1 keywords: - synthetic-data - data-masking diff --git a/DESCRIPTION b/DESCRIPTION index 1860dbb..7b0e4eb 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: masque Title: Structurally Faithful Development Surrogates for Tabular Data -Version: 0.15.0 +Version: 0.15.1 Authors@R: c( person("Max", "Moldovan", email = "max.moldovan@gmail.com", diff --git a/NEWS.md b/NEWS.md index b4b4c80..19affa4 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,9 @@ +# masque 0.15.1 + +* The interactive review in `masque()` now writes the plan table and the + edit menus as messages, so `suppressMessages()` silences them. +* The vignettes restore the `width` option they set. + # masque 0.15.0 ## Breaking changes diff --git a/R/masque.R b/R/masque.R index 33a7ae6..63f8020 100644 --- a/R/masque.R +++ b/R/masque.R @@ -210,9 +210,9 @@ masque <- function(input, sprintf("Proposed masking plan - table '%s'", label) } cli::cli_h2(hdr) - print(roles[, intersect( + cli::cli_verbatim(utils::capture.output(print(roles[, intersect( c("col", "role", "action", "kind"), names(roles) - )]) + )]))) repeat { ans <- tolower(trimws(readline( @@ -226,9 +226,9 @@ masque <- function(input, if (ans %in% c("e", "edit")) { roles <- .masque_edit(roles) cli::cli_alert_success("Roles updated.") - print(roles[, intersect( + cli::cli_verbatim(utils::capture.output(print(roles[, intersect( c("col", "role", "action", "kind"), names(roles) - )]) + )]))) next } if (ans %in% c("q", "quit")) { @@ -268,7 +268,7 @@ masque <- function(input, shown <- roles[, intersect(c("col", "role", "action", "kind"), names(roles) )] - print(shown) + cli::cli_verbatim(utils::capture.output(print(shown))) ans <- trimws(.masque_readline( "Column to edit (name or row number; blank when done): " )) @@ -316,8 +316,8 @@ masque <- function(input, # Numbered single-choice prompt; blank (or an invalid entry) keeps the # current value and returns NULL. .masque_pick <- function(what, current, vocab) { - cat(sprintf( - " %s: %s\n", what, + cli::cli_verbatim(sprintf( + " %s: %s", what, paste(sprintf("[%d] %s", seq_along(vocab), vocab), collapse = " ") )) ans <- trimws(.masque_readline(sprintf( diff --git a/cran-comments.md b/cran-comments.md index eb89f81..41e3bae 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,6 +1,14 @@ -## Submission +## Resubmission -This is the first submission of masque to CRAN. +This is a resubmission of a new package. In response to the review of +0.15.0: + +* The text that `R/masque.R` wrote to the console with `print()` and + `cat()` during the interactive review now goes through `message()` + (via cli), so it can be suppressed. The output that remains on the + console comes from `print()` and `summary()` methods. +* Each vignette saves the user's `options()` before changing `width` and + restores them in its last chunk. ## Test environments diff --git a/vignettes/confidentiality.Rmd b/vignettes/confidentiality.Rmd index 9d50b37..19a0afb 100644 --- a/vignettes/confidentiality.Rmd +++ b/vignettes/confidentiality.Rmd @@ -19,7 +19,7 @@ knitr::opts_chunk$set( fig.height = 4.5, dpi = 150 ) -options(width = 100) +old_options <- options(width = 100) library(masque) set.seed(1) ``` @@ -651,3 +651,7 @@ Epidemiology, 172*(9), 1062-1069. Zandbergen, P. A. (2014). Ensuring confidentiality of geocoded health data: assessing geographic masking strategies for individual-level data. *Advances in Medicine, 2014*, 567049. + +```{r reset-options, include = FALSE} +options(old_options) +``` diff --git a/vignettes/design_preservation.Rmd b/vignettes/design_preservation.Rmd index 1cb0536..55a39ec 100644 --- a/vignettes/design_preservation.Rmd +++ b/vignettes/design_preservation.Rmd @@ -19,7 +19,7 @@ knitr::opts_chunk$set( fig.height = 4.5, dpi = 150 ) -options(width = 100) +old_options <- options(width = 100) set.seed(1) has_agridat <- requireNamespace("agridat", quietly = TRUE) has_ggplot2 <- requireNamespace("ggplot2", quietly = TRUE) @@ -425,3 +425,7 @@ Package versions follow. ```{r session-info} sessionInfo() ``` + +```{r reset-options, include = FALSE} +options(old_options) +``` diff --git a/vignettes/getting_started.Rmd b/vignettes/getting_started.Rmd index 147bd35..bf361e2 100644 --- a/vignettes/getting_started.Rmd +++ b/vignettes/getting_started.Rmd @@ -19,7 +19,7 @@ knitr::opts_chunk$set( fig.height = 4.5, dpi = 150 ) -options(width = 100) +old_options <- options(width = 100) set.seed(1) ``` @@ -392,3 +392,7 @@ Package versions follow. ```{r session-info} sessionInfo() ``` + +```{r reset-options, include = FALSE} +options(old_options) +``` diff --git a/vignettes/recipe_anatomy.Rmd b/vignettes/recipe_anatomy.Rmd index 8b35819..b500fb8 100644 --- a/vignettes/recipe_anatomy.Rmd +++ b/vignettes/recipe_anatomy.Rmd @@ -19,7 +19,7 @@ knitr::opts_chunk$set( fig.height = 4.5, dpi = 150 ) -options(width = 100) +old_options <- options(width = 100) library(masque) set.seed(1) @@ -293,3 +293,7 @@ Package versions follow. ```{r session-info} sessionInfo() ``` + +```{r reset-options, include = FALSE} +options(old_options) +``` From 1d8009088eaa117ce064e90bfe7eeefbb314054c Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 08:45:56 +1030 Subject: [PATCH 02/66] fix: collaborate mode keeps whole-valued and two-valued numeric columns intact --- NEWS.md | 3 ++ R/synthesise_numeric_jitter.R | 46 +++++++++++----- .../testthat/test-synthesise_numeric_jitter.R | 53 +++++++++++++++++++ 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/NEWS.md b/NEWS.md index 19affa4..0427b1e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,9 @@ * The interactive review in `masque()` now writes the plan table and the edit menus as messages, so `suppressMessages()` silences them. * The vignettes restore the `width` option they set. +* In collaborate mode, a double column whose values are all whole numbers + now stays whole-numbered, as an integer column already did. A column + with only two observed values is no longer jittered and keeps both. # masque 0.15.0 diff --git a/R/synthesise_numeric_jitter.R b/R/synthesise_numeric_jitter.R index 4e48e47..9c06d77 100644 --- a/R/synthesise_numeric_jitter.R +++ b/R/synthesise_numeric_jitter.R @@ -4,8 +4,12 @@ #' Takes a synthetic column (drawn from the observed multiset by the #' local copula path) and perturbs it with uniform jitter inside the #' observed measurement resolution, then re-clips to the observed range. -#' Integer columns get bounded stochastic rounding so the output stays -#' integer-class without escaping the observed range. +#' Whole-numbered columns (integer class, or a double whose observed values +#' are all whole) get bounded stochastic rounding, so the output stays +#' whole-numbered in its original storage class without escaping the +#' observed range. A column with at most two distinct observed values is +#' left as drawn: the local copula draw already returns observed values, +#' and jitter followed by rounding would flip a share of them. #' #' Used to push `exact_match_pct` below the audit thresholds (< 1% on #' continuous outcomes; < 5% on continuous covariates) while keeping @@ -24,7 +28,12 @@ synthesise_numeric_collaborate <- function(x_obs, x_new) { cli::cli_abort("`x_obs` and `x_new` must have the same length.") } + if (length(unique(x_obs[!is.na(x_obs)])) <= 2L) { + return(x_new) + } + is_int <- is.integer(x_obs) + is_whole <- .is_whole_valued(x_obs) res <- .detect_resolution(x_obs) if (res > 0) { @@ -37,19 +46,21 @@ synthesise_numeric_collaborate <- function(x_obs, x_new) { x_new <- pmin(pmax(x_new, lo), hi) } - if (is_int) { + if (is_whole) { if (is.finite(lo) && is.finite(hi)) { - x_new <- .bounded_stochastic_round(x_new, lo, hi) + x_new <- .bounded_stochastic_round(x_new, lo, hi, as_int = is_int) } else { - x_new <- as.integer(round(x_new)) + x_new <- round(x_new) + if (is_int) x_new <- as.integer(x_new) } } x_new } -# Internal: measurement resolution per column: 1 for integers, else the -# smallest gap between sorted unique values, floored at a share of the range. +# Internal: measurement resolution per column: 1 for whole-valued columns, +# else the smallest gap between sorted unique values, floored at a share of +# the range. .detect_resolution <- function(x) { x_clean <- x[!is.na(x)] if (length(x_clean) == 0L) { @@ -59,7 +70,7 @@ synthesise_numeric_collaborate <- function(x_obs, x_new) { if (length(uvals) <= 1L) { return(0) } - if (is.integer(x_clean)) { + if (.is_whole_valued(x_clean)) { return(1) } @@ -70,12 +81,23 @@ synthesise_numeric_collaborate <- function(x_obs, x_new) { res } -# Internal: stochastic rounding to integer in [lo, hi]: up with probability -# x - floor(x), else down. -.bounded_stochastic_round <- function(x, lo, hi) { +# Internal: stochastic rounding to a whole number in [lo, hi]: up with +# probability x - floor(x), else down. Integer class unless `as_int = FALSE`. +.bounded_stochastic_round <- function(x, lo, hi, as_int = TRUE) { floor_x <- floor(x) frac <- x - floor_x rolls <- stats::runif(length(x)) rounded <- ifelse(rolls < frac, floor_x + 1, floor_x) - as.integer(pmin(pmax(rounded, lo), hi)) + rounded <- pmin(pmax(rounded, lo), hi) + if (as_int) as.integer(rounded) else rounded +} + +# Internal: TRUE for integer class, or a double whose finite non-missing +# values are all whole numbers. +.is_whole_valued <- function(x) { + if (is.integer(x)) { + return(TRUE) + } + x_fin <- x[is.finite(x)] + is.double(x) && length(x_fin) > 0L && all(x_fin == round(x_fin)) } diff --git a/tests/testthat/test-synthesise_numeric_jitter.R b/tests/testthat/test-synthesise_numeric_jitter.R index 68b94ac..549a7e4 100644 --- a/tests/testthat/test-synthesise_numeric_jitter.R +++ b/tests/testthat/test-synthesise_numeric_jitter.R @@ -154,3 +154,56 @@ test_that("Collaborate mode preserves integer storage on int outcome columns", { expect_gte(rng[1], min(df$grain_count)) expect_lte(rng[2], max(df$grain_count)) }) + +test_that("collaborate keeps a 0/1 double column on its two values", { + set.seed(1) + x_obs <- as.numeric(rbinom(300, 1, 0.3)) + expect_type(x_obs, "double") + x_new <- masque:::synthesise_numeric_local(data.frame(z = x_obs))$z + out <- masque:::synthesise_numeric_collaborate(x_obs, x_new) + expect_type(out, "double") + expect_setequal(unique(out), c(0, 1)) + expect_identical(out, x_new) +}) + +test_that("collaborate does not flip a two-valued integer column", { + set.seed(1) + x_obs <- rbinom(300, 1, 0.3) + x_new <- masque:::synthesise_numeric_local(data.frame(z = x_obs))$z + out <- masque:::synthesise_numeric_collaborate(x_obs, x_new) + expect_type(out, "integer") + expect_identical(out, x_new) +}) + +test_that("collaborate keeps a whole-valued double column whole", { + set.seed(1) + x_obs <- c(as.numeric(sample(1:2000, 400, replace = TRUE)), NA) + x_new <- masque:::synthesise_numeric_local(data.frame(z = x_obs))$z + out <- masque:::synthesise_numeric_collaborate(x_obs, x_new) + expect_type(out, "double") + kept <- out[!is.na(out)] + expect_true(all(kept == round(kept))) + expect_true(all(kept >= 1 & kept <= 2000)) + expect_identical(is.na(out), is.na(x_new)) + # Still perturbed, not a copy of the local draw + expect_lt(mean(out == x_new, na.rm = TRUE), 0.9) +}) + +test_that("collaborate mask() keeps whole-day and 0/1 double outcomes whole", { + set.seed(1) + n <- 400 + df <- data.frame( + Rep = rep(1:4, n / 4), + days = as.numeric(sample(10:3000, n, replace = TRUE)), + flag = as.numeric(rbinom(n, 1, 0.4)), + cov_n = rnorm(n), + stringsAsFactors = FALSE + ) + r <- propose_roles(df) + r$role[r$col %in% c("days", "flag")] <- "outcome" + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = 1)) + s <- synthetic(m) + expect_type(s$days, "double") + expect_true(all(s$days == round(s$days))) + expect_setequal(unique(s$flag), c(0, 1)) +}) From 12ead24ec823af3d0fbb0c477a94d4d99c6ad63a Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 08:51:36 +1030 Subject: [PATCH 03/66] feat: mask(events =) for linked time-to-event outcomes --- NEWS.md | 7 + R/events.R | 253 +++++++++++++++++++++++++++++++++ R/mask.R | 73 +++++++++- R/recipe.R | 3 + R/recipe_print.R | 9 ++ man/mask.Rd | 18 +++ tests/testthat/helper-events.R | 42 ++++++ tests/testthat/test-events.R | 155 ++++++++++++++++++++ 8 files changed, 555 insertions(+), 5 deletions(-) create mode 100644 R/events.R create mode 100644 tests/testthat/helper-events.R create mode 100644 tests/testthat/test-events.R diff --git a/NEWS.md b/NEWS.md index 0427b1e..baa4b49 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,12 @@ # masque 0.15.1 +* `mask()` gains `events =` for linked time-to-event outcomes: several + event times on one row, in the order the events occur, each with a + status. The times are re-simulated as gaps + within each status pattern, so they stay ordered, a row without the + first event keeps both times equal, and the count of each status pattern + is exact. Status patterns move across rows, within treatment when + `conditional = TRUE`. * The interactive review in `masque()` now writes the plan table and the edit menus as messages, so `suppressMessages()` silences them. * The vignettes restore the `width` option they set. diff --git a/R/events.R b/R/events.R new file mode 100644 index 0000000..f948fd0 --- /dev/null +++ b/R/events.R @@ -0,0 +1,253 @@ +# events.R -- Linked time-to-event outcomes for mask(events =). +# +# A spec names the time and status columns of one group of linked events, in +# the order the events occur. Times are turned into non-negative gaps, rows are +# grouped by their status pattern, and the gaps are re-simulated within each +# pattern, then summed back into times. Status patterns are permuted across +# rows (within treatment when the clone is conditional), so the count of each +# pattern is exact and a gap the pattern holds at zero stays zero. + +# Normalise and validate mask()'s `events` against the original table and the +# validated roles. Returns a list of `list(time =, status =)`. +.normalise_events <- function(events, df, roles) { + if (is.null(events)) { + return(list()) + } + if (!is.list(events) || !length(events)) { + .events_refuse( + "`events` must be a list of `list(time =, status =)` specs." + ) + } + if (!is.null(events$time) || !is.null(events$status)) { + events <- list(events) + } + specs <- lapply(events, .check_event_spec, df = df, roles = roles) + + used <- unlist(lapply(specs, function(s) c(s$time, s$status))) + dup <- unique(used[duplicated(used)]) + if (length(dup)) { + .events_refuse( + "Column{?s} {.field {dup}} appear{?s/} in more than one `events` spec." + ) + } + specs +} + +# Validate one spec: columns present, outcome role, whole-numbered statuses, +# no missing values, and times in order in the original. +.check_event_spec <- function(spec, df, roles) { + if (!is.list(spec) || is.null(spec$time) || is.null(spec$status)) { + .events_refuse("Each `events` spec needs `time` and `status` column names.") + } + time <- as.character(spec$time) + status <- as.character(spec$status) + if (length(time) != length(status) || !length(time)) { + .events_refuse( + "An `events` spec needs one `status` column per `time` column." + ) + } + + cols <- c(time, status) + missing <- setdiff(cols, names(df)) + if (length(missing)) { + .events_refuse("Event column{?s} {.field {missing}} not found in `df`.") + } + role_of <- roles$role[match(cols, roles$col)] + action_of <- roles$action[match(cols, roles$col)] + bad_role <- cols[role_of != "outcome" | action_of != "scramble"] + if (length(bad_role)) { + .events_refuse(c( + paste0( + "Event column{?s} {.field {bad_role}} must have role ", + "{.val outcome} and action {.val scramble}." + ), + i = "Set them with {.fn set_role}." + )) + } + + # roles_validate() has already refused a non-numeric outcome. + not_whole <- status[!vapply(df[status], .is_whole_valued, logical(1))] + if (length(not_whole)) { + .events_refuse( + "Event status{?es} {.field {not_whole}} must be whole-numbered codes." + ) + } + has_na <- cols[vapply(df[cols], anyNA, logical(1))] + if (length(has_na)) { + .events_refuse(c( + "Event column{?s} {.field {has_na}} contain{?s/} missing values.", + i = paste0( + "Gaps cannot be rebuilt across a missing time; complete or drop ", + "those rows first." + ) + )) + } + + if (length(time) > 1L) { + tm <- as.matrix(df[time]) + back <- which(apply(tm, 1L, function(r) any(diff(r) < 0))) + if (length(back)) { + .events_refuse(c( + paste0( + "Event times {.field {time}} are out of order in ", + "{length(back)} row{?s} of the original table." + ), + i = paste0( + "List the times in the order the events occur; each must be at ", + "least the one before it." + ) + )) + } + } + list(time = time, status = status) +} + +.events_refuse <- function(msg) { + cli::cli_abort( + msg, + class = c("masque_bad_events_refusal", "orchestra_refusal"), + .envir = parent.frame() + ) +} + +# Synthesise one event spec. `strata` is NULL (arm-blind) or a character +# vector of row strata, normally the treatment, within which status patterns +# are permuted and gaps re-simulated. Returns the synthetic event columns and a +# report for the recipe. +.synthesise_events <- function(df, spec, strata, jitter, min_stratum) { + n <- nrow(df) + k <- length(spec$time) + + # Gaps and status patterns --------------------------------------------------- + tm <- as.matrix(df[spec$time]) + gaps <- tm + if (k > 1L) { + gaps[, -1L] <- tm[, -1L, drop = FALSE] - tm[, -k, drop = FALSE] + } + pattern <- do.call(paste, c(lapply(df[spec$status], as.character), sep = "/")) + + # Permute whole status rows, within strata when the clone is conditional, so + # pattern counts (per stratum) are exact and no real row keeps its own. + grp <- if (is.null(strata)) rep("all", n) else as.character(strata) + perm <- seq_len(n) + for (idx in split(seq_len(n), grp)) { + perm[idx] <- idx[sample.int(length(idx))] + } + pattern_new <- pattern[perm] + cell <- paste(grp, pattern_new, sep = "|") + cell_obs <- paste(grp, pattern, sep = "|") + + # Re-simulate free gaps within each stratum x pattern cell ------------------ + free_in <- function(rows) colSums(gaps[rows, , drop = FALSE] != 0) > 0L + gaps_new <- matrix(0, nrow = n, ncol = k) + pooled <- character() + for (cl in unique(cell)) { + rows_new <- which(cell == cl) + rows_obs <- which(cell_obs == cl) + free <- free_in(rows_obs) + if (!any(free)) next + if (length(rows_obs) >= min_stratum) { + gaps_new[rows_new, free] <- .draw_gaps( + gaps[rows_obs, free, drop = FALSE], length(rows_new), jitter + ) + next + } + # Small cell: fall back to the same pattern across strata, then pool each + # free gap over every pattern in which it is free. + pat <- pattern_new[rows_new[1L]] + pat_rows <- which(pattern == pat) + if (length(pat_rows) >= min_stratum) { + gaps_new[rows_new, free] <- .draw_gaps( + gaps[pat_rows, free, drop = FALSE], length(rows_new), jitter + ) + next + } + pooled <- c(pooled, pat) + for (j in which(free)) { + pool <- .gap_pool(gaps, pattern, j) + if (all(pattern[pool] == pat)) { + .events_refuse(c( + paste0( + "Event gap {j} of {.field {spec$time}} is free only in status ", + "pattern {.val {pat}}, which has {length(pat_rows)} row{?s} ", + "(fewer than {min_stratum})." + ), + i = paste0( + "There is no other pattern to pool it with; the clone would ", + "carry those rows' real times." + ) + )) + } + gaps_new[rows_new, j] <- .draw_gaps( + gaps[pool, j, drop = FALSE], length(rows_new), jitter + ) + } + } + + # Sum gaps back into times and restore storage classes ---------------------- + times_new <- gaps_new + for (j in seq_len(k)[-1L]) { + times_new[, j] <- times_new[, j - 1L] + gaps_new[, j] + } + out <- list() + for (j in seq_len(k)) { + v <- times_new[, j] + out[[spec$time[j]]] <- if (is.integer(df[[spec$time[j]]])) { + as.integer(round(v)) + } else { + v + } + } + for (s in spec$status) { + out[[s]] <- df[[s]][perm] + } + + pooled <- sort(unique(pooled)) + list( + columns = out, + report = list( + time = spec$time, + status = spec$status, + strata = !is.null(strata), + patterns = length(unique(pattern)), + pooled = pooled + ) + ) +} + +# Rows of every status pattern in which gap `j` is free. +.gap_pool <- function(gaps, pattern, j) { + free_pat <- tapply(gaps[, j] != 0, pattern, any) + which(pattern %in% names(free_pat)[free_pat]) +} + +# Draw `n` rows of gaps from the observed gap rows through the empirical +# copula, with collaborate-mode jitter when asked. The pool has at least `n` +# rows; a pool-sized draw is jittered against the pool, then cut to `n`. +.draw_gaps <- function(obs, n, jitter) { + x_obs <- as.data.frame(obs) + x_new <- synthesise_numeric_local(x_obs) + if (isTRUE(jitter)) { + for (j in seq_along(x_new)) { + x_new[[j]] <- synthesise_numeric_collaborate(x_obs[[j]], x_new[[j]]) + } + } + as.matrix(x_new)[seq_len(n), , drop = FALSE] +} + +# One recipe/print line for an event report. +.event_report_line <- function(er) { + line <- sprintf( + "linked events %s re-simulated as gaps within %d status pattern%s%s", + paste(er$time, collapse = " -> "), er$patterns, + if (er$patterns == 1L) "" else "s", + if (isTRUE(er$strata)) ", within treatment" else ", arm-blind" + ) + if (length(er$pooled)) { + line <- paste0( + line, "; pooled small pattern", if (length(er$pooled) > 1L) "s" else "", + " ", paste(er$pooled, collapse = ", ") + ) + } + paste0(line, ".") +} diff --git a/R/mask.R b/R/mask.R index dd17a1a..a9fe5de 100644 --- a/R/mask.R +++ b/R/mask.R @@ -131,6 +131,22 @@ #' survives masking, coarsened; the recipe records the parameters and the site #' grouping it was coarsened under, and [apply_recipe()] retargets to the real #' coordinates. +#' @param events Optional linked time-to-event outcomes. One spec per group of +#' linked events, `list(time = c(...), status = c(...))`, with the time and +#' status columns listed in the order the events occur; pass several specs +#' as a list. Every column must have role `outcome` and action `scramble`, +#' times must be numeric and in order in `df`, status columns must hold +#' whole-numbered codes, and none may be missing. The times are turned into +#' gaps between successive events, the rows' status patterns are permuted +#' (within treatment when `conditional = TRUE`), and the gaps are +#' re-simulated within each status pattern by the empirical copula, then +#' summed back into times. Times therefore stay ordered, a gap that every +#' row of a pattern holds at zero stays zero, and the count of each status +#' pattern is exact. A pattern with fewer than five rows draws each of its +#' free gaps from every pattern in which that gap is free, with a +#' `masque_events_pooled` warning; when no other pattern has the gap free, +#' `mask()` stops. No hazard model is fitted and no covariate-to-event +#' relation is kept. #' @param allow_unmasked_coords Single logical. A masked table must not carry #' a real coordinate, so by default `mask()` **stops** when a column whose #' name says coordinate (`gps`, `lat`, `lon`, `easting`, ...) would be @@ -172,6 +188,7 @@ mask <- function(df, conditional = FALSE, ladder = c("hierarchy", "levels"), coords = NULL, + events = NULL, allow_unmasked_coords = FALSE, quiet = FALSE, .shared_maps = list(), @@ -297,6 +314,9 @@ mask <- function(df, roles$action[roles$col %in% coord_cols] <- "keep" } + # Linked event outcomes skip the copula; they are re-simulated as gaps. + event_specs <- .normalise_events(events, df, roles) + # Refuse a real coordinate the caller has not declared; declared pairs are # jittered below. if (!is.logical(allow_unmasked_coords) || @@ -329,13 +349,32 @@ mask <- function(df, result <- with_rng_state( seed, .mask_orchestrate( - df, roles, mode, opts, .shared_maps, conditional, ladder + df, roles, mode, opts, .shared_maps, conditional, ladder, + events = event_specs ) ) synth <- result$synth level_maps <- result$level_maps warnings_acc <- c(warnings_acc, result$warnings) + event_reports <- result$events + for (er in event_reports) { + warnings_acc <- c(warnings_acc, .event_report_line(er)) + if (length(er$pooled)) { + msg <- sprintf( + paste0( + "events: status pattern%s %s of %s had fewer than %d rows; their ", + "free gaps were drawn from every pattern in which each is free." + ), + if (length(er$pooled) > 1L) "s" else "", + paste(er$pooled, collapse = ", "), + paste(er$time, collapse = ", "), .MIN_STRATUM + ) + warning(warningCondition(msg, class = "masque_events_pooled")) + warnings_acc <- c(warnings_acc, msg) + } + } + # Warn when the ladder dropped a rung or pooled rows: the clone is then less # conditional than requested. cond_report <- result$conditional @@ -479,6 +518,7 @@ mask <- function(df, factor_meta = factor_meta, cleaning = cleaning_rec, coords = coord_reports, + events = event_reports, allow_unmasked_coords = isTRUE(allow_unmasked_coords), warnings = warnings_acc, integrity_fp = digest::digest(is.na(df), algo = "sha256") @@ -579,7 +619,8 @@ mask <- function(df, # Internal: per-action synthesis, with the RNG already seeded. The action # column decides what happens to each column; mode only adds numeric jitter. .mask_orchestrate <- function(df, roles, mode, opts, shared_maps = list(), - conditional = FALSE, ladder = "hierarchy") { + conditional = FALSE, ladder = "hierarchy", + events = list()) { synth <- df level_maps <- list() warnings <- character() @@ -596,11 +637,13 @@ mask <- function(df, level_maps[[col]] <- shared_maps[[col]] } is_shared <- roles$col %in% shared_cols + event_cols <- unlist(lapply(events, function(s) c(s$time, s$status))) + is_event <- roles$col %in% event_cols # All scrambled numeric columns go through one Gaussian copula, fitted within # each treatment x design stratum when `conditional = TRUE`. num_idx <- which(action == "scramble" & kind %in% .numeric_kinds() & - !is_shared) + !is_shared & !is_event) conditional_report <- NULL if (length(num_idx) >= 1L) { x_num <- df[, num_idx, drop = FALSE] @@ -650,12 +693,31 @@ mask <- function(df, } } + # Linked events: status patterns permuted, gaps re-simulated per pattern. + event_reports <- list() + if (length(events)) { + trt_cols <- roles$col[role == "treatment" & action != "drop"] + strata <- if (isTRUE(conditional) && length(trt_cols)) { + do.call(paste, c(lapply(df[trt_cols], as.character), sep = "/")) + } + for (spec in events) { + ev <- .synthesise_events( + df, spec, strata, + jitter = isTRUE(opts$jitter_numeric), min_stratum = .MIN_STRATUM + ) + for (col in names(ev$columns)) { + synth[[col]] <- ev$columns[[col]] + } + event_reports <- c(event_reports, list(ev$report)) + } + } + # Scrambled categorical, date and text columns are row-permuted; treatment # labels never move rows. perm_idx <- which( action == "scramble" & kind %in% c(.categorical_kinds(), .date_kinds()) & - role != "treatment" & !is_shared + role != "treatment" & !is_shared & !is_event ) for (j in perm_idx) { col <- roles$col[j] @@ -730,7 +792,8 @@ mask <- function(df, synth = synth, level_maps = level_maps, warnings = warnings, - conditional = conditional_report + conditional = conditional_report, + events = event_reports ) } diff --git a/R/recipe.R b/R/recipe.R index c72dc6e..fb5a371 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -68,6 +68,9 @@ masque_recipe <- S7::new_class( # One entry per declared coordinate pair: jitter parameters, site grouping # and site count. Empty before 0.10.0. coords = S7::new_property(class = S7::class_list, default = list()), + # One entry per linked-event spec: columns, stratification, pattern count + # and the small patterns that were pooled. Empty before 0.16.0. + events = S7::new_property(class = S7::class_list, default = list()), # TRUE when the caller deliberately wrote a real coordinate through # unmasked. FALSE, and absent on a recipe written before 0.11.0. allow_unmasked_coords = S7::new_property( diff --git a/R/recipe_print.R b/R/recipe_print.R index fe79e95..ba93b28 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -39,6 +39,15 @@ local(S7::method(print, masque_recipe) <- function(x, ...) { } } + events <- S7::prop(x, "events") + if (length(events)) { + cli::cli_h2("Linked events ({length(events)} group{?s})") + for (er in events) { + line <- .event_report_line(er) + cli::cli_bullets(c("*" = "{line}")) + } + } + n_lvl_maps <- length(x@level_maps) n_col_map <- if (is.null(x@column_name_map)) 0L else length(x@column_name_map) cli::cli_h2(paste0( diff --git a/man/mask.Rd b/man/mask.Rd index 081a94d..18c7d7c 100644 --- a/man/mask.Rd +++ b/man/mask.Rd @@ -14,6 +14,7 @@ mask( conditional = FALSE, ladder = c("hierarchy", "levels"), coords = NULL, + events = NULL, allow_unmasked_coords = FALSE, quiet = FALSE, .shared_maps = list(), @@ -115,6 +116,23 @@ survives masking, coarsened; the recipe records the parameters and the site grouping it was coarsened under, and \code{\link[=apply_recipe]{apply_recipe()}} retargets to the real coordinates.} +\item{events}{Optional linked time-to-event outcomes. One spec per group of +linked events, \code{list(time = c(...), status = c(...))}, with the time and +status columns listed in the order the events occur; pass several specs +as a list. Every column must have role \code{outcome} and action \code{scramble}, +times must be numeric and in order in \code{df}, status columns must hold +whole-numbered codes, and none may be missing. The times are turned into +gaps between successive events, the rows' status patterns are permuted +(within treatment when \code{conditional = TRUE}), and the gaps are +re-simulated within each status pattern by the empirical copula, then +summed back into times. Times therefore stay ordered, a gap that every +row of a pattern holds at zero stays zero, and the count of each status +pattern is exact. A pattern with fewer than five rows draws each of its +free gaps from every pattern in which that gap is free, with a +\code{masque_events_pooled} warning; when no other pattern has the gap free, +\code{mask()} stops. No hazard model is fitted and no covariate-to-event +relation is kept.} + \item{allow_unmasked_coords}{Single logical. A masked table must not carry a real coordinate, so by default \code{mask()} \strong{stops} when a column whose name says coordinate (\code{gps}, \code{lat}, \code{lon}, \code{easting}, ...) would be diff --git a/tests/testthat/helper-events.R b/tests/testthat/helper-events.R new file mode 100644 index 0000000..a0700e9 --- /dev/null +++ b/tests/testthat/helper-events.R @@ -0,0 +1,42 @@ +# Simulated illness-death table: recurrence then death, with censoring. A row +# without a recurrence carries time_rec == time_death. +.events_fixture <- function(n = 900L, seed = 11L) { + withr::with_seed(seed, { + arm <- rep(c("A", "B"), length.out = n) + t_rec <- stats::rexp(n, ifelse(arm == "A", 1 / 900, 1 / 1400)) + t_death <- stats::rexp(n, 1 / 2500) + cens <- stats::runif(n, 30, 3000) + status_death <- as.integer(t_death <= cens) + time_death <- round(pmin(t_death, cens)) + status_rec <- as.integer(t_rec < pmin(t_death, cens)) + time_rec <- ifelse(status_rec == 1L, pmin(round(t_rec), time_death), + time_death + ) + data.frame( + id = sprintf("P%04d", seq_len(n)), + arm = arm, + age = round(stats::rnorm(n, 62, 9)), + time_rec = as.numeric(time_rec), + status_rec = status_rec, + time_death = as.numeric(time_death), + status_death = status_death, + stringsAsFactors = FALSE + ) + }) +} + +.events_roles <- function(df) { + r <- propose_roles(df) + r <- set_role(r, "arm", role = "treatment") + for (cn in c("time_rec", "status_rec", "time_death", "status_death")) { + r <- set_role(r, cn, role = "outcome", action = "scramble") + } + r +} + +.events_spec <- list(list( + time = c("time_rec", "time_death"), + status = c("status_rec", "status_death") +)) + +.pattern_of <- function(d) paste(d$status_rec, d$status_death, sep = "/") diff --git a/tests/testthat/test-events.R b/tests/testthat/test-events.R new file mode 100644 index 0000000..1ce67fe --- /dev/null +++ b/tests/testthat/test-events.R @@ -0,0 +1,155 @@ +test_that("linked events keep their structure in the clone", { + df <- .events_fixture() + m <- suppressWarnings( + mask(df, .events_roles(df), seed = 3, events = .events_spec) + ) + s <- synthetic(m) + + expect_true(all(s$time_rec <= s$time_death)) + no_rec <- s$status_rec == 0L + expect_identical(s$time_rec[no_rec], s$time_death[no_rec]) + expect_setequal(unique(s$status_rec), unique(df$status_rec)) + expect_setequal(unique(s$status_death), unique(df$status_death)) + expect_identical( + as.vector(table(.pattern_of(s))), as.vector(table(.pattern_of(df))) + ) + expect_type(s$status_rec, "integer") + expect_false(anyNA(s[, c("time_rec", "time_death")])) +}) + +test_that("event times are re-simulated, not carried by their rows", { + df <- .events_fixture() + m <- suppressWarnings( + mask(df, .events_roles(df), seed = 3, events = .events_spec) + ) + s <- synthetic(m) + # Status patterns are permuted across rows, so most rows change pattern or + # time. + same_row <- .pattern_of(s) == .pattern_of(df) & + s$time_death == df$time_death & s$time_rec == df$time_rec + expect_lt(mean(same_row), 0.05) + # Gaps come from the rows of the same status pattern. + for (p in unique(.pattern_of(df))) { + g_obs <- with(df[.pattern_of(df) == p, ], time_death - time_rec) + g_new <- with(s[.pattern_of(s) == p, ], time_death - time_rec) + expect_true(all(g_new %in% g_obs)) + } +}) + +test_that("events are reproducible under a seed", { + df <- .events_fixture(n = 200L) + r <- .events_roles(df) + m1 <- suppressWarnings(mask(df, r, seed = 9, events = .events_spec)) + m2 <- suppressWarnings(mask(df, r, seed = 9, events = .events_spec)) + expect_identical(synthetic(m1), synthetic(m2)) +}) + +test_that("conditional events keep each arm's pattern counts", { + df <- .events_fixture() + m <- suppressWarnings(mask( + df, .events_roles(df), + seed = 3, events = .events_spec, conditional = TRUE + )) + s <- synthetic(m) + expect_identical( + table(s$arm, .pattern_of(s)), table(df$arm, .pattern_of(df)) + ) + expect_true(recipe(m)@events[[1L]]$strata) +}) + +test_that("collaborate events stay ordered, whole-day and zero-gapped", { + df <- .events_fixture() + m <- suppressWarnings(mask( + df, .events_roles(df), + mode = "collaborate", seed = 3, events = .events_spec + )) + s <- synthetic(m) + expect_true(all(s$time_rec <= s$time_death)) + expect_true(all(s$time_death == round(s$time_death))) + no_rec <- s$status_rec == 0L + expect_identical(s$time_rec[no_rec], s$time_death[no_rec]) +}) + +test_that("the recipe records the event spec and prints it", { + df <- .events_fixture(n = 200L) + m <- suppressWarnings( + mask(df, .events_roles(df), seed = 3, events = .events_spec) + ) + er <- recipe(m)@events + expect_length(er, 1L) + expect_identical(er[[1L]]$time, c("time_rec", "time_death")) + expect_false(er[[1L]]$strata) + expect_message(print(recipe(m)), "Linked events") + + # A recipe written before 0.16.0 has no events property and still prints. + old <- recipe(m) + attr(old, "events") <- NULL + expect_no_error(suppressMessages(print(old))) +}) + +test_that("a small status pattern pools its free gaps and warns", { + df <- .events_fixture() + keep <- .pattern_of(df) != "1/1" + small <- which(!keep)[1:3] + df <- df[c(which(keep), small), ] + expect_warning( + m <- mask(df, .events_roles(df), seed = 3, events = .events_spec), + class = "masque_events_pooled" + ) + s <- synthetic(m) + expect_identical(sum(.pattern_of(s) == "1/1"), 3L) + expect_true(all(s$time_rec <= s$time_death)) + expect_identical(recipe(m)@events[[1L]]$pooled, "1/1") +}) + +test_that("a free gap held only by a small pattern is refused", { + df <- .events_fixture() + # Drop "recurred, censored" rows: the recurrence-to-death gap is then free + # only in the three "recurred, died" rows. + keep <- !.pattern_of(df) %in% c("1/1", "1/0") + df <- df[c(which(keep), which(.pattern_of(df) == "1/1")[1:3]), ] + expect_error( + suppressWarnings( + mask(df, .events_roles(df), seed = 3, events = .events_spec) + ), + class = "masque_bad_events_refusal" + ) +}) + +test_that("bad event specs are refused with a classed error", { + df <- .events_fixture(n = 100L) + r <- .events_roles(df) + refuse <- function(d, ev, roles = r) { + expect_error( + suppressWarnings(mask(d, roles, seed = 1, events = ev)), + class = "masque_bad_events_refusal" + ) + } + spec <- .events_spec[[1L]] + + refuse(df, list(list(time = "time_rec"))) + refuse(df, list(list(time = spec$time, status = "status_rec"))) + refuse(df, list(list(time = c("time_rec", "nope"), status = spec$status))) + refuse(df, list(spec, spec)) + refuse(df, .events_spec, roles = set_role(r, "time_rec", role = "covariate")) + + d2 <- df + d2$status_rec <- d2$status_rec + 0.5 + refuse(d2, .events_spec, roles = .events_roles(d2)) + + d3 <- df + d3$time_death[1L] <- NA + refuse(d3, .events_spec, roles = .events_roles(d3)) + + d4 <- df + d4$time_rec[1L] <- d4$time_death[1L] + 10 + refuse(d4, .events_spec, roles = .events_roles(d4)) +}) + +test_that("a single spec may be passed without the outer list", { + df <- .events_fixture(n = 200L) + m <- suppressWarnings( + mask(df, .events_roles(df), seed = 3, events = .events_spec[[1L]]) + ) + expect_length(recipe(m)@events, 1L) +}) From 2e5f5a691ae99d454de32757590782c4da93d1cb Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:14:35 +1030 Subject: [PATCH 04/66] fix: set layer cleans before proposing and records each table's cleaning Guided masque() on a workbook or folder proposes roles on the cleaned headers. mask_set() accepts roles keyed by original or legalised names and passes each table's cleaning record to mask() through the internal .cleaning argument, so set recipes round-trip renamed headers. Duplicated input column names are refused with masque_duplicate_names_refusal. --- R/clean_table.R | 48 ++++- R/mask.R | 26 ++- R/mask_set.R | 16 +- R/masque.R | 8 +- tests/testthat/test-fix0160-front-door.R | 42 ++++ tests/testthat/test-fix0160-set-names.R | 238 +++++++++++++++++++++++ 6 files changed, 360 insertions(+), 18 deletions(-) create mode 100644 tests/testthat/test-fix0160-front-door.R create mode 100644 tests/testthat/test-fix0160-set-names.R diff --git a/R/clean_table.R b/R/clean_table.R index ea59714..ee0b50b 100644 --- a/R/clean_table.R +++ b/R/clean_table.R @@ -60,6 +60,7 @@ clean_table <- function(df, clean = c("auto", "report", "off"), quiet = FALSE) { # Names are legalised in every mode, since a name rewritten later by # make.names() breaks the round trip. `clean` governs only the steps below. orig_names <- names(df) + .refuse_duplicate_names(orig_names) legal <- .legalise_names(orig_names) name_map <- stats::setNames(legal, orig_names) name_map <- name_map[orig_names != legal] @@ -131,6 +132,26 @@ print.masque_cleaning <- function(x, ...) { ) } +# Two input columns with one name cannot be told apart by the name map, so +# the round trip would swap them; refuse before anything is renamed. +.refuse_duplicate_names <- function(nms) { + dups <- unique(nms[duplicated(nms)]) + if (!length(dups)) { + return(invisible(nms)) + } + cli::cli_abort(c( + "{cli::qty(length(dups))}Column name{?s} used more than once: {.val {dups}}.", + i = paste0( + "masque records renamed columns by their original name, so two ", + "columns with one name cannot be restored on the round trip." + ), + "*" = paste0( + "Give each column its own name first, for example ", + "{.code names(df) <- make.unique(names(df), sep = \"_\")}." + ) + ), class = c("masque_duplicate_names_refusal", "orchestra_refusal")) +} + # Trim, make.names(), then make.unique() for any collisions this introduces. .legalise_names <- function(nms) { trimmed <- trimws(nms) @@ -221,13 +242,19 @@ print.masque_cleaning <- function(x, ...) { # recipe (the data frame itself is not kept). .cleaning_record <- function(cl) { list( - name_map = cl$name_map, - level_fixes = cl$level_fixes, - near_duplicates = cl$near_duplicates, - mode = cl$mode + name_map = cl$name_map, + level_fixes = cl$level_fixes, + near_duplicates = cl$near_duplicates, + n_near_duplicates = nrow(cl$near_duplicates), + mode = cl$mode ) } +# Near-duplicate pairs found, including those not kept on the record. +.near_dup_count <- function(rec) { + rec$n_near_duplicates %||% NROW(rec$near_duplicates) +} + # Re-apply a recipe's cleaning record to the original, so a pipeline written # against the cleaned synthetic runs on it. A clean frame is left as it is. .apply_cleaning_forward <- function(df, cleaning) { @@ -289,6 +316,19 @@ trimws_keep_class <- function(x) { roles } +# Line a roles table up with the cleaned names. A table already keyed by the +# cleaned names is left alone: remapping it would move `Site.Name` onto +# `Site.Name_1` when `Site Name` and `Site.Name` both exist. +.align_roles_cols <- function(roles, name_map, clean_names) { + if (!length(name_map) || !("col" %in% names(roles))) { + return(roles) + } + if (all(roles$col %in% clean_names) && !anyDuplicated(roles$col)) { + return(roles) + } + .remap_roles_cols(roles, name_map) +} + .report_cleaning <- function(cl) { n_names <- length(cl$name_map) n_levels <- length(cl$level_fixes) diff --git a/R/mask.R b/R/mask.R index a9fe5de..2505d88 100644 --- a/R/mask.R +++ b/R/mask.R @@ -192,6 +192,7 @@ mask <- function(df, allow_unmasked_coords = FALSE, quiet = FALSE, .shared_maps = list(), + .cleaning = NULL, ...) { # A misspelled argument silently swallowed by `...` looks like success; # error on anything unused. @@ -285,18 +286,25 @@ mask <- function(df, warnings_acc <- character() # Names are legalised in every clean mode, since a rewritten name breaks the - # round trip; whitespace trimming follows `clean`. - cl <- clean_table(df, clean = clean, quiet = TRUE) - df <- cl$data - roles <- .remap_roles_cols(roles, cl$name_map) - cleaning_rec <- if (length(cl$name_map) || length(cl$level_fixes) || - nrow(cl$near_duplicates)) { - .cleaning_record(cl) + # round trip; whitespace trimming follows `clean`. A caller that cleaned + # already (mask_set(), conform_table()) passes its record instead. + if (is.null(.cleaning)) { + cl <- clean_table(df, clean = clean, quiet = TRUE) + df <- cl$data + .cleaning <- .cleaning_record(cl) + } else { + df <- .as_owned_dt(df) + } + name_map <- .cleaning$name_map + roles <- .align_roles_cols(roles, name_map, names(df)) + cleaning_rec <- if (length(name_map) || length(.cleaning$level_fixes) || + .near_dup_count(.cleaning) > 0L) { + .cleaning } else { NULL } - if (length(cl$name_map)) { - warnings_acc <- c(warnings_acc, .name_repair_message(cl$name_map)) + if (length(name_map)) { + warnings_acc <- c(warnings_acc, .name_repair_message(name_map)) } roles <- withCallingHandlers( diff --git a/R/mask_set.R b/R/mask_set.R index eb95af7..867350d 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -121,11 +121,14 @@ mask_set <- function(input, table_names <- names(tables) # Hygiene per table (names legalised, whitespace trimmed) up front, so - # link detection and roles see the cleaned schema. + # link detection and roles see the cleaned schema. Each table's record goes + # to its recipe so the set round-trips like a single table. cleaned <- lapply(tables, clean_table, clean = clean, quiet = TRUE) tables <- lapply(cleaned, `[[`, "data") + cleaning <- lapply(cleaned, .cleaning_record) - # Roles: proposed per table unless supplied. + # Roles: proposed per table unless supplied. Supplied roles may name the + # original headers or the cleaned ones. if (is.null(roles)) { roles <- lapply(tables, function(tab) { propose_roles(tab, mode = mode) @@ -133,6 +136,11 @@ mask_set <- function(input, names(roles) <- table_names } else { .check_roles_list(roles, tables) + for (nm in table_names) { + roles[[nm]] <- .align_roles_cols( + roles[[nm]], cleaning[[nm]]$name_map, names(tables[[nm]]) + ) + } } # Link detection -> shared alias maps. `.build_shared_maps()` fills each @@ -160,11 +168,11 @@ mask_set <- function(input, roles = roles[[nm]], mode = mode, seed = seed, - clean = "off", # already cleaned at set level alias_names = aln, conditional = conditional, ladder = ladder, - .shared_maps = shared_by_table[[nm]] + .shared_maps = shared_by_table[[nm]], + .cleaning = cleaning[[nm]] ) synth_tables[[nm]] <- synthetic(m) recipes[[nm]] <- recipe(m) diff --git a/R/masque.R b/R/masque.R index 63f8020..e9418cb 100644 --- a/R/masque.R +++ b/R/masque.R @@ -176,9 +176,15 @@ masque <- function(input, )) } + # Propose on the names mask_set() will mask under. It cleans again and + # raises the rename warning once, so the warning is held back here. tables <- read_set(input) proposed <- lapply(names(tables), function(nm) { - r <- propose_roles(tables[[nm]], mode = mode) + cl <- withCallingHandlers( + clean_table(tables[[nm]], clean = clean, quiet = TRUE), + masque_name_repaired = function(w) invokeRestart("muffleWarning") + ) + r <- propose_roles(cl$data, mode = mode) .masque_review(r, ask, quiet, label = nm) }) names(proposed) <- names(tables) diff --git a/tests/testthat/test-fix0160-front-door.R b/tests/testthat/test-fix0160-front-door.R new file mode 100644 index 0000000..043a427 --- /dev/null +++ b/tests/testthat/test-fix0160-front-door.R @@ -0,0 +1,42 @@ +# The guided front door: mode inheritance, argument forwarding, the summary +# and notices it prints, and input it must refuse. + +dup_frame <- function() { + set.seed(3) + d <- data.frame( + a = 1:8, a = round(stats::rnorm(8), 2), b = rep(c("x", "y"), 4), + check.names = FALSE, stringsAsFactors = FALSE + ) + d +} + +no_rename_then <- function(expr) { + withCallingHandlers( + expr, + masque_name_repaired = function(w) stop("a rename warning was raised") + ) +} + +test_that("duplicated input column names give one classed refusal", { + d <- dup_frame() + r <- propose_roles(d, mode = "local", detect = FALSE) + expect_error( + no_rename_then(clean_table(d, quiet = TRUE)), + class = "masque_duplicate_names_refusal" + ) + expect_error( + no_rename_then(mask(d, r, mode = "local", seed = 1)), + class = "masque_duplicate_names_refusal" + ) + expect_error( + no_rename_then(masque(d, mode = "local", ask = FALSE, quiet = TRUE)), + class = "masque_duplicate_names_refusal" + ) + expect_error( + no_rename_then(mask_set(list(t = d), mode = "local", quiet = TRUE)), + class = "masque_duplicate_names_refusal" + ) + # The refusal names the column the user has, not a generated one. + err <- expect_error(clean_table(d, quiet = TRUE), "\"a\"") + expect_false(grepl("a_1", conditionMessage(err), fixed = TRUE)) +}) diff --git a/tests/testthat/test-fix0160-set-names.R b/tests/testthat/test-fix0160-set-names.R new file mode 100644 index 0000000..1bfbb67 --- /dev/null +++ b/tests/testthat/test-fix0160-set-names.R @@ -0,0 +1,238 @@ +# Set layer with headers that need legalising: the guided path proposes on the +# cleaned names, roles keyed either way are accepted, and each table's recipe +# carries its cleaning record so the set round-trips like a single table. + +names_set <- function() { + set.seed(11) + sites <- c("Roseworthy", "Minnipa", "Hart", "Loxton", "Turretfield", "Yanco") + gens <- c("Scope", "Compass", "Spartacus", "Trojan") + g <- expand.grid( + Genotype = gens, Rep = 1:2, site = sites, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + trial <- data.frame( + g$site, g$Genotype, g$Rep, round(3 + stats::rnorm(nrow(g), 0, 0.4), 2), + stringsAsFactors = FALSE + ) + names(trial) <- c("Site Name", "Genotype", "Rep", "Yield (t/ha)") + site_tab <- data.frame( + sites, round(stats::runif(6, 300, 600)), stringsAsFactors = FALSE + ) + names(site_tab) <- c("Site Name", "Rainfall (mm)") + list(trial = trial, sites = site_tab) +} + +# Roles set explicitly. `key` maps a header to the name the roles use. +names_roles <- function(tabs, mode, key = identity) { + rt <- propose_roles(tabs$trial, mode = mode, detect = FALSE) + rt <- set_role(rt, key("Site Name"), role = "design") + rt <- set_role(rt, key("Genotype"), role = "treatment") + rt <- set_role(rt, key("Rep"), role = "design") + rt <- set_role(rt, key("Yield (t/ha)"), role = "outcome") + rs <- propose_roles(tabs$sites, mode = mode, detect = FALSE) + rs <- set_role(rs, key("Site Name"), role = "design") + rs <- set_role(rs, key("Rainfall (mm)"), role = "covariate") + list(trial = rt, sites = rs) +} + +no_repair_warning <- function(expr) { + withCallingHandlers( + expr, + masque_name_repaired = function(w) invokeRestart("muffleWarning") + ) +} + +clean_names_of <- function(tabs) { + lapply(tabs, function(t) no_repair_warning(clean_table(t, quiet = TRUE))$data) +} + +write_names_workbook <- function(tabs) { + path <- file.path(withr::local_tempdir(.local_envir = parent.frame()), + "trials.xlsx") + writexl::write_xlsx(tabs, path = path) + path +} + +write_names_folder <- function(tabs) { + d <- withr::local_tempdir(.local_envir = parent.frame()) + for (nm in names(tabs)) { + data.table::fwrite(tabs[[nm]], file.path(d, paste0(nm, ".csv"))) + } + d +} + +test_that("guided masque() on a workbook with illegal headers masks it", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + path <- write_names_workbook(names_set()) + for (mode in c("local", "collaborate")) { + m <- no_repair_warning( + masque(path, mode = mode, seed = 1, ask = FALSE, quiet = TRUE) + ) + expect_true(S7::S7_inherits(m, masque_set), info = mode) + expect_true( + all(c("Site.Name", "Yield..t.ha.") %in% names(synthetic(m)$trial)), + info = mode + ) + } +}) + +test_that("guided masque() on a folder with illegal headers masks it", { + d <- write_names_folder(names_set()) + for (mode in c("local", "collaborate")) { + m <- no_repair_warning( + masque(d, mode = mode, seed = 1, ask = FALSE, quiet = TRUE) + ) + expect_true(S7::S7_inherits(m, masque_set), info = mode) + expect_true( + all(c("Site.Name", "Rainfall..mm.") %in% names(synthetic(m)$sites)), + info = mode + ) + } +}) + +test_that("roles keyed by the original headers work on every set path", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + tabs <- names_set() + path <- write_names_workbook(tabs) + d <- write_names_folder(tabs) + for (mode in c("local", "collaborate")) { + roles <- names_roles(tabs, mode) + m1 <- no_repair_warning( + mask_set(tabs, roles = roles, mode = mode, seed = 1, quiet = TRUE) + ) + m2 <- no_repair_warning( + masque(path, roles = roles, mode = mode, seed = 1, quiet = TRUE) + ) + m3 <- no_repair_warning( + masque(d, roles = roles, mode = mode, seed = 1, quiet = TRUE) + ) + for (m in list(m1, m2, m3)) { + expect_named( + synthetic(m)$trial, c("Site.Name", "Genotype", "Rep", "Yield..t.ha."), + info = mode + ) + rec <- recipe(m)@recipes$trial + expect_equal( + rec@roles$role[match(c("Genotype", "Yield..t.ha."), rec@roles$col)], + c("treatment", "outcome"), + info = mode + ) + } + } +}) + +test_that("roles keyed by the legalised names keep working", { + tabs <- names_set() + for (mode in c("local", "collaborate")) { + roles <- names_roles(clean_names_of(tabs), mode, key = make.names) + m <- no_repair_warning( + mask_set(tabs, roles = roles, mode = mode, seed = 1, quiet = TRUE) + ) + expect_named( + synthetic(m)$trial, c("Site.Name", "Genotype", "Rep", "Yield..t.ha."), + info = mode + ) + } +}) + +collision_set <- function() { + set.seed(5) + a <- data.frame( + rep(c("north", "south", "east"), each = 6), + rep(c("x1", "x2", "x3"), 6), + round(stats::rnorm(18, 4, 1), 2), + stringsAsFactors = FALSE + ) + names(a) <- c("Site Name", "Site.Name", "Yield") + b <- data.frame( + c("north", "south", "east", "north", "south", "east"), + round(stats::runif(6, 1, 9), 1), + stringsAsFactors = FALSE + ) + names(b) <- c("Site Name", "depth") + list(a = a, b = b) +} + +test_that("Site Name and Site.Name legalise without a duplicate, both keyings", { + tabs <- collision_set() + cleaned <- clean_names_of(tabs) + expect_named(cleaned$a, c("Site.Name", "Site.Name_1", "Yield")) + + roles_orig <- lapply(tabs, propose_roles, mode = "local", detect = FALSE) + roles_clean <- lapply(cleaned, propose_roles, mode = "local", detect = FALSE) + for (roles in list(roles_orig, roles_clean)) { + m <- no_repair_warning( + mask_set(tabs, roles = roles, mode = "local", seed = 1, quiet = TRUE) + ) + expect_named(synthetic(m)$a, c("Site.Name", "Site.Name_1", "Yield")) + back <- unmask(synthetic(m), recipe(m)) + expect_named(back$a, names(tabs$a)) + } + + # The single-table verb accepts both keyings too. + for (r in list(roles_orig$a, roles_clean$a)) { + m1 <- no_repair_warning(mask(tabs$a, r, mode = "local", seed = 1)) + expect_named(synthetic(m1), c("Site.Name", "Site.Name_1", "Yield")) + } + + # The guided path proposes on the cleaned names. + mg <- no_repair_warning( + masque(tabs, mode = "local", seed = 1, ask = FALSE, quiet = TRUE) + ) + expect_named(synthetic(mg)$a, c("Site.Name", "Site.Name_1", "Yield")) +}) + +test_that("each set recipe carries its cleaning record and round-trips", { + tabs <- names_set() + for (mode in c("local", "collaborate")) { + roles <- names_roles(tabs, mode) + m <- no_repair_warning( + mask_set(tabs, roles = roles, mode = mode, seed = 1, quiet = TRUE) + ) + rec <- recipe(m) + expect_false(is.null(rec@recipes$trial@cleaning), info = mode) + expect_equal( + rec@recipes$trial@cleaning$name_map, + c("Site Name" = "Site.Name", "Yield (t/ha)" = "Yield..t.ha."), + info = mode + ) + + # Forward on the raw tables lands in the synthetic namespace. + fwd <- apply_recipe(tabs, rec) + for (nm in names(tabs)) { + expect_identical(names(fwd[[nm]]), names(synthetic(m)[[nm]]), info = mode) + } + + # Back from the clone restores the original headers. + back <- unmask(synthetic(m), rec) + for (nm in names(tabs)) { + expect_identical(names(back[[nm]]), names(tabs[[nm]]), info = mode) + } + + # Forward then back reproduces the original, column by column. + rt <- unmask(fwd, rec) + for (nm in names(tabs)) { + for (col in names(tabs[[nm]])) { + expect_identical( + as.character(rt[[nm]][[col]]), as.character(tabs[[nm]][[col]]), + info = paste(mode, nm, col) + ) + } + } + + # The set round trip matches the single-table one for the same table. + m1 <- no_repair_warning( + mask(tabs$sites, roles$sites, mode = mode, seed = 1) + ) + expect_identical( + names(apply_recipe(tabs$sites, recipe(m1))), names(fwd$sites), + info = mode + ) + expect_identical( + names(unmask(synthetic(m1), recipe(m1))), names(back$sites), + info = mode + ) + } +}) From 7c33ec2ecdcb2abf61a41742115de109c05f1bc1 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:16:14 +1030 Subject: [PATCH 05/66] fix: apply_recipe keeps linked keys the roles mark drop A linked join key is aliased by the shared map in every table, so the clone keeps it. .recipe_dropped_cols() now skips any column with a level map, and mask_set() records linked keys as aliased on each table's recipe. --- R/mask_set.R | 16 +++++- R/recipe_apply.R | 16 +++--- tests/testthat/test-fix0160-link-keys.R | 75 +++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 8 deletions(-) create mode 100644 tests/testthat/test-fix0160-link-keys.R diff --git a/R/mask_set.R b/R/mask_set.R index 867350d..9af0206 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -175,7 +175,9 @@ mask_set <- function(input, .cleaning = cleaning[[nm]] ) synth_tables[[nm]] <- synthetic(m) - recipes[[nm]] <- recipe(m) + recipes[[nm]] <- .record_links_retained( + recipe(m), names(shared_by_table[[nm]]) + ) audits[[nm]] <- m@audit } @@ -315,6 +317,18 @@ mask_set <- function(input, list(groups = kept_groups, shared_by_table = shared_by_table) } +# A linked key is aliased by the shared map whatever its own action, so the +# recipe records it as aliased. +.record_links_retained <- function(rec, shared_cols) { + if (!length(shared_cols)) { + return(rec) + } + roles <- rec@roles + roles$action[roles$col %in% shared_cols] <- "alias" + rec@roles <- roles + rec +} + .table_alias_names <- function(alias_names, cols, linked_cols) { if (isFALSE(alias_names)) { return(FALSE) diff --git a/R/recipe_apply.R b/R/recipe_apply.R index 446bf34..d71527d 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -290,16 +290,18 @@ unmask <- function(x, rec, column = NULL) { } # Internal: columns mask() dropped. Recipes from masque 0.5.0 and earlier -# have no action column; `ignore` was dropped in collaborate mode only. +# have no action column; `ignore` was dropped in collaborate mode only. A +# column with a level map is in the clone (a linked key always is). .recipe_dropped_cols <- function(rec) { roles <- rec@roles - if ("action" %in% names(roles)) { - return(roles$col[roles$action == "drop"]) - } - if (identical(rec@mode, "collaborate")) { - return(roles$col[roles$role == "ignore"]) + dropped <- if ("action" %in% names(roles)) { + roles$col[roles$action == "drop"] + } else if (identical(rec@mode, "collaborate")) { + roles$col[roles$role == "ignore"] + } else { + character() } - character() + setdiff(dropped, names(rec@level_maps)) } # Internal: maps an original-label to its synthetic-label. diff --git a/tests/testthat/test-fix0160-link-keys.R b/tests/testthat/test-fix0160-link-keys.R new file mode 100644 index 0000000..9df1700 --- /dev/null +++ b/tests/testthat/test-fix0160-link-keys.R @@ -0,0 +1,75 @@ +# A linked join key that a table's roles mark `drop` is kept, aliased, in the +# synthetic set, so apply_recipe() must keep it too. + +link_set <- function() { + set.seed(21) + plot_id <- sprintf("P%03d", 1:24) + agronomy <- data.frame( + plot_id = plot_id, + site = rep(c("north", "south", "east", "west"), each = 6), + genotype = rep(c("Scope", "Compass", "Mace"), 8), + yield = round(stats::rnorm(24, 3, 0.5), 2), + stringsAsFactors = FALSE + ) + keep <- sort(sample(24, 16)) + quality <- data.frame( + plot_id = plot_id[keep], + protein = round(stats::rnorm(16, 12, 1), 2), + stringsAsFactors = FALSE + ) + list(agronomy = agronomy, quality = quality) +} + +link_roles <- function(tabs, mode) { + ra <- propose_roles(tabs$agronomy, mode = mode, detect = FALSE) + ra <- set_role(ra, "plot_id", role = "id", action = "drop") + ra <- set_role(ra, "site", role = "design") + ra <- set_role(ra, "genotype", role = "treatment") + ra <- set_role(ra, "yield", role = "outcome") + rq <- propose_roles(tabs$quality, mode = mode, detect = FALSE) + rq <- set_role(rq, "plot_id", role = "id", action = "drop") + rq <- set_role(rq, "protein", role = "outcome") + list(agronomy = ra, quality = rq) +} + +test_that("apply_recipe keeps a linked key the roles mark drop", { + tabs <- link_set() + for (mode in c("local", "collaborate")) { + m <- mask_set( + tabs, roles = link_roles(tabs, mode), mode = mode, seed = 1, + quiet = TRUE + ) + syn <- synthetic(m) + expect_true("plot_id" %in% names(syn$quality), info = mode) + + fwd <- apply_recipe(tabs, recipe(m)) + for (nm in names(tabs)) { + expect_true("plot_id" %in% names(fwd[[nm]]), info = paste(mode, nm)) + expect_identical(names(fwd[[nm]]), names(syn[[nm]]), info = mode) + expect_identical( + as.character(fwd[[nm]]$plot_id), as.character(syn[[nm]]$plot_id), + info = paste(mode, nm) + ) + } + + # A join written on the clone resolves on the retargeted original. + j <- merge(fwd$agronomy, fwd$quality, by = "plot_id") + expect_equal(nrow(j), nrow(tabs$quality), info = mode) + + # unmask() and apply_recipe() agree on the key. + back <- unmask(fwd, recipe(m)) + expect_identical(back$quality$plot_id, tabs$quality$plot_id, info = mode) + } +}) + +test_that("the bundle records a linked key as retained", { + tabs <- link_set() + m <- mask_set( + tabs, roles = link_roles(tabs, "collaborate"), mode = "collaborate", + seed = 1, quiet = TRUE + ) + for (nm in names(tabs)) { + r <- recipe(m)@recipes[[nm]]@roles + expect_false(r$action[r$col == "plot_id"] == "drop", info = nm) + } +}) From 86b145f2a964e54f85a46fe1e21a1c82a82fedb2 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:15:46 +1030 Subject: [PATCH 06/66] fix: coords accepts a bare single pair; an explicit non-keep action on a declared coordinate is refused F5: .normalise_coords() treats a top-level list carrying lat and lon as one pair, as .normalise_events() does. N3: a declared coordinate whose action was chosen explicitly (or whose provenance is unknown) is refused with masque_coords_action_refusal; a proposed default still yields to the declaration and the recipe notes it. --- R/jitter_coordinates.R | 49 ++++++ R/mask.R | 13 +- .../testthat/test-fix0160-conditional-whole.R | 51 ++++++ tests/testthat/test-fix0160-coords.R | 153 ++++++++++++++++++ 4 files changed, 263 insertions(+), 3 deletions(-) create mode 100644 tests/testthat/test-fix0160-conditional-whole.R create mode 100644 tests/testthat/test-fix0160-coords.R diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index 4e68c1d..8850a84 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -462,6 +462,10 @@ jitter_coordinates <- function(df, lat_col, lon_col, ) )) } + # A single pair may be passed bare, with its parameters beside it. + if (any(c("lat", "lon") %in% names(coords))) { + coords <- list(coords) + } lapply(coords, function(spec) { if (is.character(spec)) spec <- as.list(spec) if (is.null(spec$lat) || is.null(spec$lon)) { @@ -544,6 +548,51 @@ jitter_coordinates <- function(df, lat_col, lon_col, unique(unlist(lapply(specs, function(s) c(s$lat, s$lon)), use.names = FALSE)) } +# Declared coordinates are kept and coarsened. A column still on its proposed +# action yields to the declaration, with a note for the recipe; an explicit +# non-keep action, or one whose provenance is unknown, is refused. +.coords_take_keep <- function(roles, coord_cols) { + idx <- which(roles$col %in% coord_cols & roles$action != "keep") + if (!length(idx)) { + return(list(roles = roles, note = character())) + } + cols <- roles$col[idx] + acts <- roles$action[idx] + pa <- attr(roles, "proposed_actions") + proposed <- if (is.null(pa)) { + rep(FALSE, length(idx)) + } else { + cols %in% names(pa) & acts == unname(pa[cols]) + } + if (!all(proposed)) { + bad <- cols[!proposed] + bad_act <- unique(acts[!proposed]) + noun <- if (length(bad) == 1L) "column" else "columns" + verb <- if (length(bad) == 1L) "has" else "have" + bad_code <- paste0("c(", paste0("\"", bad, "\"", collapse = ", "), ")") + cli::cli_abort(c( + paste0( + "Declared coordinate {noun} {.field {bad}} {verb} the action ", + "{.val {bad_act}} in {.arg roles}." + ), + x = paste0( + "A coordinate declared to {.arg coords} is coarsened and kept, which ", + "would override the reviewed action." + ), + i = paste0( + "Set the action to {.val keep} ({.code set_role(roles, {bad_code}, ", + "action = \"keep\")}), or remove the pair from {.arg coords}." + ) + ), class = c("masque_coords_action_refusal", "orchestra_refusal")) + } + roles$action[idx] <- "keep" + note <- sprintf( + "coords: %s had the proposed action %s; set to keep for the geomask.", + paste(cols, collapse = ", "), paste(unique(acts), collapse = "/") + ) + list(roles = roles, note = note) +} + # Jitter each spec in place and return the frame with one report per spec. # Sites are grouped on `original`; each spec draws from its own sub-stream. .apply_coord_jitter <- function(synth, specs, seed, original = NULL) { diff --git a/R/mask.R b/R/mask.R index a9fe5de..6cdedf6 100644 --- a/R/mask.R +++ b/R/mask.R @@ -123,8 +123,13 @@ #' jitter (see [jitter_coordinates()]) instead of being copula-scrambled into #' implausible locations. A pair is a named vector `c(lat = "lat_col", #' lon = "lon_col")` or a named list that also carries jitter parameters -#' (`by`, `method`, `min_km`, `max_km`, `sd_km`, `on_land`); pass several -#' pairs as a list. Defaults to a donut of 5-20 km on land. One displacement +#' (`by`, `method`, `min_km`, `max_km`, `sd_km`, `on_land`); a single pair +#' may be passed bare, and several pairs as a list of pairs. Defaults to a +#' donut of 5-20 km on land. A declared column must have the action `keep` +#' in `roles`, or still carry the action [propose_roles()] proposed, which +#' the declaration replaces with `keep` (noted on the recipe); an +#' explicitly chosen `drop` or `scramble` is refused with a +#' `masque_coords_action_refusal` error. One displacement #' is drawn per **site** and broadcast to that site's rows: `by` names the #' columns of `df` that identify a site, and is omitted when rows sharing an #' identical input coordinate already identify one. A declared pair always @@ -311,7 +316,9 @@ mask <- function(df, coord_specs <- .normalise_coords(coords, df) coord_cols <- .coord_cols(coord_specs) if (length(coord_cols)) { - roles$action[roles$col %in% coord_cols] <- "keep" + taken <- .coords_take_keep(roles, coord_cols) + roles <- taken$roles + warnings_acc <- c(warnings_acc, taken$note) } # Linked event outcomes skip the copula; they are re-simulated as gaps. diff --git a/tests/testthat/test-fix0160-conditional-whole.R b/tests/testthat/test-fix0160-conditional-whole.R new file mode 100644 index 0000000..05bb2bf --- /dev/null +++ b/tests/testthat/test-fix0160-conditional-whole.R @@ -0,0 +1,51 @@ +# A conditional clone adds the dropped design terms back as shifts. Whole-valued +# columns must stay whole on the way back, in either storage class (F4). + +# 24 genotypes x 6 trials x 3 reps: genotype x trial cells hold 3 rows, so the +# ladder drops rep and trial and carries them as shifts. +.fx_count_met <- function(as_integer = FALSE) { + set.seed(4) + trials <- sprintf("T%d", 1:6) + gens <- sprintf("G%02d", 1:24) + g <- expand.grid( + genotype = gens, rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + lam <- 20 + 4 * match(g$trial, trials) + 3 * (match(g$genotype, gens) %% 3) + cnt <- stats::rpois(nrow(g), lambda = lam) + g$count <- if (as_integer) as.integer(cnt) else as.numeric(cnt) + g$height <- round(stats::rnorm(nrow(g), 80, 6), 1) + g[, c("trial", "rep", "genotype", "count", "height")] +} + +.fx_count_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "genotype", role = "treatment", + action = if (mode == "collaborate") "alias" else "keep") + r <- set_role(r, "count", role = "outcome", action = "scramble") + r <- set_role(r, "height", role = "covariate", action = "scramble") + r +} + +test_that("a shifted conditional clone keeps whole-valued columns whole", { + for (storage in c("double", "integer")) { + df <- .fx_count_met(as_integer = storage == "integer") + for (mode in c("local", "collaborate")) { + m <- suppressWarnings(mask( + df, .fx_count_roles(df, mode), mode = mode, seed = 1, + conditional = TRUE + )) + rec <- recipe(m) + # The design that shows the defect: rungs dropped, terms shifted. + expect_true(length(rec@conditioning_shifted) > 0L) + v <- synthetic(m)$count + lab <- paste(storage, mode) + expect_true(all(v == round(v), na.rm = TRUE), label = lab) + expect_identical(class(v), class(df$count), label = lab) + # One-decimal covariates are not whole and are not rounded. + h <- synthetic(m)$height + expect_true(any(h != round(h)), label = lab) + } + } +}) diff --git a/tests/testthat/test-fix0160-coords.R b/tests/testthat/test-fix0160-coords.R new file mode 100644 index 0000000..aa7d4a1 --- /dev/null +++ b/tests/testthat/test-fix0160-coords.R @@ -0,0 +1,153 @@ +# The documented `coords` forms (F5) and the reviewed plan holding against a +# coordinate declaration (N3). + +# Four sites, ten rows each, one true coordinate per site. +.fx_sites <- function() { + set.seed(5) + sites <- data.frame( + site = sprintf("S%d", 1:4), + lat = c(-34.9285, -33.8688, -37.8136, -27.4698), + lon = c(138.6007, 151.2093, 144.9631, 153.0251), + stringsAsFactors = FALSE + ) + df <- sites[rep(1:4, each = 10), ] + rownames(df) <- NULL + df$genotype <- rep(sprintf("G%02d", 1:10), 4) + df$yield <- round(stats::rnorm(40, 3, 0.5), 2) + df +} + +.fx_site_roles <- function(df, coord_action = "keep") { + r <- suppressWarnings(propose_roles(df, mode = "local", detect = FALSE)) + r <- set_role(r, "site", role = "design", action = "keep") + r <- set_role(r, "genotype", role = "treatment", action = "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + if (!is.null(coord_action)) { + r <- set_role(r, c("lat", "lon"), action = coord_action) + } + r +} + +.fx_one_displacement_per_site <- function(df, syn) { + per_site <- tapply(paste(syn$lat, syn$lon), df$site, function(v) { + length(unique(v)) + }) + moved <- tapply(syn$lat != df$lat | syn$lon != df$lon, df$site, all) + all(per_site == 1L) && all(moved) && + length(unique(paste(syn$lat, syn$lon))) == 4L +} + +# ---- F5: every documented form of `coords` --------------------------------- + +test_that("each documented coords form gives one displacement per site", { + df <- .fx_sites() + roles <- .fx_site_roles(df) + forms <- list( + vector = c(lat = "lat", lon = "lon", on_land = FALSE), + wrapped = list(list(lat = "lat", lon = "lon", on_land = FALSE)), + bare = list(lat = "lat", lon = "lon", on_land = FALSE), + bare_params = list( + lat = "lat", lon = "lon", min_km = 1, max_km = 3, on_land = FALSE + ) + ) + for (nm in names(forms)) { + m <- suppressWarnings( + mask(df, roles, mode = "local", seed = 1, coords = forms[[nm]]) + ) + syn <- synthetic(m) + expect_true(.fx_one_displacement_per_site(df, syn), label = nm) + rec <- recipe(m) + expect_length(rec@coords, 1L) + expect_identical(rec@coords[[1]]$n_sites, 4L) + } +}) + +test_that("the bare list form honours its jitter parameters", { + df <- .fx_sites() + m <- suppressWarnings(mask( + df, .fx_site_roles(df), mode = "local", seed = 1, + coords = list(lat = "lat", lon = "lon", min_km = 1, max_km = 3, + on_land = FALSE) + )) + rec <- recipe(m)@coords[[1]] + expect_identical(rec$min_km, 1) + expect_identical(rec$max_km, 3) + syn <- synthetic(m) + r <- 6371 + dlat <- (syn$lat - df$lat) * pi / 180 + dlon <- (syn$lon - df$lon) * pi / 180 + a <- sin(dlat / 2)^2 + + cos(df$lat * pi / 180) * cos(syn$lat * pi / 180) * sin(dlon / 2)^2 + d <- 2 * r * asin(pmin(1, sqrt(a))) + expect_true(all(d >= 0.9 & d <= 3.1)) +}) + +# ---- N3: a declaration never overrides a reviewed action ------------------- + +test_that("an explicit drop or scramble on a declared coordinate is refused", { + df <- .fx_sites() + for (act in c("drop", "scramble")) { + roles <- .fx_site_roles(df, coord_action = act) + expect_error( + suppressWarnings(mask( + df, roles, mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )), + class = "masque_coords_action_refusal" + ) + } +}) + +test_that("a direct edit away from the proposal is also explicit", { + df <- .fx_sites() + roles <- .fx_site_roles(df, coord_action = NULL) + roles$action[roles$col == "lat"] <- "scramble" + err <- tryCatch( + suppressWarnings(mask( + df, roles, mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )), + masque_coords_action_refusal = function(e) e + ) + expect_s3_class(err, "orchestra_refusal") + expect_match(conditionMessage(err), "lat") + expect_match(conditionMessage(err), "keep") +}) + +test_that("the proposed default yields to the declaration and says so", { + df <- .fx_sites() + roles <- .fx_site_roles(df, coord_action = NULL) + # propose_roles() proposes drop for a coordinate-named column. + expect_true(all(roles$action[roles$col %in% c("lat", "lon")] != "keep")) + m <- suppressWarnings(mask( + df, roles, mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )) + rec <- recipe(m) + expect_identical(rec@roles$action[rec@roles$col %in% c("lat", "lon")], + c("keep", "keep")) + expect_true(any(grepl("set to keep for the geomask", rec@warnings))) + expect_true(.fx_one_displacement_per_site(df, synthetic(m))) +}) + +test_that("an explicit keep needs no note", { + df <- .fx_sites() + m <- suppressWarnings(mask( + df, .fx_site_roles(df, coord_action = "keep"), mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )) + expect_false(any(grepl("for the geomask", recipe(m)@warnings))) +}) + +test_that("a roles table without provenance cannot override a declaration", { + df <- .fx_sites() + roles <- .fx_site_roles(df, coord_action = NULL) + attr(roles, "proposed_actions") <- NULL + expect_error( + suppressWarnings(mask( + df, roles, mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )), + class = "masque_coords_action_refusal" + ) +}) From 1617fa4ee3bf35851d9689e75633dda2ae2b9046 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:16:40 +1030 Subject: [PATCH 07/66] fix: a shifted conditional clone keeps whole-valued doubles whole The design shifts added back after the within-stratum copula were rounded only for integer storage. Whole-valued doubles are now rounded the same way, keeping their double class. The range is not clipped. The red test, test-fix0160-conditional-whole.R, landed in 86b145f by mistake. --- R/mask.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R/mask.R b/R/mask.R index 6cdedf6..855f10a 100644 --- a/R/mask.R +++ b/R/mask.R @@ -675,10 +675,14 @@ mask <- function(df, conditional_report$groups, min_stratum = .MIN_STRATUM ) + # Whole-valued columns are rounded in their own storage class; the range + # is left alone, since clipping stacks cells on the real extremes. for (col in names(x_num_new)) { v <- x_num_new[[col]] + shift[, col] x_num_new[[col]] <- if (is.integer(x_num[[col]])) { as.integer(round(v)) + } else if (.is_whole_valued(x_num[[col]])) { + round(v) } else { v } From 429307c829676f8ef4acedfef5213fdd011c0a2a Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:17:21 +1030 Subject: [PATCH 08/66] fix: redact print, format, str and knit_print on every recipe-holding class The recipe bundle printed through S7's default str(), listing every level map and the seed; str() showed the seed of a single recipe; format() errored on the bundle, the masque object and the set. The bundle now prints a header, link names and each table's redacted recipe, and all four classes share a one-line format(), a property-free str() and a knit_print() registered when knitr loads. --- R/recipe_print.R | 68 ++++++- R/recipe_set.R | 65 +++++++ .../testthat/test-fix0160-redaction-surface.R | 175 ++++++++++++++++++ tests/testthat/test-print-dispatch.R | 15 +- 4 files changed, 321 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-fix0160-redaction-surface.R diff --git a/R/recipe_print.R b/R/recipe_print.R index ba93b28..f9adbdb 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -146,12 +146,78 @@ local(S7::method(print, masque_obj) <- function(x, ...) { invisible(x)}) +# local(): see the plot method in detect_plot.R. +local(S7::method(format, masque_obj) <- function(x, ...) { + sprintf( + "", + x@mode, nrow(x@synthetic), ncol(x@synthetic), + if (is.null(x@audit)) "not run" else "stored" + )}) + + +# str() on a recipe-holding object shows the one-line format() and nothing +# else: listing properties would print the maps, the seed, factor levels and +# any property added later. +.str_redacted <- function(object, ...) { + nest_lev <- list(...)$nest.lev %||% 0L + cat(" ", format(object), "\n", sep = "") + if (nest_lev == 0L) { + cat( + " Private: properties are not listed. Use print() for the redacted", + " summary and reveal_maps() for the level maps.\n", + sep = "" + ) + } + invisible(NULL) +} + +# The set classes register theirs in recipe_set.R, which collates later. +local({ + str <- utils::str + S7::method(str, masque_recipe) <- .str_redacted + S7::method(str, masque_obj) <- .str_redacted +}) + + +# The redacted print as plain text lines, in the order they print. +.redacted_lines <- function(x) { + lines <- utils::capture.output( + withCallingHandlers( + print(x), + message = function(m) { + cat(conditionMessage(m)) + invokeRestart("muffleMessage") + } + ) + ) + cli::ansi_strip(lines) +} + +# knitr is suggested only: the method is registered when knitr loads. +.knit_print_redacted <- function(x, ...) { + knitr::asis_output(paste0( + "\n```\n", paste(.redacted_lines(x), collapse = "\n"), "\n```\n" + )) +} + +# S7 finds an external generic by its name in the namespace, so it is bound +# here; S7 registers the methods when knitr loads. +knit_print <- S7::new_external_generic("knitr", "knit_print", "x") + +local({ + S7::method(knit_print, masque_recipe) <- .knit_print_redacted + S7::method(knit_print, masque_obj) <- .knit_print_redacted +}) + + #' Reveal the level maps held inside a recipe #' #' Explicit, audited reveal: prints each `original -> synthetic` level map #' held by the recipe. **Use sparingly.** Recipe maps are at least as #' sensitive as the original data; printing them defeats the redaction -#' built into [print()] and [summary()]. +#' built into the `print()`, `format()`, `str()` and `knitr::knit_print()` +#' methods of recipes, recipe bundles, `masque` and `masque_set` objects. +#' `saveRDS()` and `dput()` write the maps by design. #' #' @param rec A `masque_recipe` object (e.g., from `recipe(m)`). #' @return `rec`, invisibly. diff --git a/R/recipe_set.R b/R/recipe_set.R index 365e5c0..74e76fd 100644 --- a/R/recipe_set.R +++ b/R/recipe_set.R @@ -77,3 +77,68 @@ local(S7::method(print, masque_set) <- function(x, ...) { ) invisible(x)}) + +# local(): see the plot method in detect_plot.R. +local(S7::method(print, masque_recipe_set) <- function(x, ...) { + cli::cli_h1("masque_recipe_set") + cli::cli_bullets(c( + "*" = sprintf("Created: %s", format(x@created_at, "%Y-%m-%d %H:%M:%S %Z")), + "*" = sprintf("Mode: %s", x@mode), + "*" = sprintf("Tables: %d", length(x@recipes)), + "*" = sprintf("masque version: %s", x@masque_version) + )) + + # Link names and tables only; the shared maps stay redacted. + if (length(x@links)) { + cli::cli_h2(sprintf("Cross-table links (%d)", length(x@links))) + for (lk in x@links) { + cli::cli_li(sprintf( + "{.val %s} shared across {.val %s}", + lk$name, paste(lk$tables, collapse = ", ") + )) + } + } + + for (nm in names(x@recipes)) { + cli::cli_text("") + cli::cli_rule(left = "table: {nm}") + print(x@recipes[[nm]]) + } + + cli::cli_text("") + cli::cli_alert_danger( + "The recipe bundle is PRIVATE - never share it with the synthetic set." + ) + cli::cli_text( + "Use {.code reveal_maps(rec, table = )} to inspect level maps explicitly." + ) + invisible(x)}) + + +# local(): see the plot method in detect_plot.R. +local(S7::method(format, masque_recipe_set) <- function(x, ...) { + sprintf( + "", + x@mode, length(x@recipes), length(x@links) + )}) + + +# local(): see the plot method in detect_plot.R. +local(S7::method(format, masque_set) <- function(x, ...) { + sprintf( + "", + x@mode, length(x@synthetic), length(x@recipe@links) + )}) + + +# Redacted str() and knit_print(): see recipe_print.R. +local({ + str <- utils::str + S7::method(str, masque_recipe_set) <- .str_redacted + S7::method(str, masque_set) <- .str_redacted +}) + +local({ + S7::method(knit_print, masque_recipe_set) <- .knit_print_redacted + S7::method(knit_print, masque_set) <- .knit_print_redacted +}) diff --git a/tests/testthat/test-fix0160-redaction-surface.R b/tests/testthat/test-fix0160-redaction-surface.R new file mode 100644 index 0000000..1bbfb29 --- /dev/null +++ b/tests/testthat/test-fix0160-redaction-surface.R @@ -0,0 +1,175 @@ +# F14 + N1: no accidental display of a recipe-holding object shows an original +# label or the seed. saveRDS(), dput() and reveal_maps() are the deliberate +# routes to the maps and are not covered here. + +.leak_tokens <- c("LEAKTOKEN_ALPHA", "LEAKTOKEN_BETA", "424242") + +.leak_plots <- function() { + data.frame( + site = factor(rep(c("LEAKTOKEN_ALPHA", "LEAKTOKEN_BETA"), each = 30L)), + rep = rep(rep(1:3, each = 10L), 2L), + gen = factor(rep(sprintf("G%02d", 1:10), 6L)), + yield = round(5 + seq_len(60L) %% 7 / 3, 2) + ) +} + +.leak_roles <- function(df) { + r <- propose_roles(df, mode = "collaborate") + for (col in names(df)) { + spec <- switch(col, + site = list(role = "design", action = "alias"), + rep = list(role = "design", action = "keep"), + gen = list(role = "treatment", action = "alias"), + yield = list(role = "outcome", action = "scramble"), + rain = list(role = "covariate", action = "scramble"), + year = list(role = "design", action = "keep") + ) + r <- set_role(r, col, role = spec$role, action = spec$action) + } + r +} + +.leak_single <- function() { + df <- .leak_plots() + mask(df, .leak_roles(df), mode = "collaborate", seed = 424242L) +} + +.leak_set <- function() { + sites <- data.frame( + site = factor(rep(c("LEAKTOKEN_ALPHA", "LEAKTOKEN_BETA"), 5L)), + year = rep(2016:2020, each = 2L), + rain = c(410, 395, 520, 480, 300, 350, 610, 590, 455, 470) + ) + tables <- list(plots = .leak_plots(), sites = sites) + roles <- lapply(tables, .leak_roles) + mask_set( + tables, roles = roles, mode = "collaborate", seed = 424242L, quiet = TRUE + ) +} + +# Every display route a user or an IDE can reach without asking for the maps. +.display_routes <- function(x) { + knit <- if (requireNamespace("knitr", quietly = TRUE)) { + value <- NULL + printed <- capture_full(value <- knitr::knit_print(x)) + # knitr writes a character result into the document as it stands. + text <- if (is.character(value)) as.character(value) else character() + paste(c(printed, text), collapse = "\n") + } else { + "" + } + list( + print = capture_full(print(x)), + format = paste(format(x), collapse = "\n"), + str = capture_full(utils::str(x)), + str_nested = capture_full(utils::str(list(held = x))), + summary = capture_full(print(summary(x))), + knit_print = knit + ) +} + +.expect_no_tokens <- function(routes, what) { + for (route in names(routes)) { + for (tok in .leak_tokens) { + expect_false( + grepl(tok, routes[[route]], fixed = TRUE), + info = sprintf("%s: %s() shows %s", what, route, tok) + ) + } + } +} + +test_that("no display route of a single recipe or masque shows a label or the seed", { + skip_if_not_installed("knitr") + m <- .leak_single() + rec <- recipe(m) + # The fixture holds the tokens where an accidental dump would find them. + expect_true(any(grepl("LEAKTOKEN_ALPHA", names(rec@level_maps$site)))) + expect_identical(rec@seed, 424242L) + + .expect_no_tokens(.display_routes(rec), "masque_recipe") + .expect_no_tokens(.display_routes(m), "masque") +}) + +test_that("no display route of a recipe bundle or masque_set shows a label or the seed", { + skip_if_not_installed("knitr") + ms <- .leak_set() + bundle <- recipe(ms) + expect_true(any(grepl( + "LEAKTOKEN_BETA", names(bundle@recipes$plots@level_maps$site) + ))) + + .expect_no_tokens(.display_routes(bundle), "masque_recipe_set") + .expect_no_tokens(.display_routes(ms), "masque_set") +}) + +test_that("format() returns one redacted line for every recipe-holding class", { + m <- .leak_single() + ms <- .leak_set() + for (x in list(recipe(m), m, recipe(ms), ms)) { + out <- format(x) + expect_type(out, "character") + expect_length(out, 1L) + } + expect_match(format(recipe(ms)), "masque_recipe_set") + expect_match(format(ms), "masque_set") +}) + +test_that("the bundle print shows its header, link names and each table's redacted recipe", { + ms <- .leak_set() + out <- capture_full(print(recipe(ms))) + expect_match(out, "masque_recipe_set") + expect_match(out, "site", fixed = TRUE) + expect_match(out, "plots", fixed = TRUE) + expect_match(out, "sites", fixed = TRUE) + # One redacted per-table block per table, each with its own private banner. + per_table <- regmatches(out, gregexpr("Seed: present \\(redacted\\)", out)) + expect_length(per_table[[1L]], 2L) + expect_match(out, "PRIVATE") + # A link's map is not shown, only its name and tables. + expect_false(grepl("site_K", out, fixed = TRUE)) +}) + +test_that("str() lists no property values, including an unknown one holding labels", { + m <- .leak_single() + rec <- recipe(m) + rec@cleaning <- list(near_duplicates = data.frame( + a = "LEAKTOKEN_GAMMA", b = "LEAKTOKEN_GAMMB" + )) + out <- capture_full(utils::str(rec)) + expect_false(grepl("LEAKTOKEN_GAMM", out, fixed = TRUE)) + expect_match(out, "masque_recipe") + expect_match(out, "reveal_maps", fixed = TRUE) +}) + +test_that("knit_print() returns redacted text for knitr", { + skip_if_not_installed("knitr") + m <- .leak_single() + value <- knitr::knit_print(recipe(m)) + expect_s3_class(value, "knit_asis") + expect_match(as.character(value), "masque_recipe") + expect_match(as.character(value), "redacted") +}) + +test_that("a knitted document shows no label or seed", { + skip_if_not_installed("knitr") + skip_if_not_installed("rmarkdown") + m <- .leak_single() + ms <- .leak_set() + env <- new.env() + env$rec <- recipe(m) + env$bundle <- recipe(ms) + env$m <- m + env$ms <- ms + src <- withr::local_tempfile(fileext = ".Rmd") + out <- withr::local_tempfile(fileext = ".md") + writeLines(c( + "```{r}", "rec", "bundle", "m", "ms", "str(bundle)", "```" + ), src) + knitr::knit(src, output = out, envir = env, quiet = TRUE) + text <- paste(readLines(out), collapse = "\n") + for (tok in .leak_tokens) { + expect_false(grepl(tok, text, fixed = TRUE), info = tok) + } + expect_match(text, "masque_recipe_set") +}) diff --git a/tests/testthat/test-print-dispatch.R b/tests/testthat/test-print-dispatch.R index 8e5288f..f3378a3 100644 --- a/tests/testthat/test-print-dispatch.R +++ b/tests/testthat/test-print-dispatch.R @@ -60,7 +60,18 @@ test_that("print methods dispatch from an installed copy of the package", { recipe_lines = length(capture.output( print(recipe(mask(iris, propose_roles(iris), seed = 1L))), type = "message" - )) + )), + # str() and knit_print() redaction must also register when installed. + str_seed = any(grepl("424242", capture.output( + str(recipe(mask(iris, propose_roles(iris), seed = 424242L))) + ), fixed = TRUE)), + knit_class = if (requireNamespace("knitr", quietly = TRUE)) { + class(knitr::knit_print( + recipe(mask(iris, propose_roles(iris), seed = 1L)) + ))[1L] + } else { + "knit_asis" + } ) }, args = list(pkg = pkg, lib = lib, from_source = from_source) @@ -69,4 +80,6 @@ test_that("print methods dispatch from an installed copy of the package", { expect_true(res$cleaning) expect_false(res$raw_list) expect_gt(res$recipe_lines, 0L) + expect_false(res$str_seed) + expect_identical(res$knit_class, "knit_asis") }) From 824dacc7ceac2518de022866b7861c0d71ff426f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:18:48 +1030 Subject: [PATCH 09/66] fix: refuse a collaborate clone below five rows when a column is re-simulated A one-row clone returned the original value and a three-row clone the original permuted. Collaborate mode now refuses with masque_too_few_rows_refusal when nrow < 5 and any column is re-simulated; local mode warns with masque_too_few_rows. A short table of kept, aliased or dropped columns is unaffected. Two set fixtures in the guided and safety-gate tests grow their 2-row side tables to 5 rows. --- R/mask.R | 54 +++++++++++++++- tests/testthat/test-fix0160-small-n.R | 92 +++++++++++++++++++++++++++ tests/testthat/test-masque-guided.R | 5 +- tests/testthat/test-safety-gate.R | 6 +- 4 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 tests/testthat/test-fix0160-small-n.R diff --git a/R/mask.R b/R/mask.R index 855f10a..dd08cbd 100644 --- a/R/mask.R +++ b/R/mask.R @@ -53,7 +53,13 @@ #' @param mode Either `"local"` or `"collaborate"`. When omitted, inherit #' `attr(roles, "mode")`, falling back to `"local"` for a roles table with #' no mode provenance. An explicit collaborate-to-local downgrade raises a -#' classed `masque_mode_downgrade` warning. +#' classed `masque_mode_downgrade` warning. A table of fewer than five rows +#' in which any column would be re-simulated (a scrambled covariate, +#' outcome, date or text column, or an `events` spec) is refused in +#' collaborate mode with `masque_too_few_rows_refusal`, because such a +#' clone is the original values resampled; local mode warns with +#' `masque_too_few_rows`. Kept, aliased and dropped columns are not +#' re-simulated, so a short lookup table of those masks in either mode. #' @param seed Optional integer for reproducibility. #' @param clean Label and column-name hygiene before masking, passed to #' [clean_table()]: one of `"auto"` (default - trim whitespace and @@ -334,6 +340,7 @@ mask <- function(df, ) } .guard_unmasked_coords(df, roles, coord_cols, allow_unmasked_coords) + warnings_acc <- c(warnings_acc, .check_row_floor(df, roles, mode, event_specs)) opts <- mode_defaults(mode) @@ -827,6 +834,51 @@ mask <- function(df, # .conditioning_ladder(). .MIN_STRATUM <- 5L +# Internal: below .MIN_STRATUM rows a re-simulated column is the original +# resampled. Collaborate refuses, local warns; returns the recipe line. +.check_row_floor <- function(df, roles, mode, events = list()) { + n <- nrow(df) + if (n >= .MIN_STRATUM) { + return(character()) + } + resim <- roles$col[ + roles$action == "scramble" & + !roles$role %in% c("treatment", "design") & + roles$kind %in% c(.numeric_kinds(), .categorical_kinds(), .date_kinds()) + ] + resim <- unique(c(resim, unlist(lapply(events, function(s) { + c(s$time, s$status) + })))) + if (!length(resim)) { + return(character()) + } + if (identical(mode, "collaborate")) { + cli::cli_abort(c( + paste0( + "{.arg df} has {n} row{?s}, fewer than {(.MIN_STRATUM)}, and ", + "{.field {resim}} would be re-simulated." + ), + x = paste0( + "A re-simulated column this short is the original values resampled, ", + "so a collaborate clone would disclose them." + ), + i = paste0( + "Keep, alias or drop {cli::qty(length(resim))}{?that column/those ", + "columns}, or mask a table of at least {(.MIN_STRATUM)} rows." + ) + ), class = c("masque_too_few_rows_refusal", "orchestra_refusal")) + } + msg <- sprintf( + paste0( + "df has %d row%s, fewer than %d, so re-simulating %s resamples the ", + "original values; the clone is for owner development only." + ), + n, if (n == 1L) "" else "s", .MIN_STRATUM, paste(resim, collapse = ", ") + ) + warning(warningCondition(msg, class = "masque_too_few_rows")) + msg +} + # Internal: relabel through a shared `original -> alias` map, keeping the # type. Numeric values are matched by their character form. .relabel_with_map <- function(x, map) { diff --git a/tests/testthat/test-fix0160-small-n.R b/tests/testthat/test-fix0160-small-n.R new file mode 100644 index 0000000..beacdf5 --- /dev/null +++ b/tests/testthat/test-fix0160-small-n.R @@ -0,0 +1,92 @@ +# Below five rows a re-simulated column is the original resampled (F20). + +.fx_small <- function(n) { + set.seed(20) + data.frame( + site = sprintf("S%d", seq_len(n)), + genotype = rep(c("G1", "G2"), length.out = n), + yield = round(stats::rnorm(n, 4, 0.5), 2), + stringsAsFactors = FALSE + ) +} + +.fx_small_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, "site", role = "design", action = "keep") + r <- set_role(r, "genotype", role = "treatment", + action = if (mode == "collaborate") "alias" else "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r +} + +test_that("collaborate mode refuses a re-simulated column below five rows", { + for (n in c(1L, 3L, 4L)) { + df <- .fx_small(n) + err <- tryCatch( + suppressWarnings(mask( + df, .fx_small_roles(df, "collaborate"), mode = "collaborate", seed = 1 + )), + masque_too_few_rows_refusal = function(e) e + ) + expect_s3_class(err, "orchestra_refusal") + expect_match(conditionMessage(err), "yield") + } +}) + +test_that("collaborate mode masks five rows", { + df <- .fx_small(5L) + expect_no_error(suppressWarnings(mask( + df, .fx_small_roles(df, "collaborate"), mode = "collaborate", seed = 1 + ))) +}) + +test_that("local mode warns below five rows and masks", { + for (n in c(1L, 3L, 4L)) { + df <- .fx_small(n) + expect_warning( + m <- mask(df, .fx_small_roles(df, "local"), mode = "local", seed = 1), + class = "masque_too_few_rows" + ) + expect_identical(nrow(synthetic(m)), n) + } + df <- .fx_small(5L) + expect_no_warning( + mask(df, .fx_small_roles(df, "local"), mode = "local", seed = 1), + class = "masque_too_few_rows" + ) +}) + +test_that("a small lookup table with nothing re-simulated is not refused", { + sites <- data.frame( + site = c("Loxton", "Minnipa", "Roseworthy", "Turretfield"), + region = c("Riverland", "Eyre", "Mid North", "Mid North"), + station_id = c(24024L, 18052L, 23122L, 23001L), + stringsAsFactors = FALSE + ) + r <- suppressWarnings(propose_roles(sites, mode = "collaborate", + detect = FALSE)) + r <- set_role(r, "site", role = "design", action = "alias") + r <- set_role(r, "region", role = "covariate", action = "alias") + r <- set_role(r, "station_id", role = "id", action = "alias") + m <- suppressWarnings(mask(sites, r, mode = "collaborate", seed = 1)) + expect_identical(nrow(synthetic(m)), 4L) + expect_false(any(synthetic(m)$site %in% sites$site)) +}) + +test_that("an event spec below five rows is refused in collaborate mode", { + df <- data.frame( + arm = c("A", "B", "A", "B"), + t_ev = c(2.5, 4.1, 3.3, 6.0), + ev = c(1L, 0L, 1L, 1L) + ) + r <- suppressWarnings(propose_roles(df, mode = "collaborate", detect = FALSE)) + r <- set_role(r, "arm", role = "treatment", action = "alias") + r <- set_role(r, c("t_ev", "ev"), role = "outcome", action = "scramble") + expect_error( + suppressWarnings(mask( + df, r, mode = "collaborate", seed = 1, + events = list(time = "t_ev", status = "ev") + )), + class = "masque_too_few_rows_refusal" + ) +}) diff --git a/tests/testthat/test-masque-guided.R b/tests/testthat/test-masque-guided.R index bc2f4ec..e396656 100644 --- a/tests/testthat/test-masque-guided.R +++ b/tests/testthat/test-masque-guided.R @@ -19,7 +19,8 @@ test_that("masque() dispatches a named list to the set path", { tabs <- list( a = data.frame(gen = rep(c("x", "y"), 3), v = rnorm(6), stringsAsFactors = FALSE), - b = data.frame(gen = c("x", "y"), w = c(1, 2), stringsAsFactors = FALSE) + b = data.frame(gen = c("x", "y", "x", "y", "x"), w = c(1, 2, 4, 3, 5), + stringsAsFactors = FALSE) ) m <- masque(tabs, mode = "collaborate", seed = 1, ask = FALSE, quiet = TRUE) expect_true(S7::S7_inherits(m, masque_set)) @@ -64,7 +65,7 @@ test_that("masque() writes single-table output to a CSV", { test_that("masque() writes a set to a folder", { tabs <- list( a = data.frame(gen = rep(c("x", "y"), 3), v = rnorm(6)), - b = data.frame(gen = c("x", "y"), w = c(1, 2)) + b = data.frame(gen = c("x", "y", "x", "y", "x"), w = c(1, 2, 4, 3, 5)) ) d <- withr::local_tempdir() out <- file.path(d, "masked_set") diff --git a/tests/testthat/test-safety-gate.R b/tests/testthat/test-safety-gate.R index 988c1bd..24011fc 100644 --- a/tests/testthat/test-safety-gate.R +++ b/tests/testthat/test-safety-gate.R @@ -52,7 +52,8 @@ test_that("guided masque() surfaces the HIGH leakage warning (set)", { f <- high_fixture() tables <- list( trial = f$df, - sites = data.frame(loc = c("x", "y"), depth = c(1.2, 3.4)) + sites = data.frame(loc = c("x", "y", "z", "u", "v"), + depth = c(1.2, 3.4, 2.2, 0.8, 2.9)) ) roles <- list(trial = f$roles, sites = propose_roles(tables$sites)) seen <- FALSE @@ -89,7 +90,8 @@ test_that("blocked write: write_set() refuses and writes nothing", { f <- high_fixture() tables <- list( trial = f$df, - sites = data.frame(loc = c("x", "y"), depth = c(1.2, 3.4)) + sites = data.frame(loc = c("x", "y", "z", "u", "v"), + depth = c(1.2, 3.4, 2.2, 0.8, 2.9)) ) roles <- list(trial = f$roles, sites = propose_roles(tables$sites)) m <- suppressWarnings(mask_set( From b4646b08d4c84e64b73bbf05f6683f519e242b99 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:20:38 +1030 Subject: [PATCH 10/66] fix: save, read, reveal and audit a recipe bundle and a masque_set save_recipe(), read_recipe(), reveal_maps() and audit_mask() refused the set objects that masque() tells users to pass them. All four now dispatch on the bundle: reveal_maps(rec, table = ) reveals one table or each, audit_mask() returns and prints one audit per table and takes a named list of originals. A wrong file names the class found and the version that reads bundles. audit_mask(original =) now applies the recipe's cleaning and reads aliased column names back, which it reported as dropped. --- R/audit.R | 96 ++++++++- R/recipe_io.R | 46 ++++- R/recipe_print.R | 56 +++++- .../testthat/test-fix0160-redaction-surface.R | 5 +- tests/testthat/test-fix0160-set-verbs.R | 188 ++++++++++++++++++ 5 files changed, 364 insertions(+), 27 deletions(-) create mode 100644 tests/testthat/test-fix0160-set-verbs.R diff --git a/R/audit.R b/R/audit.R index 4187d3a..941e6a0 100644 --- a/R/audit.R +++ b/R/audit.R @@ -41,12 +41,16 @@ #' (yield as the outcome, design columns kept) scores `low` on every #' column, and none of its yields equals the original value. #' -#' @param m A `masque` object from [mask()]. +#' @param m A `masque` object from [mask()], or a `masque_set` from +#' [mask_set()]. #' @param original Optional. Required when `m@audit` is NULL (typically #' in local mode). Used to recompute exact-match-pct etc. on demand. +#' The original data frame for a `masque`; for a `masque_set`, a named +#' list holding the original table of every table in the set. #' @param print Logical; if TRUE (default), print a styled report. #' -#' @return The audit data.table, returned invisibly. +#' @return The audit data.table, returned invisibly. For a `masque_set`, a +#' named list of per-table audit data.tables, printed table by table. #' #' @examples #' r <- propose_roles(iris) @@ -58,9 +62,15 @@ #' @seealso [mask()]. #' @export audit_mask <- function(m, original = NULL, print = TRUE) { + if (S7::S7_inherits(m, masque_set)) { + return(.audit_mask_set(m, original, print)) + } if (!S7::S7_inherits(m, masque_obj)) { cli::cli_abort( - "`m` must be a {.cls masque} object; got {.cls {class(m)[1]}}.", + paste0( + "`m` must be a {.cls masque} or {.cls masque_set} object; got ", + "{.cls {class(m)[1]}}." + ), class = c("masque_bad_masque_obj_refusal", "orchestra_refusal") ) } @@ -77,15 +87,89 @@ audit_mask <- function(m, original = NULL, print = TRUE) { i = "Supply {.arg original} to recompute." ), class = c("masque_audit_missing_refusal", "orchestra_refusal")) } - audit <- .compute_audit(original, m@synthetic, m@recipe, m@mode) + audit <- .audit_recompute(original, m@synthetic, m@recipe, m@mode) } if (isTRUE(print)) .print_audit(audit, m@mode) invisible(audit) } -.print_audit <- function(audit, mode) { - cli::cli_h1(sprintf("masque audit (mode = %s)", mode)) +# Internal: audit_mask() for a masque_set; one audit per table, in set order. +.audit_mask_set <- function(m, original, print) { + tables <- names(m@synthetic) + audits <- m@audit + if (is.null(audits)) { + if (is.null(original)) { + cli::cli_abort(c( + paste0( + "Audits are not stored on this masque_set ", + "(local mode does not auto-audit)." + ), + i = "Supply {.arg original} as a named list of the original tables." + ), class = c("masque_audit_missing_refusal", "orchestra_refusal")) + } + if (!is.list(original) || is.data.frame(original) || + is.null(names(original))) { + cli::cli_abort( + "`original` must be a named list of tables for a masque_set.", + class = c("masque_bad_original_refusal", "orchestra_refusal") + ) + } + missing <- setdiff(tables, names(original)) + if (length(missing)) { + cli::cli_abort( + c( + "`original` is missing table{?s}: {.val {missing}}.", + i = "Set tables: {.val {tables}}." + ), + class = c("masque_bad_original_refusal", "orchestra_refusal") + ) + } + audits <- lapply(tables, function(nm) { + .audit_recompute( + original[[nm]], m@synthetic[[nm]], m@recipe@recipes[[nm]], m@mode + ) + }) + names(audits) <- tables + } else { + audits <- lapply(audits, data.table::copy) + } + + if (isTRUE(print)) { + for (nm in tables) .print_audit(audits[[nm]], m@mode, table = nm) + } + invisible(audits) +} + +# Internal: recompute an audit from the caller's original. The recipe's +# cleaning renames the original as mask() did, and aliased column names in +# the clone are read back to the names the recipe's roles use. +.audit_recompute <- function(original, synth, rec, mode) { + if (!is.data.frame(original)) { + cli::cli_abort( + "`original` must be a data frame; got {.cls {class(original)[1]}}.", + class = c("masque_bad_original_refusal", "orchestra_refusal") + ) + } + original <- .apply_cleaning_forward(original, rec@cleaning) + cmap <- rec@column_name_map + if (!is.null(cmap) && length(cmap)) { + inv <- stats::setNames(names(cmap), unlist(cmap, use.names = FALSE)) + nm <- names(synth) + hits <- nm %in% names(inv) + nm[hits] <- unname(inv[nm[hits]]) + synth <- data.table::copy(synth) + data.table::setnames(synth, nm) + } + .compute_audit(original, synth, rec, mode) +} + +.print_audit <- function(audit, mode, table = NULL) { + cli::cli_h1(if (is.null(table)) { + sprintf("masque audit (mode = %s)", mode) + } else { + sprintf("masque audit: table %s (mode = %s)", table, mode) + }) n_high <- sum(audit$leakage_class == "high") n_med <- sum(audit$leakage_class == "medium") diff --git a/R/recipe_io.R b/R/recipe_io.R index 6614abe..1e394d2 100644 --- a/R/recipe_io.R +++ b/R/recipe_io.R @@ -17,7 +17,8 @@ #' organisation's access controls and key-management practice, not #' alongside the synthetic output. #' -#' @param rec A `masque_recipe` object, e.g. from `recipe(m)`. +#' @param rec A `masque_recipe`, or the `masque_recipe_set` bundle of a +#' [mask_set()] result, e.g. from `recipe(m)`. #' @param path File path. By convention, `.rds` extension. #' @param include_simulator Logical. Has no effect; the recipe is always #' written without simulator state. @@ -35,9 +36,13 @@ #' @seealso [read_recipe()], [recipe()]. #' @export save_recipe <- function(rec, path, include_simulator = FALSE) { - if (!S7::S7_inherits(rec, masque_recipe)) { + if (!.is_recipe_like(rec)) { cli::cli_abort( - "`rec` must be a {.cls masque_recipe} object; got {.cls {class(rec)[1]}}." + paste0( + "`rec` must be a {.cls masque_recipe} or {.cls masque_recipe_set} ", + "object; got {.cls {class(rec)[1]}}." + ), + class = c("masque_bad_recipe_refusal", "orchestra_refusal") ) } if (!is.character(path) || length(path) != 1L || !nzchar(path)) { @@ -54,13 +59,13 @@ save_recipe <- function(rec, path, include_simulator = FALSE) { #' Read a masque recipe from disk #' #' Loads a recipe written by [save_recipe()]. Validates that the file -#' contains a `masque_recipe` and informs (does not error) if the recipe -#' was written by a different package version than the one currently -#' installed. +#' contains a `masque_recipe` or a `masque_recipe_set` bundle and informs +#' (does not error) if it was written by a different package version than +#' the one currently installed. Bundles are read from masque 0.16.0 on. #' #' @param path File path. #' -#' @return A `masque_recipe` object. +#' @return A `masque_recipe`, or a `masque_recipe_set` bundle. #' #' @examples #' r <- propose_roles(iris) @@ -81,16 +86,32 @@ read_recipe <- function(path) { cli::cli_abort("File not found: {.file {path}}.") } rec <- readRDS(path) - if (!S7::S7_inherits(rec, masque_recipe)) { + if (!.is_recipe_like(rec)) { cli::cli_abort( - "File at {.file {path}} does not contain a {.cls masque_recipe}." + c( + paste0( + "File at {.file {path}} does not contain a masque recipe; ", + "found {.cls {class(rec)[1]}}." + ), + i = paste0( + "{.fn read_recipe} reads a {.cls masque_recipe}, or a ", + "{.cls masque_recipe_set} bundle from masque 0.16.0 on, as ", + "written by {.fn save_recipe}." + ) + ), + class = c("masque_bad_recipe_file_refusal", "orchestra_refusal") ) } + what <- if (S7::S7_inherits(rec, masque_recipe_set)) { + "masque_recipe_set" + } else { + "masque_recipe" + } current <- as.character(utils::packageVersion("masque")) if (!identical(rec@masque_version, current)) { cli::cli_inform(c( i = paste0( - "Recipe was written by masque {rec@masque_version}; ", + "This {.cls {what}} was written by masque {rec@masque_version}; ", "current is {current}." ), "*" = paste0( @@ -101,3 +122,8 @@ read_recipe <- function(path) { } rec } + +# A single recipe or a set bundle: the objects save_recipe() writes. +.is_recipe_like <- function(x) { + S7::S7_inherits(x, masque_recipe) || S7::S7_inherits(x, masque_recipe_set) +} diff --git a/R/recipe_print.R b/R/recipe_print.R index f9adbdb..aca56f2 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -155,9 +155,7 @@ local(S7::method(format, masque_obj) <- function(x, ...) { )}) -# str() on a recipe-holding object shows the one-line format() and nothing -# else: listing properties would print the maps, the seed, factor levels and -# any property added later. +# str() shows format() only: any property, present or future, may hold labels. .str_redacted <- function(object, ...) { nest_lev <- list(...)$nest.lev %||% 0L cat(" ", format(object), "\n", sep = "") @@ -219,7 +217,11 @@ local({ #' methods of recipes, recipe bundles, `masque` and `masque_set` objects. #' `saveRDS()` and `dput()` write the maps by design. #' -#' @param rec A `masque_recipe` object (e.g., from `recipe(m)`). +#' @param rec A `masque_recipe` object (e.g., from `recipe(m)`), or the +#' `masque_recipe_set` bundle of a [mask_set()] result. +#' @param table For a bundle, the names of the tables whose maps to reveal. +#' `NULL` (default) reveals every table. Must be `NULL` for a single +#' recipe. #' @return `rec`, invisibly. #' @examples #' r <- propose_roles(iris) @@ -229,18 +231,57 @@ local({ #' rec <- recipe(m) #' reveal_maps(rec) #' @export -reveal_maps <- function(rec) { - if (!S7::S7_inherits(rec, masque_recipe)) { +reveal_maps <- function(rec, table = NULL) { + is_set <- S7::S7_inherits(rec, masque_recipe_set) + if (!is_set && !S7::S7_inherits(rec, masque_recipe)) { cli::cli_abort( - "`rec` must be a {.cls masque_recipe} object; got {.cls {class(rec)[1]}}." + paste0( + "`rec` must be a {.cls masque_recipe} or {.cls masque_recipe_set} ", + "object; got {.cls {class(rec)[1]}}." + ), + class = c("masque_bad_recipe_refusal", "orchestra_refusal") ) } + tables <- if (is_set) names(rec@recipes) else character() + if (!is.null(table)) { + if (!is_set) { + cli::cli_abort( + "`table` applies to a recipe bundle; `rec` is a single recipe.", + class = c("masque_bad_table_refusal", "orchestra_refusal") + ) + } + if (!is.character(table) || !length(table) || anyNA(table) || + !all(table %in% tables)) { + cli::cli_abort( + c( + "`table` must name tables in the bundle.", + i = "Bundle tables: {.val {tables}}." + ), + class = c("masque_bad_table_refusal", "orchestra_refusal") + ) + } + tables <- table + } cli::cli_alert_warning( "Revealing sensitive level maps. Proceed at your discretion." ) cli::cli_text("") + if (!is_set) { + .reveal_one(rec) + return(invisible(rec)) + } + for (nm in tables) { + cli::cli_rule(left = "table: {nm}") + .reveal_one(rec@recipes[[nm]]) + cli::cli_text("") + } + invisible(rec) +} + +# Internal: print one recipe's column-name map, level maps and seed. +.reveal_one <- function(rec) { if (!is.null(rec@column_name_map) && length(rec@column_name_map)) { cli::cli_h3("column names (original -> alias)") cmap <- rec@column_name_map @@ -268,7 +309,6 @@ reveal_maps <- function(rec) { cli::cli_h3("seed") cat(sprintf(" %d\n", rec@seed)) } - invisible(rec) } diff --git a/tests/testthat/test-fix0160-redaction-surface.R b/tests/testthat/test-fix0160-redaction-surface.R index 1bbfb29..3e94fd2 100644 --- a/tests/testthat/test-fix0160-redaction-surface.R +++ b/tests/testthat/test-fix0160-redaction-surface.R @@ -1,6 +1,5 @@ -# F14 + N1: no accidental display of a recipe-holding object shows an original -# label or the seed. saveRDS(), dput() and reveal_maps() are the deliberate -# routes to the maps and are not covered here. +# F14 + N1: no accidental display of a recipe-holding object shows a label or +# the seed. saveRDS(), dput() and reveal_maps() show the maps by design. .leak_tokens <- c("LEAKTOKEN_ALPHA", "LEAKTOKEN_BETA", "424242") diff --git a/tests/testthat/test-fix0160-set-verbs.R b/tests/testthat/test-fix0160-set-verbs.R new file mode 100644 index 0000000..189271f --- /dev/null +++ b/tests/testthat/test-fix0160-set-verbs.R @@ -0,0 +1,188 @@ +# F15: save_recipe(), read_recipe(), reveal_maps() and audit_mask() accept the +# recipe bundle and the masque_set that masque() and mask_set() return. + +.verbs_tables <- function() { + set.seed(15L) + plots <- data.frame( + site = factor(rep(c("Alpha", "Bravo", "Charlie"), each = 24L)), + rep = rep(rep(1:3, each = 8L), 3L), + gen = factor(rep(sprintf("Line%02d", 1:8), 9L)), + yield = round(rnorm(72L, 5, 0.6), 2) + ) + sites <- data.frame( + site = factor(rep(c("Alpha", "Bravo", "Charlie"), 3L)), + year = rep(2018:2020, each = 3L), + rain = round(rnorm(9L, 450, 60)) + ) + list(plots = plots, sites = sites) +} + +.verbs_roles <- function(df, mode) { + r <- propose_roles(df, mode = mode) + for (col in names(df)) { + spec <- switch(col, + site = list(role = "design", action = "alias"), + rep = list(role = "design", action = "keep"), + year = list(role = "design", action = "keep"), + gen = list(role = "treatment", action = "alias"), + yield = list(role = "outcome", action = "scramble"), + rain = list(role = "covariate", action = "scramble") + ) + r <- set_role(r, col, role = spec$role, action = spec$action) + } + r +} + +.verbs_set <- function(mode = "collaborate") { + tables <- .verbs_tables() + roles <- lapply(tables, .verbs_roles, mode = mode) + mask_set(tables, roles = roles, mode = mode, seed = 15L, quiet = TRUE) +} + +test_that("save_recipe() and read_recipe() round-trip a recipe bundle", { + bundle <- recipe(.verbs_set()) + path <- withr::local_tempfile(fileext = ".rds") + expect_identical(save_recipe(bundle, path), path) + back <- read_recipe(path) + expect_true(S7::S7_inherits(back, masque:::masque_recipe_set)) + expect_equal(back, bundle) + expect_identical(names(back@recipes), names(bundle@recipes)) + expect_identical(back@links, bundle@links) + for (nm in names(bundle@recipes)) { + expect_identical( + back@recipes[[nm]]@level_maps, bundle@recipes[[nm]]@level_maps + ) + expect_identical(back@recipes[[nm]]@seed, bundle@recipes[[nm]]@seed) + } +}) + +test_that("a read bundle retargets the original tables as the live one does", { + ms <- .verbs_set() + path <- withr::local_tempfile(fileext = ".rds") + save_recipe(recipe(ms), path) + tables <- .verbs_tables() + expect_equal( + apply_recipe(tables, read_recipe(path)), + apply_recipe(tables, recipe(ms)) + ) +}) + +test_that("read_recipe() names the class it found in a wrong file", { + path <- withr::local_tempfile(fileext = ".rds") + saveRDS(iris, path) + err <- expect_error(read_recipe(path), class = "orchestra_refusal") + expect_s3_class(err, "masque_bad_recipe_file_refusal") + msg <- conditionMessage(err) + expect_match(msg, "data.frame", fixed = TRUE) + expect_match(msg, "masque_recipe_set", fixed = TRUE) + expect_match(msg, "0.16.0", fixed = TRUE) +}) + +test_that("read_recipe() names the class and both versions on a mismatch", { + bundle <- recipe(.verbs_set()) + bundle@masque_version <- "0.0.9" + path <- withr::local_tempfile(fileext = ".rds") + saveRDS(bundle, path) + msgs <- testthat::capture_messages(back <- read_recipe(path)) + joined <- paste(msgs, collapse = "\n") + expect_match(joined, "masque_recipe_set", fixed = TRUE) + expect_match(joined, "0.0.9", fixed = TRUE) + expect_match( + joined, as.character(utils::packageVersion("masque")), fixed = TRUE + ) + expect_true(S7::S7_inherits(back, masque:::masque_recipe_set)) +}) + +test_that("reveal_maps() on a bundle reveals each table, or the one named", { + bundle <- recipe(.verbs_set()) + all_tables <- capture_full(reveal_maps(bundle)) + expect_match(all_tables, "Revealing sensitive") + expect_match(all_tables, "plots", fixed = TRUE) + expect_match(all_tables, "sites", fixed = TRUE) + expect_match(all_tables, "Alpha", fixed = TRUE) + expect_match(all_tables, "Line03", fixed = TRUE) + + # The genotype map lives on the plots table only. + one <- capture_full(reveal_maps(bundle, table = "sites")) + expect_match(one, "Alpha", fixed = TRUE) + expect_false(grepl("Line03", one, fixed = TRUE)) + + expect_invisible(reveal_maps(bundle, table = "plots")) + expect_error( + reveal_maps(bundle, table = "nope"), + class = "masque_bad_table_refusal" + ) + expect_error( + reveal_maps(bundle@recipes$plots, table = "plots"), + class = "masque_bad_table_refusal" + ) +}) + +test_that("audit_mask() on a collaborate set returns and prints one audit per table", { + ms <- .verbs_set() + out <- audit_mask(ms, print = FALSE) + expect_type(out, "list") + expect_identical(names(out), c("plots", "sites")) + for (nm in names(out)) { + expect_s3_class(out[[nm]], "data.table") + expect_identical(out[[nm]]$col, names(.verbs_tables()[[nm]])) + } + printed <- capture_full(audit_mask(ms)) + expect_match(printed, "masque audit", fixed = TRUE) + expect_match(printed, "plots", fixed = TRUE) + expect_match(printed, "sites", fixed = TRUE) +}) + +test_that("audit_mask() on a local set takes the originals as a named list", { + ms <- .verbs_set(mode = "local") + expect_error(audit_mask(ms), class = "masque_audit_missing_refusal") + tables <- .verbs_tables() + out <- audit_mask(ms, original = tables, print = FALSE) + expect_identical(names(out), names(tables)) + expect_s3_class(out$plots, "data.table") + # Each table's audit equals the single-table audit of that table's clone. + single <- masque:::.compute_audit( + tables$sites, synthetic(ms)$sites, recipe(ms)@recipes$sites, "local" + ) + expect_equal(out$sites, single, ignore_attr = TRUE) + + expect_error( + audit_mask(ms, original = tables$plots), + class = "masque_bad_original_refusal" + ) + expect_error( + audit_mask(ms, original = tables["plots"]), + class = "masque_bad_original_refusal" + ) +}) + +test_that("audit_mask(original =) reads through aliased and repaired column names", { + r <- propose_roles(iris) + r <- set_role(r, "Sepal.Length", role = "outcome") + m <- mask(iris, r, mode = "local", seed = 1L, alias_names = TRUE) + a <- audit_mask(m, original = iris, print = FALSE) + expect_false(any(a$alias_status == "dropped")) + expect_true(is.finite(a$exact_match_pct[a$col == "Sepal.Length"])) + + d <- data.frame( + `Site Name` = rep(c("north", "south"), 10L), + y = seq_len(20L) / 4, + check.names = FALSE + ) + r2 <- propose_roles(d) + r2 <- set_role(r2, "Site Name", role = "design", action = "keep") + r2 <- set_role(r2, "y", role = "outcome", action = "scramble") + m2 <- suppressWarnings(mask(d, r2, mode = "local", seed = 1L)) + a2 <- audit_mask(m2, original = d, print = FALSE) + expect_identical(a2$n_unique_levels[a2$col == "Site.Name"], 2L) +}) + +test_that("audit_mask(original =) refuses an original that is not a table", { + r <- propose_roles(iris) + r <- set_role(r, "Sepal.Length", role = "outcome") + m <- mask(iris, r, mode = "local", seed = 1L) + expect_error( + audit_mask(m, original = list(iris)), + class = "masque_bad_original_refusal" + ) +}) From f6771894f76a3701ed8fe9e0f69c61603ac6b46c Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:21:29 +1030 Subject: [PATCH 11/66] fix: cap a linked event's final time at the pattern's longest follow-up Summed gaps could run past the observed maximum (28.8 to 44.0 against 23.7). Each row's final time, with the times tied to it by zero gaps, is now capped at the longest final time of the rows its gaps were drawn from; earlier times are left alone, and a row whose earlier time already passes the cap is scaled as a whole so no positive gap becomes a tie. The event report and recipe line record the capped and rescaled counts. --- R/events.R | 43 ++++++- R/mask.R | 8 +- tests/testthat/test-events.R | 7 +- tests/testthat/test-fix0160-events-clip.R | 136 ++++++++++++++++++++++ 4 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 tests/testthat/test-fix0160-events-clip.R diff --git a/R/events.R b/R/events.R index f948fd0..d9cc8f7 100644 --- a/R/events.R +++ b/R/events.R @@ -140,6 +140,8 @@ # Re-simulate free gaps within each stratum x pattern cell ------------------ free_in <- function(rows) colSums(gaps[rows, , drop = FALSE] != 0) > 0L gaps_new <- matrix(0, nrow = n, ncol = k) + # Longest observed final time of the rows each draw came from. + cap <- rep(Inf, n) pooled <- character() for (cl in unique(cell)) { rows_new <- which(cell == cl) @@ -150,6 +152,7 @@ gaps_new[rows_new, free] <- .draw_gaps( gaps[rows_obs, free, drop = FALSE], length(rows_new), jitter ) + cap[rows_new] <- max(tm[rows_obs, k]) next } # Small cell: fall back to the same pattern across strata, then pool each @@ -160,8 +163,12 @@ gaps_new[rows_new, free] <- .draw_gaps( gaps[pat_rows, free, drop = FALSE], length(rows_new), jitter ) + cap[rows_new] <- max(tm[pat_rows, k]) next } + # A pooled pattern's own maximum is a real row's time; cap at the + # table's longest follow-up instead. + cap[rows_new] <- max(tm[, k]) pooled <- c(pooled, pat) for (j in which(free)) { pool <- .gap_pool(gaps, pattern, j) @@ -189,6 +196,8 @@ for (j in seq_len(k)[-1L]) { times_new[, j] <- times_new[, j - 1L] + gaps_new[, j] } + capped <- .cap_final_time(times_new, gaps_new, cap) + times_new <- capped$times out <- list() for (j in seq_len(k)) { v <- times_new[, j] @@ -210,11 +219,37 @@ status = spec$status, strata = !is.null(strata), patterns = length(unique(pattern)), - pooled = pooled + pooled = pooled, + clipped = capped$clipped, + rescaled = capped$rescaled ) ) } +# Cap each row's final time at `cap`. The final time moves together with the +# times tied to it by zero gaps, so those gaps stay zero; the earlier times +# are left alone. When the time before that block is already at or past the +# cap, capping would turn a positive gap into a zero gap (a tie the pattern +# need not have) or a negative one, so the whole row is scaled by +# cap / final time instead: order kept, zero gaps zero, positive gaps +# positive. +.cap_final_time <- function(times, gaps, cap) { + k <- ncol(times) + over <- which(times[, k] > cap) + rescaled <- 0L + for (i in over) { + pos <- which(gaps[i, -1L] > 0) + 1L + z <- if (length(pos)) max(pos) else 1L + if (z == 1L || times[i, z - 1L] < cap[i]) { + times[i, z:k] <- cap[i] + } else { + times[i, ] <- times[i, ] * cap[i] / times[i, k] + rescaled <- rescaled + 1L + } + } + list(times = times, clipped = length(over), rescaled = rescaled) +} + # Rows of every status pattern in which gap `j` is free. .gap_pool <- function(gaps, pattern, j) { free_pat <- tapply(gaps[, j] != 0, pattern, any) @@ -249,5 +284,11 @@ " ", paste(er$pooled, collapse = ", ") ) } + if (isTRUE(er$clipped > 0L)) { + line <- paste0( + line, "; final time capped at the pattern's longest observed ", + "follow-up in ", er$clipped, " row", if (er$clipped > 1L) "s" else "" + ) + } paste0(line, ".") } diff --git a/R/mask.R b/R/mask.R index dd08cbd..a202843 100644 --- a/R/mask.R +++ b/R/mask.R @@ -153,7 +153,13 @@ #' re-simulated within each status pattern by the empirical copula, then #' summed back into times. Times therefore stay ordered, a gap that every #' row of a pattern holds at zero stays zero, and the count of each status -#' pattern is exact. A pattern with fewer than five rows draws each of its +#' pattern is exact. A summed final time is capped at the longest final +#' time observed in the rows its gaps were drawn from (the pattern, within +#' treatment when conditional), so no clone row runs past the observed +#' follow-up. Only the final time and the times tied to it move; a row +#' whose earlier time already passes the cap is scaled down as a whole, +#' which keeps its order and its zero gaps. The recipe records how many +#' rows were capped. A pattern with fewer than five rows draws each of its #' free gaps from every pattern in which that gap is free, with a #' `masque_events_pooled` warning; when no other pattern has the gap free, #' `mask()` stops. No hazard model is fitted and no covariate-to-event diff --git a/tests/testthat/test-events.R b/tests/testthat/test-events.R index 1ce67fe..137a438 100644 --- a/tests/testthat/test-events.R +++ b/tests/testthat/test-events.R @@ -28,11 +28,14 @@ test_that("event times are re-simulated, not carried by their rows", { same_row <- .pattern_of(s) == .pattern_of(df) & s$time_death == df$time_death & s$time_rec == df$time_rec expect_lt(mean(same_row), 0.05) - # Gaps come from the rows of the same status pattern. + # Gaps come from the rows of the same status pattern, except where a row's + # final time was capped at the pattern's longest follow-up. for (p in unique(.pattern_of(df))) { g_obs <- with(df[.pattern_of(df) == p, ], time_death - time_rec) g_new <- with(s[.pattern_of(s) == p, ], time_death - time_rec) - expect_true(all(g_new %in% g_obs)) + capped <- s$time_death[.pattern_of(s) == p] == + max(df$time_death[.pattern_of(df) == p]) + expect_true(all(g_new %in% g_obs | capped)) } }) diff --git a/tests/testthat/test-fix0160-events-clip.R b/tests/testthat/test-fix0160-events-clip.R new file mode 100644 index 0000000..a026a92 --- /dev/null +++ b/tests/testthat/test-fix0160-events-clip.R @@ -0,0 +1,136 @@ +# Summed gaps may not run past the longest observed follow-up of a status +# pattern; only the final time is capped, so no tie is invented (F12). + +# Progression then death, censored at most at 24: 300 rows, two arms. +.fx_pfs <- function() { + withr::with_seed(7L, { + n <- 300L + arm <- sample(rep(c("A", "B"), each = n / 2)) + age <- round(stats::rnorm(n, 60, 10)) + t_d <- stats::rexp(n, rate = ifelse(arm == "A", 0.10, 0.07)) + t_p <- stats::rexp(n, rate = ifelse(arm == "A", 0.15, 0.10)) + cens <- stats::runif(n, 2, 24) + time_death <- round(pmin(t_d, cens), 2) + died <- as.integer(t_d <= cens) + time_prog <- round(pmin(t_p, time_death), 2) + prog <- as.integer(t_p <= time_death) + data.frame( + arm = arm, age = age, time_prog = time_prog, prog = prog, + time_death = time_death, died = died, stringsAsFactors = FALSE + ) + }) +} + +.fx_pfs_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, "arm", role = "treatment", + action = if (mode == "collaborate") "alias" else "keep") + r <- set_role(r, "age", role = "covariate", action = "scramble") + r <- set_role(r, c("time_prog", "prog", "time_death", "died"), + role = "outcome", action = "scramble") + r +} + +.fx_pfs_spec <- list(list( + time = c("time_prog", "time_death"), status = c("prog", "died") +)) + +.fx_pfs_runs <- list( + list(mode = "local", cond = FALSE), + list(mode = "local", cond = TRUE), + list(mode = "collaborate", cond = FALSE) +) + +test_that("the final time never exceeds the pattern's observed maximum", { + df <- .fx_pfs() + pat <- paste(df$prog, df$died, sep = "/") + obs_max <- tapply(df$time_death, pat, max) + clipped <- integer() + for (ru in .fx_pfs_runs) { + roles <- .fx_pfs_roles(df, ru$mode) + for (s in 1:15) { + m <- suppressWarnings(mask( + df, roles, mode = ru$mode, seed = s, conditional = ru$cond, + events = .fx_pfs_spec + )) + syn <- synthetic(m) + lab <- sprintf("%s cond=%s seed=%d", ru$mode, ru$cond, s) + pat_new <- paste(syn$prog, syn$died, sep = "/") + clone_max <- tapply(syn$time_death, pat_new, max) + expect_true(all(clone_max <= obs_max[names(clone_max)]), label = lab) + expect_identical(sum(syn$time_prog > syn$time_death), 0L, label = lab) + expect_identical(c(table(pat_new)), c(table(pat)), label = lab) + no_prog <- syn$prog == 0L + expect_identical(syn$time_prog[no_prog], syn$time_death[no_prog], + label = lab) + clipped <- c(clipped, recipe(m)@events[[1]]$clipped) + } + } + expect_type(clipped, "integer") + expect_true(all(clipped >= 0L)) + # Every seed ran past the cap before the fix. + expect_true(sum(clipped > 0L) >= 15L) +}) + +test_that("a conditional clone caps within each arm", { + df <- .fx_pfs() + cell <- paste(df$arm, df$prog, df$died, sep = "/") + obs_max <- tapply(df$time_death, cell, max) + roles <- .fx_pfs_roles(df, "local") + for (s in 1:5) { + m <- suppressWarnings(mask( + df, roles, mode = "local", seed = s, conditional = TRUE, + events = .fx_pfs_spec + )) + syn <- synthetic(m) + cell_new <- paste(syn$arm, syn$prog, syn$died, sep = "/") + clone_max <- tapply(syn$time_death, cell_new, max) + expect_true(all(clone_max <= obs_max[names(clone_max)])) + } +}) + +test_that("intermediate times are not clipped", { + # Local draws return observed gaps, so a progression time outside the + # observed set can only come from a row rescaled as a whole. + df <- .fx_pfs() + roles <- .fx_pfs_roles(df, "local") + for (s in 1:15) { + m <- suppressWarnings(mask( + df, roles, mode = "local", seed = s, events = .fx_pfs_spec + )) + syn <- synthetic(m) + ev <- recipe(m)@events[[1]] + prog_rows <- syn$prog == 1L + off_set <- sum(!syn$time_prog[prog_rows] %in% df$time_prog[df$prog == 1L]) + expect_lte(off_set, ev$rescaled) + } +}) + +test_that("the recipe records how many rows were capped", { + df <- .fx_pfs() + m <- suppressWarnings(mask( + df, .fx_pfs_roles(df, "local"), mode = "local", seed = 1, + events = .fx_pfs_spec + )) + ev <- recipe(m)@events[[1]] + expect_true(ev$clipped > 0L) + expect_true(any(grepl("final time capped", recipe(m)@warnings))) +}) + +test_that("the cap moves the final block, or rescales a row it cannot clip", { + # Row 1: plain clip of the last time. Row 2: the last two times are tied by a + # zero gap and move together. Row 3: the second time is already past the + # cap, so the row is scaled. Row 4: under the cap, untouched. + gaps <- rbind(c(2, 3, 4), c(2, 5, 0), c(6, 5, 1), c(1, 1, 1)) + times <- t(apply(gaps, 1L, cumsum)) + cap <- c(7, 6, 8, 10) + out <- .cap_final_time(times, gaps, cap) + expect_identical(out$clipped, 3L) + expect_identical(out$rescaled, 1L) + expect_equal(out$times[1, ], c(2, 5, 7)) + expect_equal(out$times[2, ], c(2, 6, 6)) + expect_equal(out$times[3, ], c(6, 11, 12) * 8 / 12) + expect_equal(out$times[4, ], c(1, 2, 3)) + expect_true(all(t(apply(out$times, 1L, diff)) >= 0)) + expect_true(all(out$times[, 3] <= cap)) +}) From b965d883970a0a6e363ee7eb459ec5497d3e9b86 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:21:54 +1030 Subject: [PATCH 12/66] fix: classed refusals for a non-masque accessor input, a non-.rds recipe path and a fractional seed recipe() and synthetic() passed the second message fragment into cli_abort()'s class, so the class was the template string and the message stopped at 'got'. save_recipe() wrote RDS under any name, including .csv; it now refuses a path not ending in .rds. A seed of 1.5 behaved as 1 and a seed of Inf failed inside set.seed(); every seed now passes one check (whole, finite, integer range) with masque_bad_seed_refusal, reached from mask(), mask_set() and masque(). --- R/recipe.R | 14 ++++-- R/recipe_io.R | 21 ++++++-- R/rng.R | 35 ++++++++++--- tests/testthat/test-fix0160-refusals.R | 70 ++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 15 deletions(-) create mode 100644 tests/testthat/test-fix0160-refusals.R diff --git a/R/recipe.R b/R/recipe.R index fb5a371..a40701a 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -125,8 +125,11 @@ synthetic <- function(m) { } if (!S7::S7_inherits(m, masque_obj)) { cli::cli_abort( - "`m` must be a {.cls masque} or {.cls masque_set} object; got ", - "{.cls {class(m)[1]}}." + paste0( + "`m` must be a {.cls masque} or {.cls masque_set} object; got ", + "{.cls {class(m)[1]}}." + ), + class = c("masque_bad_masque_obj_refusal", "orchestra_refusal") ) } data.table::copy(m@synthetic) @@ -154,8 +157,11 @@ recipe <- function(m) { } if (!S7::S7_inherits(m, masque_obj)) { cli::cli_abort( - "`m` must be a {.cls masque} or {.cls masque_set} object; got ", - "{.cls {class(m)[1]}}." + paste0( + "`m` must be a {.cls masque} or {.cls masque_set} object; got ", + "{.cls {class(m)[1]}}." + ), + class = c("masque_bad_masque_obj_refusal", "orchestra_refusal") ) } m@recipe diff --git a/R/recipe_io.R b/R/recipe_io.R index 1e394d2..573ca75 100644 --- a/R/recipe_io.R +++ b/R/recipe_io.R @@ -19,7 +19,8 @@ #' #' @param rec A `masque_recipe`, or the `masque_recipe_set` bundle of a #' [mask_set()] result, e.g. from `recipe(m)`. -#' @param path File path. By convention, `.rds` extension. +#' @param path File path ending in `.rds` (any case); other extensions are +#' refused. #' @param include_simulator Logical. Has no effect; the recipe is always #' written without simulator state. #' @@ -45,12 +46,26 @@ save_recipe <- function(rec, path, include_simulator = FALSE) { class = c("masque_bad_recipe_refusal", "orchestra_refusal") ) } - if (!is.character(path) || length(path) != 1L || !nzchar(path)) { - cli::cli_abort("`path` must be a single non-empty string.") + if (!is.character(path) || length(path) != 1L || is.na(path) || + !nzchar(path)) { + cli::cli_abort( + "`path` must be a single non-empty string.", + class = c("masque_bad_recipe_path_refusal", "orchestra_refusal") + ) } if (!is.logical(include_simulator) || length(include_simulator) != 1L) { cli::cli_abort("`include_simulator` must be a single logical.") } + # The file is RDS whatever its name says; a .csv name would mislead. + if (!grepl("\\.rds$", path, ignore.case = TRUE)) { + cli::cli_abort( + c( + "`path` must end in {.file .rds}; got {.file {path}}.", + i = "{.fn save_recipe} writes R serialisation (RDS)." + ), + class = c("masque_bad_recipe_path_refusal", "orchestra_refusal") + ) + } # include_simulator is currently a no-op (recipe carries no simulator state). saveRDS(rec, file = path, version = 3L) invisible(path) diff --git a/R/rng.R b/R/rng.R index 2ccb857..d29840c 100644 --- a/R/rng.R +++ b/R/rng.R @@ -17,13 +17,32 @@ with_rng_state <- function(seed, expr) { if (is.null(seed)) { return(withr::with_preserve_seed(expr)) } - if (!is.numeric(seed) || length(seed) != 1L || is.na(seed)) { - cli::cli_abort( - paste0( - "`seed` must be a single integer (or NULL); got ", - "{.cls {class(seed)[1]}} of length {length(seed)}." - ) - ) - } + .check_seed(seed) withr::with_seed(as.integer(seed), expr) } + +# A seed is a whole, finite number within the integer range; set.seed() +# would truncate 1.5 to 1 and reject 2^31 with a bare error. +.check_seed <- function(seed) { + ok <- is.numeric(seed) && length(seed) == 1L && !is.na(seed) && + is.finite(seed) && seed == round(seed) && + abs(seed) <= .Machine$integer.max + if (ok) { + return(invisible(seed)) + } + got <- if (is.numeric(seed) && length(seed) == 1L) { + format(seed, digits = 15L) + } else { + sprintf("<%s> of length %d", class(seed)[1L], length(seed)) + } + cli::cli_abort( + c( + paste0( + "`seed` must be a single integer (or NULL): a whole, finite number ", + "within the integer range." + ), + x = "Got {got}." + ), + class = c("masque_bad_seed_refusal", "orchestra_refusal") + ) +} diff --git a/tests/testthat/test-fix0160-refusals.R b/tests/testthat/test-fix0160-refusals.R new file mode 100644 index 0000000..913b2f4 --- /dev/null +++ b/tests/testthat/test-fix0160-refusals.R @@ -0,0 +1,70 @@ +# F19 and F22: refusals with a complete message and a proper class. + +.refusal_roles <- function() { + r <- propose_roles(iris) + set_role(r, "Sepal.Length", role = "outcome") +} + +test_that("recipe() and synthetic() refuse a non-masque with a full message", { + for (accessor in list(recipe, synthetic)) { + err <- expect_error(accessor(iris), class = "orchestra_refusal") + expect_identical( + class(err)[1:2], + c("masque_bad_masque_obj_refusal", "orchestra_refusal") + ) + expect_false(any(grepl("{", class(err), fixed = TRUE))) + expect_match(conditionMessage(err), "got ", fixed = TRUE) + } +}) + +test_that("save_recipe() refuses a path that is not .rds and writes nothing", { + rec <- recipe(mask(iris, .refusal_roles(), mode = "local", seed = 1L)) + for (ext in c(".csv", ".xlsx", ".rds.bak", "")) { + path <- file.path(tempdir(), paste0("masque-recipe-ext", ext)) + unlink(path) + expect_error( + save_recipe(rec, path), + class = "masque_bad_recipe_path_refusal" + ) + expect_false(file.exists(path)) + } + upper <- withr::local_tempfile(fileext = ".RDS") + expect_identical(save_recipe(rec, upper), upper) + expect_true(S7::S7_inherits(read_recipe(upper), masque:::masque_recipe)) +}) + +test_that("a seed that is not a whole finite integer is refused", { + r <- .refusal_roles() + for (bad in list(1.5, Inf, -Inf, NaN, 2^31, -0.25)) { + expect_error( + mask(iris, r, mode = "local", seed = bad), + class = "masque_bad_seed_refusal" + ) + } + expect_error( + mask(iris, r, mode = "local", seed = "1"), + class = "masque_bad_seed_refusal" + ) + # A whole double is the same seed as the integer. + expect_identical( + synthetic(mask(iris, r, mode = "local", seed = 7)), + synthetic(mask(iris, r, mode = "local", seed = 7L)) + ) +}) + +test_that("mask_set() and masque() route a bad seed to the same refusal", { + tables <- list( + a = data.frame(site = rep(c("s1", "s2"), 5L), y = seq_len(10L) / 3), + b = data.frame(site = c("s1", "s2"), z = c(1.5, 2.5)) + ) + expect_error( + mask_set(tables, mode = "local", seed = 1.5, quiet = TRUE), + class = "masque_bad_seed_refusal" + ) + expect_error( + masque(iris, roles = .refusal_roles(), mode = "local", seed = 1.5, + ask = FALSE, quiet = TRUE + ), + class = "masque_bad_seed_refusal" + ) +}) From c485951546bca12455a74e8de1af0ff8e64555ce Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:23:20 +1030 Subject: [PATCH 13/66] perf: group on the NA matrix for the audit's NA-pattern share The audit pasted every row of the NA matrix into a string, the slowest step on a large table. data.table grouping on the logical columns gives the same share at a fraction of the cost; the namespace now declares itself data.table-aware so its [ uses data.table semantics. --- R/leakage.R | 20 +++++-- tests/testthat/test-fix0160-na-pattern.R | 67 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 tests/testthat/test-fix0160-na-pattern.R diff --git a/R/leakage.R b/R/leakage.R index abf0fe7..31bd839 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -97,14 +97,19 @@ data.table::as.data.table(do.call(rbind, rows)) } +# Share of rows whose NA pattern no other row shares, grouped on the NA matrix. .global_na_pattern_uniqueness <- function(df) { - if (nrow(df) == 0L) { + n <- nrow(df) + if (n == 0L) { return(NA_real_) } - na_mat <- is.na(df) - patterns <- apply(na_mat, 1L, function(r) paste(as.integer(r), collapse = "")) - freqs <- table(patterns) - sum(freqs == 1L) / nrow(df) + if (ncol(df) == 0L) { + return(if (n == 1L) 1 else 0) + } + na_dt <- data.table::as.data.table(lapply(df, is.na)) + data.table::setnames(na_dt, sprintf("v%d", seq_along(na_dt))) + counts <- na_dt[, .N, by = c(names(na_dt))]$N + sum(counts == 1L) / n } .classify_leakage <- function( @@ -194,3 +199,8 @@ if (length(bits) == 0L) bits <- "ok" paste(bits, collapse = "; ") } + +# data.table's `[` uses data.table semantics only in a namespace that declares +# itself aware; masque imports data.table through `::` alone. +.datatable.aware <- TRUE +utils::globalVariables(c(".N", ".I")) diff --git a/tests/testthat/test-fix0160-na-pattern.R b/tests/testthat/test-fix0160-na-pattern.R new file mode 100644 index 0000000..dc2a151 --- /dev/null +++ b/tests/testthat/test-fix0160-na-pattern.R @@ -0,0 +1,67 @@ +# The NA-pattern uniqueness share, grouped rather than pasted row by row. + +# The row-paste computation the audit used before 0.16.0. +.na_uniqueness_paste <- function(df) { + if (nrow(df) == 0L) { + return(NA_real_) + } + na_mat <- is.na(df) + patterns <- apply(na_mat, 1L, function(r) paste(as.integer(r), collapse = "")) + freqs <- table(patterns) + sum(freqs == 1L) / nrow(df) +} + +test_that("the grouped NA-pattern share equals the row-paste share", { + set.seed(16L) + n <- 400L + mixed <- data.frame( + a = ifelse(runif(n) < 0.2, NA, rnorm(n)), + b = factor(ifelse(runif(n) < 0.4, NA, sample(letters[1:3], n, TRUE))), + c = as.Date(ifelse(runif(n) < 0.05, NA, 19000 + seq_len(n))), + d = ifelse(runif(n) < 0.01, NA_character_, "x"), + e = ifelse(runif(n) < 0.3, NA, TRUE) + ) + fixtures <- list( + complete = data.frame(a = 1:10, b = letters[1:10]), + all_missing = data.frame(a = rep(NA_real_, 6L), b = rep(NA_character_, 6L)), + one_row = data.frame(a = NA, b = 1), + one_row_complete = data.frame(a = 1, b = 2), + alternating = data.frame(a = c(1, NA, 3, NA, 5), b = c(NA, 2, NA, 4, NA)), + mixed = mixed, + duplicate_names = stats::setNames( + data.frame(c(1, NA, 3), c(NA, NA, 3), c(1, 2, NA)), c("x", "x", "x y") + ), + no_columns = data.frame(row.names = 1:3), + no_rows = data.frame(a = numeric(), b = character()) + ) + for (nm in names(fixtures)) { + expect_identical( + masque:::.global_na_pattern_uniqueness(fixtures[[nm]]), + .na_uniqueness_paste(fixtures[[nm]]), + info = nm + ) + } + expect_identical(masque:::.global_na_pattern_uniqueness(fixtures$complete), 0) + expect_identical( + masque:::.global_na_pattern_uniqueness(fixtures$all_missing), 0 + ) + expect_true(is.na(masque:::.global_na_pattern_uniqueness(fixtures$no_rows))) +}) + +test_that("the grouped NA-pattern share is much faster than the row paste", { + skip_on_cran() + set.seed(17L) + n <- 200000L + big <- as.data.frame(lapply(seq_len(12L), function(j) { + x <- rnorm(n) + x[runif(n) < 0.05 * j / 4] <- NA + x + })) + t_paste <- system.time(ref <- .na_uniqueness_paste(big))[["elapsed"]] + t_grouped <- system.time( + out <- masque:::.global_na_pattern_uniqueness(big) + )[["elapsed"]] + expect_identical(out, ref) + # Measured near 100x; 5x leaves room for a loaded machine. + expect_lt(t_grouped * 5, t_paste) +}) From 20816eb8b56b32a0334a87a5a343f4d94ed96d50 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:24:35 +1030 Subject: [PATCH 14/66] fix(detect): take block columns from the environments once scope resolves (F6) After environment scope resolves, environment columns leave block_cols, whole_plot_col and spatial_cols, and block_cols becomes the within- environment block basis when every environment agrees on it, or empty when they disagree. A pooled block hint that lost its place is dropped. evidence$block_revision records the change and print() explains it. env = FALSE is unchanged. --- R/detect_environment.R | 77 +++++++++++++ R/detect_print.R | 12 +- tests/testthat/test-fix0160-blocks.R | 162 +++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-fix0160-blocks.R diff --git a/R/detect_environment.R b/R/detect_environment.R index fe6c6a5..d7af13e 100644 --- a/R/detect_environment.R +++ b/R/detect_environment.R @@ -843,5 +843,82 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 if (length(treatment_cols) > 0L) { out@treatment_col <- treatment_cols } + .revise_blocks_after_scope(out, env_cols) +} + +# Once scope is resolved, an environment column is never a block: blocks are +# the within-environment basis when every environment agrees on it, else none. +.revise_blocks_after_scope <- function(out, env_cols) { + pooled <- out@block_cols + bases <- unique(out@per_env$block_cols) + agree <- length(bases) == 1L + consensus <- if (agree && nzchar(bases)) { + strsplit(bases, " : ", fixed = TRUE)[[1L]] + } else { + character() + } + pooled_text <- if (length(pooled) == 0L) { + "none" + } else { + paste(pooled, collapse = " : ") + } + + status <- if (setequal(pooled, consensus)) { + "unchanged" + } else if (length(consensus) > 0L) { + "consensus" + } else { + "cleared" + } + message <- switch(status, + unchanged = "Pooled blocks match the within-environment basis.", + consensus = sprintf( + paste0( + "Blocks are the within-environment consensus %s; ", + "the pooled verdict had %s." + ), + paste(consensus, collapse = " : "), pooled_text + ), + cleared = if (agree) { + sprintf( + paste0( + "Blocks left empty: no environment records a block basis; ", + "the pooled verdict had %s." + ), + pooled_text + ) + } else { + sprintf( + paste0( + "Blocks left empty: the within-environment block bases ", + "disagree (%s); the pooled verdict had %s." + ), + paste(ifelse(nzchar(bases), bases, "none"), collapse = "; "), + pooled_text + ) + } + ) + + out@block_cols <- consensus + out@whole_plot_col <- setdiff(out@whole_plot_col, env_cols) + if (any(out@spatial_cols %in% env_cols)) { + out@spatial_cols <- character() + } + # A pooled block hint that lost its place in block_cols loses its role hint. + stale <- setdiff(pooled, c(consensus, env_cols)) + rec <- out@recommended_roles + if (length(stale) > 0L && nrow(rec) > 0L) { + out@recommended_roles <- rec[!(rec$role == "design" & rec$col %in% stale), ] + } + + evidence <- out@evidence + evidence$block_revision <- list( + status = status, + pooled = pooled, + block_cols = consensus, + per_env_bases = bases, + message = message + ) + out@evidence <- evidence out } diff --git a/R/detect_print.R b/R/detect_print.R index 26675ec..c1c01d9 100644 --- a/R/detect_print.R +++ b/R/detect_print.R @@ -25,10 +25,18 @@ local(S7::method(print, design_summary) <- function(x, ...) { } cli::cli_rule(left = pooled_label) + # After scope, the block columns come from the environments; say so. + revision <- x@evidence$block_revision + revision_bullet <- character(0L) + if (!is.null(revision) && !identical(revision$status, "unchanged")) { + revision_bullet <- c("i" = gsub("([{}])", "\\1\\1", revision$message)) + } + if (x@class_label == "none") { cli::cli_bullets(c( "i" = "No experimental design detected above threshold.", - "*" = "Top rule scores all below 0.5. Treat as observational." + "*" = "Top rule scores all below 0.5. Treat as observational.", + revision_bullet )) } else { bullets <- character(0L) @@ -64,7 +72,7 @@ local(S7::method(print, design_summary) <- function(x, ...) { ) ) } - cli::cli_bullets(bullets) + cli::cli_bullets(c(bullets, revision_bullet)) } # Alternates. diff --git a/tests/testthat/test-fix0160-blocks.R b/tests/testthat/test-fix0160-blocks.R new file mode 100644 index 0000000..0b2d274 --- /dev/null +++ b/tests/testthat/test-fix0160-blocks.R @@ -0,0 +1,162 @@ +# F6: after environment scope resolves, block columns come from the +# within-environment block bases, never from an environment column. +# Every expected answer below follows from how the fixture is built. + +# 2 years x 4 sites x 24 genotypes x 3 reps; in each site-year-rep, six blocks +# of four, labels B1-B6 restarted in every rep. True blocks: rep : block. +.f6_two_year_met <- function() { + set.seed(6) + years <- c(2023L, 2024L) + sites <- sprintf("S%d", 1:4) + gens <- sprintf("G%02d", 1:24) + g <- expand.grid( + genotype = gens, rep = 1:3, site = sites, year = years, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$block <- NA_character_ + for (y in years) for (s in sites) for (r in 1:3) { + idx <- which(g$year == y & g$site == s & g$rep == r) + g$block[idx] <- sprintf("B%d", sample(rep(1:6, each = 4))) + } + g$yield <- round(3 + rnorm(nrow(g), 0, 0.35), 2) + g[, c("year", "site", "rep", "block", "genotype", "yield")] +} + +# 2 sites, complete blocks B1-B4 restarted in each site, plot P01-P24 the +# position within block. A cyclic allocation with one swap makes plot +# positions look almost like regular incomplete blocks. True blocks: block. +.f6_plot_artefact <- function() { + set.seed(13) + g <- expand.grid( + plot = 1:24, block = 1:4, site = c("North", "South"), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + idx <- (g$plot - 1L + 6L * (g$block - 1L)) %% 24L + 1L + b2 <- which(g$block == 2L) + swap <- c(b2[g$plot[b2] == 1L], b2[g$plot[b2] == 19L]) + idx[swap] <- idx[rev(swap)] + labels <- list( + North = sample(sprintf("G%02d", 1:24)), + South = sample(sprintf("G%02d", 1:24)) + ) + g$genotype <- vapply( + seq_len(nrow(g)), function(i) labels[[g$site[i]]][idx[i]], character(1L) + ) + g$block <- sprintf("B%d", g$block) + g$plot <- sprintf("P%02d", g$plot) + g$yield <- round(4 + rnorm(nrow(g), 0, 0.4), 2) + g[, c("site", "block", "plot", "genotype", "yield")] +} + +# 2 sites x 4 reps x 24 genotypes, six blocks of four in each rep with labels +# restarted inside each replicate. True blocks: rep : block. +.f6_restarted_lattice <- function() { + set.seed(11) + sites <- c("North", "South") + gens <- sprintf("G%02d", 1:24) + g <- expand.grid( + plot = 1:24, rep = 1:4, site = sites, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$block <- sprintf("B%d", (g$plot - 1L) %/% 4L + 1L) + g$genotype <- NA_character_ + for (s in sites) for (r in 1:4) { + idx <- which(g$site == s & g$rep == r) + g$genotype[idx] <- sample(gens) + } + g$yield <- round(4 + rnorm(nrow(g), 0, 0.4), 2) + g[, c("site", "rep", "block", "plot", "genotype", "yield")] +} + +# Two sites with different blocking: S1 an RCBD on rep, S2 a CRD with no rep +# recorded. There is no common block basis. +.f6_two_site_disagree <- function() { + set.seed(12) + gens <- sprintf("G%d", 1:8) + s1 <- expand.grid( + genotype = gens, rep = 1:3, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + s1$site <- "S1" + s2 <- data.frame( + genotype = sample(rep(gens, 3)), rep = NA_integer_, site = "S2", + stringsAsFactors = FALSE + ) + d <- rbind(s1, s2) + d$yield <- round(5 + rnorm(nrow(d), 0, 0.5), 2) + d[, c("site", "rep", "genotype", "yield")] +} + +.f6_print <- function(x) { + paste(capture.output(print(x), type = "message"), collapse = "\n") +} + +test_that("the F6 fixtures have the structure they are built with", { + d <- .f6_two_year_met() + per_rep <- data.table::as.data.table(d)[ + , .(n_blocks = data.table::uniqueN(block), n = .N), + by = .(year, site, rep) + ] + expect_true(all(per_rep$n_blocks == 6L) && all(per_rep$n == 24L)) + expect_true(all(table(paste(d$year, d$site, d$rep, d$block)) == 4L)) + + a <- .f6_plot_artefact() + expect_true(all(table(a$site, a$block, a$genotype) == 1L)) + + r <- .f6_restarted_lattice() + expect_true(all(table(r$site, r$rep, r$genotype) == 1L)) + + s <- .f6_two_site_disagree() + expect_true(all(is.na(s$rep[s$site == "S2"]))) + expect_true(all(table(s$site, s$genotype) == 3L)) +}) + +test_that("a two-year MET takes rep : block, not year, as its blocks", { + d <- .f6_two_year_met() + for (env in list(NULL, c("site", "year"))) { + ds <- detect_design(d, env = env) + expect_true(isTRUE(ds@is_met)) + expect_false("year" %in% ds@block_cols) + expect_false("site" %in% ds@block_cols) + expect_setequal(ds@block_cols, c("rep", "block")) + expect_identical(ds@evidence$block_revision$status, "consensus") + } + expect_match(.f6_print(detect_design(d)), "within-environment consensus") +}) + +test_that("a stray plot does not survive the block revision", { + d <- .f6_plot_artefact() + ds <- detect_design(d) + expect_true(isTRUE(ds@is_met)) + expect_identical(ds@env_cols, "site") + expect_identical(ds@block_cols, "block") + expect_false(any(c("plot", "site") %in% ds@block_cols)) + stale <- ds@recommended_roles$col[ + ds@recommended_roles$source == "design_rule" & + ds@recommended_roles$role == "design" + ] + expect_false("plot" %in% stale) +}) + +test_that("block labels restarted inside each replicate give rep : block", { + ds <- detect_design(.f6_restarted_lattice()) + expect_true(isTRUE(ds@is_met)) + expect_setequal(ds@block_cols, c("rep", "block")) + expect_false(any(c("plot", "site") %in% ds@block_cols)) +}) + +test_that("disagreeing within-environment blocks leave block_cols empty", { + ds <- detect_design(.f6_two_site_disagree()) + expect_true(isTRUE(ds@is_met)) + expect_identical(ds@env_cols, "site") + expect_identical(ds@block_cols, character()) + expect_identical(ds@evidence$block_revision$status, "cleared") + expect_match(ds@evidence$block_revision$message, "disagree") + expect_match(.f6_print(ds), "disagree") +}) + +test_that("env = FALSE keeps the pooled block columns", { + ds <- detect_design(.f6_two_year_met(), env = FALSE) + expect_identical(ds@block_cols, "year") + expect_null(ds@evidence$block_revision) +}) From e4b89c4094e1303c96b303a3585e823e2e022bb0 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:25:49 +1030 Subject: [PATCH 15/66] fix: conform decisions go on the recipe and apply_recipe replays them mask() records the effective merge map, the applied type changes, the modes and max_levels from a conformance object (new recipe property conform, default NULL) and uses the object's cleaning record. apply_recipe() replays merges then types after the cleaning; a recipe saved before the property existed applies unchanged. Merge chains are resolved to the label left in the data; a source with two candidate targets is not merged and raises masque_conform_ambiguous_merge. Categories are found independently of types, so types = 'off' still merges. mask_set() and masque() accept a conformance object per table. --- R/conform_table.R | 281 ++++++++++++++----- R/mask.R | 6 + R/mask_set.R | 38 ++- R/masque.R | 16 +- R/recipe.R | 7 + R/recipe_apply.R | 23 +- tests/testthat/test-fix0160-conform-recipe.R | 199 +++++++++++++ 7 files changed, 490 insertions(+), 80 deletions(-) create mode 100644 tests/testthat/test-fix0160-conform-recipe.R diff --git a/R/conform_table.R b/R/conform_table.R index a7a5003..4c03305 100644 --- a/R/conform_table.R +++ b/R/conform_table.R @@ -23,6 +23,21 @@ #' numeric columns keep their storage. Anything not changed for these reasons #' is reported as a `not applied` row with the reason. #' +#' Merges follow chains: if `"Nsw"` merges onto `"nsw"` and `"nsw"` onto +#' `"NSW"`, both end on `"NSW"`. A label whose merges end on two different +#' labels (`"colur"` beside `"color"` and `"colour"` that occur equally +#' often) is not merged; it is reported, and a `masque_conform_ambiguous_merge` +#' warning is raised under `merge_labels = "auto"`. Codes that differ only +#' inside a run of digits (`"G01"` and `"G02"`) are never treated as +#' near-duplicates. Which columns hold categories is decided the same way +#' whatever `types` is, so `types = "off"` still merges labels and leaves +#' storage as it is. +#' +#' Pass the result to [mask()] (or as a table of [mask_set()]): the recipe +#' records the merges and type changes, and [apply_recipe()] replays them on +#' the original so a pipeline built on the conformed clone sees the same +#' labels and storage. +#' #' @param df A data frame. #' @param merge_labels One of `"report"` (default), `"auto"` or `"off"`. #' Governs near-duplicate category labels, and treats the two kinds @@ -48,11 +63,16 @@ #' \item `cleaning` - the `masque_cleaning` record from [clean_table()]; #' \item `merges` - one row per near-duplicate pair considered #' (`col`, `from`, `to`, `n_from`, `n_to`, `applied`, `reason`); +#' \item `merge_map` - the merges as applied to the data (`col`, `from`, +#' `to`), one row per merged-away label, each `to` a label left in +#' the column. This is what [mask()] records and [apply_recipe()] +#' replays; #' \item `types` - one row per column whose storage was considered #' (`col`, `from`, `to`, `applied`, `reason`); #' \item `assumptions` - every decision as a plain sentence #' (`col`, `assumption`, `applied`), which is the thing to read; -#' \item `modes` - the modes applied. +#' \item `modes` - the modes applied; +#' \item `max_levels` - the cardinality threshold used. #' } #' #' @examples @@ -94,13 +114,19 @@ conform_table <- function(df, out_df <- cl$data # Storage first: numbers or dates held as text are not categories, and - # merging "4.2" into "4.4" would destroy data. + # merging "4.2" into "4.4" would destroy data. Which columns are categories + # is read from a report-only storage pass, so it holds under types = "off". tps <- .conform_types(out_df, types, max_levels) + cat_cols <- .categorical_cols( + out_df, .conform_types(out_df, "report", max_levels), max_levels + ) - cat_cols <- .categorical_cols(out_df, tps, max_levels) merges <- .conform_merges(out_df, cl$near_duplicates, merge_labels, cat_cols) - if (identical(merge_labels, "auto") && nrow(merges)) { - out_df <- .apply_merges(out_df, merges[merges$applied, , drop = FALSE]) + resolved <- .resolve_merges(merges) + merges <- resolved$merges + if (identical(merge_labels, "auto")) { + .warn_ambiguous_merges(resolved$ambiguous) + out_df <- .apply_merge_map(out_df, resolved$map) } if (identical(types, "auto") && nrow(tps)) { out_df <- .apply_types(out_df, tps[tps$applied, , drop = FALSE]) @@ -110,9 +136,11 @@ conform_table <- function(df, data = .as_owned_dt(out_df), cleaning = cl, merges = data.table::as.data.table(merges), + merge_map = data.table::as.data.table(resolved$map), types = data.table::as.data.table(tps), assumptions = .conform_assumptions(merges, tps), - modes = c(clean = clean, merge_labels = merge_labels, types = types) + modes = c(clean = clean, merge_labels = merge_labels, types = types), + max_levels = max_levels ) class(out) <- "masque_conformance" @@ -151,79 +179,173 @@ print.masque_conformance <- function(x, ...) { ) } +.empty_merge_map <- function() { + data.frame( + col = character(), from = character(), to = character(), + stringsAsFactors = FALSE + ) +} + +utils::globalVariables(c( + ".N", "a", "a_wins", "b", "candidate", "col", "first_a", "first_b", + "from", + "i.first", "i.from", "i.n", "i.to", "kind", "label", "n_a", "n_b", "n_to", + "pos", "reason", "to", "x.to" +)) + # For each near-duplicate pair in a categorical column the commoner spelling # wins. A case-only tie goes to the first seen; an edit-distance tie is refused. MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing .conform_merges <- function(df, near_dups, mode, cat_cols) { + # data.table syntax in this frame only (data.table >= 1.15.0). + .datatable.aware <- TRUE if (identical(mode, "off") || is.null(near_dups) || !nrow(near_dups)) { return(.empty_merges()) } - near_dups <- near_dups[near_dups$col %in% cat_cols, , drop = FALSE] - if (!nrow(near_dups)) return(.empty_merges()) + nd <- data.table::as.data.table(near_dups)[col %in% cat_cols] + # A one-character edit between short labels is not evidence of a typo: + # "a" and "b" are one edit apart and are plainly different categories. + nd <- nd[kind == "case" | pmin(nchar(a), nchar(b)) >= MIN_EDIT_NCHAR] + if (!nrow(nd)) { + return(.empty_merges()) + } - rows <- list() - for (i in seq_len(nrow(near_dups))) { - col <- near_dups$col[i] - a <- near_dups$a[i] - b <- near_dups$b[i] - kind <- near_dups$kind[i] - is_case <- identical(kind, "case") - - # A one-character edit between short labels is not evidence of a typo: - # "a" and "b" are one edit apart and are plainly different categories. - if (!is_case && min(nchar(a), nchar(b)) < MIN_EDIT_NCHAR) next - - v <- as.character(df[[col]]) - na_ <- sum(v == a, na.rm = TRUE) - nb_ <- sum(v == b, na.rm = TRUE) - - if (na_ == nb_ && !is_case) { - # differing by an edit, with no majority: this may be two real categories - rows[[length(rows) + 1L]] <- data.frame( - col = col, from = a, to = b, n_from = na_, n_to = nb_, applied = FALSE, - reason = paste("the spellings differ by one edit and occur equally", - "often; masque will not choose between them"), - stringsAsFactors = FALSE) - next - } + # Count and first position of every label in the columns concerned. + labels <- data.table::rbindlist(lapply(unique(nd$col), function(cc) { + x <- as.character(df[[cc]]) + data.table::data.table(col = cc, label = x, pos = seq_along(x)) + })) + counts <- labels[!is.na(label), list(n = .N, first = min(pos)), + by = c("col", "label") + ] + nd[counts, `:=`(n_a = i.n, first_a = i.first), on = c("col", a = "label")] + nd[counts, `:=`(n_b = i.n, first_b = i.first), on = c("col", b = "label")] - if (na_ > nb_) { keep <- a; drop <- b } - else if (nb_ > na_) { keep <- b; drop <- a } - else { - # a case-only tie: the two ARE one category, only the spelling is open. - # The first spelling to appear wins, which is deterministic and stated. - first <- v[!is.na(v) & v %in% c(a, b)][1] - keep <- first; drop <- if (identical(first, a)) b else a - } + # The commoner spelling wins; a case-only tie goes to the first seen. + nd[, candidate := kind == "case" | n_a != n_b] + nd[, a_wins := n_a > n_b | (n_a == n_b & kind == "case" & first_a < first_b)] + nd[, `:=`( + from = data.table::fifelse(a_wins, b, a), + to = data.table::fifelse(a_wins, a, b) + )] + nd[, reason := data.table::fifelse( + kind == "case", + sprintf("differs from %s by capitalisation only", to), + sprintf("differs from %s by a single character", to) + )] + nd[candidate == FALSE, reason := paste( + "the spellings differ by one edit and occur equally", + "often; masque will not choose between them" + )] + out <- nd[, list( + col, from, to, + n_from = pmin(n_a, n_b), n_to = pmax(n_a, n_b), + applied = candidate & identical(mode, "auto"), + reason, candidate + )] + as.data.frame(out) +} - rows[[length(rows) + 1L]] <- data.frame( - col = col, from = drop, to = keep, - n_from = min(na_, nb_), n_to = max(na_, nb_), - applied = identical(mode, "auto"), - reason = if (is_case) { - sprintf("differs from %s by capitalisation only", keep) - } else { - sprintf("differs from %s by a single character", keep) - }, - stringsAsFactors = FALSE) +# The effective merge map: each source follows its merges until it reaches a +# label nothing is merged away from. A source that reaches two such labels has +# no single target, so none of its merges is applied. +.resolve_merges <- function(merges) { + .datatable.aware <- TRUE + none <- list( + merges = merges[, setdiff(names(merges), "candidate"), drop = FALSE], + map = .empty_merge_map(), + ambiguous = data.frame( + col = character(), from = character(), to = character(), + stringsAsFactors = FALSE + ) + ) + if (!nrow(merges) || !any(merges$candidate)) { + return(none) } - if (!length(rows)) return(.empty_merges()) - do.call(rbind, rows) + + edges <- unique(data.table::as.data.table(merges)[ + candidate == TRUE, list(col, from, to) + ]) + paths <- data.table::copy(edges) + # Each pass moves every path one merge further; the merge graph has no + # cycles (a merge never goes to a rarer label), so nrow(edges) passes bound it. + for (i1 in seq_len(nrow(edges))) { + step <- edges[paths, list(col, from = i.from, to = x.to), + on = c("col", from = "to"), nomatch = NULL + ] + if (!nrow(step)) break + paths <- unique(data.table::rbindlist(list( + paths[!edges, on = c("col", to = "from")], step + ))) + } + sinks <- paths[, list( + n_to = data.table::uniqueN(to), + to = paste(sort(unique(to)), collapse = '", "') + ), by = c("col", "from")] + + ambiguous <- sinks[n_to > 1L, list(col, from, to)] + map <- sinks[n_to == 1L, list(col, from, to)] + + report <- data.table::as.data.table(merges) + if (nrow(ambiguous)) { + report[ambiguous, `:=`( + applied = FALSE, + reason = sprintf( + paste0( + "could be merged into more than one label (\"%s\"); masque will ", + "not choose between them" + ), + i.to + ) + ), on = c("col", "from")] + } + report[, candidate := NULL] + list( + merges = as.data.frame(report), + map = as.data.frame(map), + ambiguous = as.data.frame(ambiguous) + ) +} + +.warn_ambiguous_merges <- function(ambiguous) { + if (!nrow(ambiguous)) { + return(invisible()) + } + lines <- sprintf( + "%s: \"%s\" -> \"%s\"", ambiguous$col, ambiguous$from, ambiguous$to + ) + # Labels are data: cli inserts them by reference, never parses them. + bullets <- stats::setNames( + sprintf("{lines[%d]}", seq_along(lines)), rep("*", length(lines)) + ) + cli::cli_warn(c( + paste0( + "{cli::qty(length(lines))}Not merged: {length(lines)} label{?s} with ", + "more than one candidate target." + ), + bullets, + i = "Merge these by hand if one target is right." + ), class = "masque_conform_ambiguous_merge") } -.apply_merges <- function(df, m) { - if (!nrow(m)) return(df) - for (i in seq_len(nrow(m))) { - col <- m$col[i] - x <- df[[col]] +# Apply an effective merge map (`col`, `from`, `to`) to a table. +.apply_merge_map <- function(df, map) { + if (is.null(map) || !nrow(map)) { + return(df) + } + for (cc in intersect(unique(map$col), names(df))) { + m <- map[map$col == cc, , drop = FALSE] + x <- df[[cc]] if (is.factor(x)) { lv <- levels(x) - lv[lv == m$from[i]] <- m$to[i] - df[[col]] <- factor(lv[as.integer(x)], levels = unique(lv)) + hit <- match(lv, m$from) + lv[!is.na(hit)] <- m$to[hit[!is.na(hit)]] + df[[cc]] <- factor(lv[as.integer(x)], levels = unique(lv)) } else { - x[!is.na(x) & x == m$from[i]] <- m$to[i] - df[[col]] <- x + hit <- match(x, m$from) + x[!is.na(hit)] <- m$to[hit[!is.na(hit)]] + df[[cc]] <- x } } df @@ -309,7 +431,7 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing } .apply_types <- function(df, t) { - if (!nrow(t)) return(df) + if (is.null(t) || !nrow(t)) return(df) for (i in seq_len(nrow(t))) { col <- t$col[i] df[[col]] <- switch( @@ -323,6 +445,39 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing df } +# --- the recipe record ----------------------------------------------------- + +# What mask() stores from a conformance object: the effective merge map, the +# applied type changes, the modes and max_levels, as plain data frames. +.conform_record <- function(cf) { + types <- as.data.frame(cf$types) + types <- types[types$applied, c("col", "from", "to"), drop = FALSE] + rownames(types) <- NULL + map <- as.data.frame(cf$merge_map %||% .empty_merge_map()) + rownames(map) <- NULL + list( + merges = map[, c("col", "from", "to"), drop = FALSE], + types = types, + modes = as.list(cf$modes), + max_levels = as.integer(cf$max_levels %||% NA_integer_) + ) +} + +# Replay a recipe's conform record on a cleaned original: merges, then types, +# in the order conform_table() applied them. +.apply_conform_forward <- function(df, conform) { + if (is.null(conform)) { + return(df) + } + merges <- conform$merges + types <- conform$types + if (!is.null(types)) { + types <- types[types$col %in% names(df), , drop = FALSE] + } + df <- .apply_merge_map(df, merges) + .apply_types(df, types) +} + # --- the assumption report -------------------------------------------------- # The thing a reader should actually look at: every decision as a sentence, diff --git a/R/mask.R b/R/mask.R index 2505d88..8e1da29 100644 --- a/R/mask.R +++ b/R/mask.R @@ -214,7 +214,12 @@ mask <- function(df, # Undo any RNG change made inside mask() on exit, whichever path made it. withr::local_preserve_seed() + # A conformance object carries its own cleaning and the judged decisions; + # both go on the recipe so apply_recipe() can replay them. + conform_rec <- NULL if (inherits(df, "masque_conformance")) { + conform_rec <- .conform_record(df) + if (is.null(.cleaning)) .cleaning <- .cleaning_record(df$cleaning) df <- df$data } if (!is.data.frame(df)) { @@ -525,6 +530,7 @@ mask <- function(df, storage_classes = storage_classes, factor_meta = factor_meta, cleaning = cleaning_rec, + conform = conform_rec, coords = coord_reports, events = event_reports, allow_unmasked_coords = isTRUE(allow_unmasked_coords), diff --git a/R/mask_set.R b/R/mask_set.R index 9af0206..d472a6a 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -20,7 +20,9 @@ #' unmasked instead. #' #' @param input A folder path, an `.xlsx` / `.xls` path, or a named list -#' of data frames - anything [read_set()] accepts. +#' of data frames - anything [read_set()] accepts. A list element may also +#' be a [conform_table()] result: its cleaning record and conform +#' decisions are used for that table and recorded on its recipe. #' @param roles Optional named list of roles tables, one per input table #' (names must match). When `NULL` (default), [propose_roles()] runs on #' each table for the chosen `mode`. Supply edited tables for full @@ -117,15 +119,26 @@ mask_set <- function(input, mode <- match.arg(mode, c("local", "collaborate")) clean <- match.arg(clean) - tables <- read_set(input) + # A table given as a conform_table() result was cleaned and conformed + # already; its own record is used and its decisions go on its recipe. + split <- .split_conformance(input) + conformed <- split$conformed + tables <- read_set(split$input) table_names <- names(tables) # Hygiene per table (names legalised, whitespace trimmed) up front, so # link detection and roles see the cleaned schema. Each table's record goes # to its recipe so the set round-trips like a single table. - cleaned <- lapply(tables, clean_table, clean = clean, quiet = TRUE) - tables <- lapply(cleaned, `[[`, "data") - cleaning <- lapply(cleaned, .cleaning_record) + cleaning <- stats::setNames(vector("list", length(tables)), table_names) + for (nm in table_names) { + if (nm %in% names(conformed)) { + cleaning[[nm]] <- .cleaning_record(conformed[[nm]]$cleaning) + } else { + cl <- clean_table(tables[[nm]], clean = clean, quiet = TRUE) + tables[[nm]] <- cl$data + cleaning[[nm]] <- .cleaning_record(cl) + } + } # Roles: proposed per table unless supplied. Supplied roles may name the # original headers or the cleaned ones. @@ -164,7 +177,7 @@ mask_set <- function(input, for (nm in table_names) { aln <- .table_alias_names(alias_names, names(tables[[nm]]), linked_cols) m <- mask( - tables[[nm]], + conformed[[nm]] %||% tables[[nm]], roles = roles[[nm]], mode = mode, seed = seed, @@ -201,6 +214,19 @@ mask_set <- function(input, ) } +# Pull conform_table() results out of a list input: `input` with each one +# replaced by its data, and the conformance objects by table name. +.split_conformance <- function(input) { + if (!is.list(input) || is.data.frame(input) || + inherits(input, "masque_conformance")) { + return(list(input = input, conformed = list())) + } + is_cf <- vapply(input, inherits, logical(1L), what = "masque_conformance") + conformed <- input[is_cf] + input[is_cf] <- lapply(conformed, `[[`, "data") + list(input = input, conformed = conformed) +} + # The shape half of the roles check, split out so it can run before the # mode is inferred from the roles tables' provenance attribute. .check_roles_shape <- function(roles) { diff --git a/R/masque.R b/R/masque.R index e9418cb..9f90fd0 100644 --- a/R/masque.R +++ b/R/masque.R @@ -127,7 +127,7 @@ masque <- function(input, # A set is a folder, an Excel workbook, or a named list of data frames. # A single data frame or a single tabular file is one table. .masque_is_set <- function(input) { - if (is.data.frame(input)) { + if (is.data.frame(input) || inherits(input, "masque_conformance")) { return(FALSE) } if (is.list(input)) { @@ -150,7 +150,11 @@ masque <- function(input, .masque_guided_one <- function(input, roles, mode, seed, clean, alias_names, conditional, ladder, ask, quiet) { - df <- if (is.data.frame(input)) input else .read_one_file(input) + df <- if (is.data.frame(input) || inherits(input, "masque_conformance")) { + input + } else { + .read_one_file(input) + } if (!is.null(roles)) { return(mask( df, roles, mode = mode, seed = seed, clean = clean, @@ -158,7 +162,9 @@ masque <- function(input, )) } - proposed <- propose_roles(df, mode = mode) + proposed <- propose_roles( + if (inherits(df, "masque_conformance")) df$data else df, mode = mode + ) proposed <- .masque_review(proposed, ask, quiet, label = NULL) mask( df, proposed, mode = mode, seed = seed, clean = clean, @@ -178,7 +184,8 @@ masque <- function(input, # Propose on the names mask_set() will mask under. It cleans again and # raises the rename warning once, so the warning is held back here. - tables <- read_set(input) + split <- .split_conformance(input) + tables <- read_set(split$input) proposed <- lapply(names(tables), function(nm) { cl <- withCallingHandlers( clean_table(tables[[nm]], clean = clean, quiet = TRUE), @@ -188,6 +195,7 @@ masque <- function(input, .masque_review(r, ask, quiet, label = nm) }) names(proposed) <- names(tables) + tables[names(split$conformed)] <- split$conformed mask_set( tables, roles = proposed, mode = mode, seed = seed, clean = clean, diff --git a/R/recipe.R b/R/recipe.R index fb5a371..1c08ba5 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -65,6 +65,13 @@ masque_recipe <- S7::new_class( class = S7::new_union(S7::class_list, NULL), default = NULL ), + # The conform_table() decisions mask() was given: the effective merge + # map, the applied type changes, the modes and max_levels. NULL when the + # input was not a conformance object, and absent before 0.16.0. + conform = S7::new_property( + class = S7::new_union(S7::class_list, NULL), + default = NULL + ), # One entry per declared coordinate pair: jitter parameters, site grouping # and site count. Empty before 0.10.0. coords = S7::new_property(class = S7::class_list, default = list()), diff --git a/R/recipe_apply.R b/R/recipe_apply.R index d71527d..c660397 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -7,17 +7,22 @@ #' #' Operations applied (in order): #' -#' 1. **Verify integrity** by comparing the NA mask of `original` to the +#' 1. **Re-apply cleaning and conform decisions**: the legalised names and +#' trimmed labels from [clean_table()], then the label merges and type +#' changes from [conform_table()] when the clone was built on a +#' conformance object. +#' 2. **Verify integrity** by comparing the NA mask of `original` to the #' SHA-256 fingerprint stored on the recipe (controlled by #' `check_integrity`). -#' 2. **Drop** every column whose resolved `action` was `"drop"` on the -#' roles table `mask()` masked with, in either mode. -#' 3. **Subset and reorder** to the columns the recipe knows about. -#' 4. **Re-label factors / characters** for any column with a level map +#' 3. **Drop** every column whose resolved `action` was `"drop"` on the +#' roles table `mask()` masked with, in either mode. A linked join key +#' of a set is aliased in the clone and is kept. +#' 4. **Subset and reorder** to the columns the recipe knows about. +#' 5. **Re-label factors / characters** for any column with a level map #' held by the recipe (i.e., treatment and categorical covariates in #' `collaborate` mode; or treatment in `local` mode with opt-in #' permutation). Unknown non-NA values fail closed. -#' 5. **Rename columns** per `recipe@column_name_map` when `mask()` was +#' 6. **Rename columns** per `recipe@column_name_map` when `mask()` was #' called with `alias_names` (otherwise `NULL`, and names are #' unchanged). #' @@ -64,8 +69,12 @@ apply_recipe <- function(original, rec, check_integrity = TRUE) { cli::cli_abort("`check_integrity` must be a single logical.") } - # Re-apply the recipe's cleaning (legal names, trimmed whitespace). + # Re-apply the recipe's cleaning (legal names, trimmed whitespace), then the + # conform_table() merges and type changes the clone was built on. original <- .apply_cleaning_forward(original, rec@cleaning) + # A recipe saved before 0.16.0 has no `conform` property at all. + conform <- if (S7::prop_exists(rec, "conform")) rec@conform else NULL + original <- .apply_conform_forward(original, conform) dropped <- .recipe_dropped_cols(rec) retained <- setdiff(rec@roles$col, dropped) diff --git a/tests/testthat/test-fix0160-conform-recipe.R b/tests/testthat/test-fix0160-conform-recipe.R new file mode 100644 index 0000000..87e5dbc --- /dev/null +++ b/tests/testthat/test-fix0160-conform-recipe.R @@ -0,0 +1,199 @@ +# conform_table() decisions travel on the recipe: apply_recipe() replays the +# effective merge map and the type changes on the original. + +state_raw <- function() { + set.seed(2) + state <- c( + rep("NSW", 20), rep("nsw", 5), rep("Nsw", 3), rep("N.S.W.", 2), + rep("VIC", 15), rep("QLD", 15) + ) + data.frame( + rep = rep(1:3, each = 20), + genotype = rep(sprintf("G%d", 1:4), 15), + state = sample(state), + yield = round(stats::rnorm(60, 3, 0.5), 2), + stringsAsFactors = FALSE + ) +} + +state_roles <- function(df, mode, state_action = NULL) { + r <- propose_roles(df, mode = mode, detect = FALSE) + r <- set_role(r, "rep", role = "design") + r <- set_role(r, "genotype", role = "treatment") + r <- set_role(r, "state", role = "covariate", action = state_action) + set_role(r, "yield", role = "outcome") +} + +state_cf <- function(raw = state_raw()) { + conform_table(raw, merge_labels = "auto", types = "auto", quiet = TRUE) +} + +test_that("mask() records conform decisions with their final targets", { + cf <- state_cf() + m <- mask(cf, state_roles(cf$data, "local"), mode = "local", seed = 1) + rec <- recipe(m)@conform + expect_type(rec, "list") + expect_s3_class(rec$merges, "data.frame") + expect_false(inherits(rec$merges, "data.table")) + expect_setequal(rec$merges$from, c("nsw", "Nsw")) + expect_true(all(rec$merges$col == "state")) + expect_true(all(rec$merges$to == "NSW")) + expect_setequal(rec$types$col, c("genotype", "state")) + expect_true(all(rec$types$to == "factor")) + expect_identical(rec$max_levels, 20L) + expect_identical(rec$modes$merge_labels, "auto") + expect_identical(rec$modes$types, "auto") +}) + +test_that("apply_recipe replays merges and types in local mode", { + raw <- state_raw() + cf <- state_cf(raw) + m <- mask(cf, state_roles(cf$data, "local"), mode = "local", seed = 1) + fwd <- apply_recipe(raw, recipe(m)) + expect_identical(fwd$state, cf$data$state) + expect_identical(fwd$genotype, cf$data$genotype) +}) + +test_that("apply_recipe replays merges and types in collaborate mode", { + raw <- state_raw() + cf <- state_cf(raw) + # Kept: the replayed column equals the conformed one. + m1 <- mask(cf, state_roles(cf$data, "collaborate", "keep"), + mode = "collaborate", seed = 1 + ) + expect_identical(apply_recipe(raw, recipe(m1))$state, cf$data$state) + + # Aliased: the replayed column lands on the clone's aliases and unmasks to + # the conformed labels. + m2 <- mask(cf, state_roles(cf$data, "collaborate"), + mode = "collaborate", seed = 1 + ) + rec <- recipe(m2) + fwd <- apply_recipe(raw, rec) + expect_identical(fwd$state, apply_recipe(cf$data, rec)$state) + expect_setequal( + as.character(unique(fwd$state)), + as.character(unique(synthetic(m2)$state)) + ) + expect_identical( + as.character(unmask(fwd, rec)$state), as.character(cf$data$state) + ) +}) + +test_that("a recipe with no conform decisions replays nothing", { + raw <- state_raw() + m <- mask(raw, state_roles(raw, "local"), mode = "local", seed = 1) + expect_null(recipe(m)@conform) + expect_identical(apply_recipe(raw, recipe(m))$state, raw$state) +}) + +test_that("illegal headers and conform decisions replay together", { + raw <- state_raw() + names(raw) <- c("Rep", "Genotype", "State Name", "Yield (t/ha)") + cf <- suppressWarnings( + conform_table(raw, merge_labels = "auto", types = "auto", quiet = TRUE) + ) + r <- propose_roles(cf$data, mode = "local", detect = FALSE) + r <- set_role(r, "Rep", role = "design") + r <- set_role(r, "Genotype", role = "treatment") + r <- set_role(r, "Yield..t.ha.", role = "outcome") + m <- mask(cf, r, mode = "local", seed = 1) + rec <- recipe(m) + fwd <- apply_recipe(raw, rec) + expect_identical(names(fwd), names(synthetic(m))) + expect_identical(fwd$State.Name, cf$data$State.Name) + expect_identical(names(unmask(synthetic(m), rec)), names(raw)) +}) + +test_that("mask_set() takes a conformance object per table and replays it", { + raw <- state_raw() + cf <- state_cf(raw) + other <- data.frame( + genotype = sprintf("G%d", 1:4), height = c(80, 95, 102, 88), + stringsAsFactors = FALSE + ) + ro <- propose_roles(other, mode = "local", detect = FALSE) + ro <- set_role(ro, "genotype", role = "treatment") + roles <- list(trial = state_roles(cf$data, "local"), lines = ro) + m <- mask_set(list(trial = cf, lines = other), roles = roles, + mode = "local", seed = 1, quiet = TRUE + ) + expect_false(is.null(recipe(m)@recipes$trial@conform)) + fwd <- apply_recipe(list(trial = raw, lines = other), recipe(m)) + expect_identical(fwd$trial$state, cf$data$state) +}) + +colour_frame <- function() { + data.frame( + colour = c(rep("color", 5), rep("colour", 5), "colur", rep("red", 4)), + stringsAsFactors = FALSE + ) +} + +test_that("a source with two candidate targets is not merged", { + expect_warning( + cf <- conform_table( + colour_frame(), merge_labels = "auto", types = "auto", quiet = TRUE + ), + class = "masque_conform_ambiguous_merge" + ) + expect_true(all(c("color", "colour", "colur") %in% cf$data$colour)) + amb <- cf$merges[cf$merges$from == "colur", , drop = FALSE] + expect_gt(nrow(amb), 0L) + expect_true(all(!amb$applied)) + expect_true(all(grepl("more than one", amb$reason))) + expect_false("colur" %in% cf$merge_map$from) +}) + +test_that("merges follow chains to the label left in the data", { + cf <- state_cf() + expect_setequal(cf$merge_map$from, c("nsw", "Nsw")) + expect_true(all(cf$merge_map$to == "NSW")) + expect_true(all(cf$merge_map$to %in% as.character(cf$data$state))) +}) + +test_that("types = 'off' still merges categories and leaves storage alone", { + raw <- state_raw() + cf <- conform_table(raw, merge_labels = "auto", types = "off", quiet = TRUE) + expect_type(cf$data$state, "character") + expect_false(any(c("nsw", "Nsw") %in% cf$data$state)) + expect_equal(nrow(cf$types), 0L) + # Numbers held as text are still not categories. + num <- data.frame(y = c("3.1", "4.2", "4.4", "4.4", "5.0")) + cn <- conform_table(num, merge_labels = "auto", types = "off", quiet = TRUE) + expect_equal(nrow(cn$merges), 0L) + expect_identical(cn$data$y, num$y) +}) + +test_that("a recipe saved before the conform property existed still applies", { + raw <- state_raw() + rec <- recipe(mask(raw, state_roles(raw, "local"), mode = "local", seed = 1)) + # Rebuild the object as an older masque wrote it: the class definition it + # carries has no `conform` property and the attribute is absent. + cls <- attr(rec, "S7_class") + old_cls <- S7::new_class( + "masque_recipe", package = "masque", + properties = cls@properties[setdiff(names(cls@properties), "conform")] + ) + old <- rec + attr(old, "conform") <- NULL + attr(old, "S7_class") <- old_cls + expect_error(old@conform) + expect_identical(apply_recipe(raw, old), apply_recipe(raw, rec)) +}) + +test_that("masque() takes a conformance object as one table or in a set", { + raw <- state_raw() + cf <- state_cf(raw) + m <- masque(cf, roles = state_roles(cf$data, "local"), mode = "local", + seed = 1, ask = FALSE, quiet = TRUE + ) + expect_true(S7::S7_inherits(m, masque_obj)) + expect_identical(apply_recipe(raw, recipe(m))$state, cf$data$state) + + ms <- masque(list(trial = cf), mode = "local", seed = 1, ask = FALSE, + quiet = TRUE + ) + expect_true(S7::S7_inherits(ms, masque_set)) + expect_false(is.null(recipe(ms)@recipes$trial@conform)) +}) From bbdb939817d888f90f0b0f62492b3104bbbcc25f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:27:49 +1030 Subject: [PATCH 16/66] fix: coded labels are not near-duplicates; cap the report and record Labels that differ only inside a run of digits (G01/G02, Plot12/Plot13) are different codes: clean_table() no longer lists them and conform_table(merge_labels = 'auto') no longer merges them (it merged genotype codes into each other before). The hygiene report prints 20 pairs and the count; a recipe keeps the count and 20 pairs as a plain data frame, so a saved recipe reads back identical(). --- R/clean_table.R | 90 ++++++++++++++++++------- R/conform_table.R | 4 +- tests/testthat/test-fix0160-near-dups.R | 68 +++++++++++++++++++ tests/testthat/test-fix0160-set-names.R | 7 +- 4 files changed, 137 insertions(+), 32 deletions(-) create mode 100644 tests/testthat/test-fix0160-near-dups.R diff --git a/R/clean_table.R b/R/clean_table.R index ee0b50b..ad6403b 100644 --- a/R/clean_table.R +++ b/R/clean_table.R @@ -25,6 +25,11 @@ #' fixes and advisories is printed. Set `TRUE` to suppress it (the #' report object is returned either way). #' +#' @details Two input columns with the same name are refused with a +#' `masque_duplicate_names_refusal` error: the recipe records renamed +#' columns by their original name, so the two could not be told apart on +#' the round trip. Rename them first. +#' #' @return An object of class `masque_cleaning`: a list with #' \itemize{ #' \item `data` - the cleaned (or, under `report` / `off`, @@ -35,7 +40,10 @@ #' values were trimmed, each a named character `original -> clean`; #' \item `near_duplicates` - a data.table of report-only label pairs #' (`col`, `a`, `b`, `kind`) that look like typos but were left -#' untouched; +#' untouched. Labels that differ only inside a run of digits +#' (`"G01"` and `"G02"`, `"Plot12"` and `"Plot13"`) are different +#' codes and are not listed. The printed report shows the first 20 +#' pairs and the count, and a recipe keeps the same; #' \item `mode` - the `clean` mode applied. #' } #' @@ -140,7 +148,10 @@ print.masque_cleaning <- function(x, ...) { return(invisible(nms)) } cli::cli_abort(c( - "{cli::qty(length(dups))}Column name{?s} used more than once: {.val {dups}}.", + paste0( + "{cli::qty(length(dups))}Column name{?s} used more than once: ", + "{.val {dups}}." + ), i = paste0( "masque records renamed columns by their original name, so two ", "columns with one name cannot be restored on the round trip." @@ -206,46 +217,66 @@ print.masque_cleaning <- function(x, ...) { } # Report only: pairs of distinct labels that differ by case or by one edit -# (utils::adist()). +# (utils::adist()). An edit inside a run of digits ("G01" and "G02") is a +# different code, not a typo. .near_duplicate_pairs <- function(x) { vals <- sort(unique(stats::na.omit(as.character(x)))) if (length(vals) < 2L) { return(.empty_near_dups()) } - pairs <- list() lower <- tolower(vals) d <- utils::adist(vals) - for (i in seq_along(vals)) { - for (k in seq_len(i - 1L)) { - kind <- if (lower[i] == lower[k]) { - "case" - } else if (d[i, k] == 1L) { - "edit1" - } else { - next - } - pairs[[length(pairs) + 1L]] <- data.frame( - a = vals[k], b = vals[i], kind = kind, - stringsAsFactors = FALSE - ) - } + # Upper triangle in column order: pairs listed by `b`, then by `a`. + idx <- which( + upper.tri(d) & (d == 1L | outer(lower, lower, "==")), + arr.ind = TRUE + ) + if (!nrow(idx)) { + return(.empty_near_dups()) } - if (!length(pairs)) { + a <- vals[idx[, 1L]] + b <- vals[idx[, 2L]] + kind <- ifelse(lower[idx[, 1L]] == lower[idx[, 2L]], "case", "edit1") + keep <- kind == "case" | !.edit_in_digits(a, b) + if (!any(keep)) { return(.empty_near_dups()) } - out <- do.call(rbind, pairs) - out$col <- NA_character_ - out + data.frame( + a = a[keep], b = b[keep], kind = kind[keep], col = NA_character_, + stringsAsFactors = FALSE + ) } +# For pairs one edit apart: is the edited character a digit, either a digit +# changed for a digit or a digit added or removed? +.edit_in_digits <- function(a, b) { + vapply(seq_along(a), function(i) { + x <- strsplit(a[[i]], "", fixed = TRUE)[[1L]] + y <- strsplit(b[[i]], "", fixed = TRUE)[[1L]] + n <- min(length(x), length(y)) + p <- which(x[seq_len(n)] != y[seq_len(n)])[1L] + if (is.na(p)) p <- n + 1L + if (length(x) == length(y)) { + return(grepl("[0-9]", x[[p]]) && grepl("[0-9]", y[[p]])) + } + longer <- if (length(x) > length(y)) x else y + grepl("[0-9]", longer[[p]]) + }, logical(1L)) +} + +# Near-duplicate pairs printed by the hygiene report and kept on a recipe. +.NEAR_DUP_SHOWN <- 20L + # Distil a masque_cleaning object to the reversible record stored on the -# recipe (the data frame itself is not kept). +# recipe (the data frame itself is not kept). Near-duplicates are advisory: +# the count and the first pairs, as a plain data frame. .cleaning_record <- function(cl) { + nd <- as.data.frame(cl$near_duplicates) list( name_map = cl$name_map, level_fixes = cl$level_fixes, - near_duplicates = cl$near_duplicates, - n_near_duplicates = nrow(cl$near_duplicates), + near_duplicates = utils::head(nd, .NEAR_DUP_SHOWN), + n_near_duplicates = nrow(nd), mode = cl$mode ) } @@ -359,12 +390,19 @@ trimws_keep_class <- function(x) { cli::cli_alert_warning( "{n_dups} near-duplicate label pair{?s} found - NOT changed:" ) - for (i in seq_len(n_dups)) { + shown <- min(n_dups, .NEAR_DUP_SHOWN) + for (i in seq_len(shown)) { row <- cl$near_duplicates[i, ] cli::cli_li( "{.field {row$col}}: {.val {row$a}} ~ {.val {row$b}} ({row$kind})" ) } + if (n_dups > shown) { + cli::cli_text(paste0( + "... and {n_dups - shown} more; the full list is ", + "{.code $near_duplicates}." + )) + } cli::cli_text( "Resolve these by hand if they are typos; masque will not merge them." ) diff --git a/R/conform_table.R b/R/conform_table.R index 4c03305..ce19a77 100644 --- a/R/conform_table.R +++ b/R/conform_table.R @@ -268,8 +268,8 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing candidate == TRUE, list(col, from, to) ]) paths <- data.table::copy(edges) - # Each pass moves every path one merge further; the merge graph has no - # cycles (a merge never goes to a rarer label), so nrow(edges) passes bound it. + # Each pass moves every path one merge further. Merges never go to a rarer + # label, so the graph has no cycles and nrow(edges) passes bound the loop. for (i1 in seq_len(nrow(edges))) { step <- edges[paths, list(col, from = i.from, to = x.to), on = c("col", from = "to"), nomatch = NULL diff --git a/tests/testthat/test-fix0160-near-dups.R b/tests/testthat/test-fix0160-near-dups.R new file mode 100644 index 0000000..472cda2 --- /dev/null +++ b/tests/testthat/test-fix0160-near-dups.R @@ -0,0 +1,68 @@ +# Near-duplicate labels: coded vocabularies are not typos, the report is +# capped, and the recipe keeps a short plain record. + +near_dup_count <- function(x) { + nrow(clean_table(data.frame(g = x), quiet = TRUE)$near_duplicates) +} + +test_that("an edit inside a run of digits is not a near-duplicate", { + expect_equal(near_dup_count(rep(sprintf("G%02d", 1:24), 2)), 0L) + expect_equal(near_dup_count(sprintf("G%03d", 1:80)), 0L) + expect_equal(near_dup_count(c("Plot12", "Plot13", "Plot2", "Plot21")), 0L) + expect_equal(near_dup_count(c("Year2023", "Year2024")), 0L) +}) + +test_that("real near-duplicates are still reported", { + nd <- clean_table( + data.frame(g = c("north", "North", "Ashfield", "Ashfiel", "G0l", "G01")), + quiet = TRUE + )$near_duplicates + pairs <- vapply(seq_len(nrow(nd)), function(i) { + paste(sort(c(nd$a[i], nd$b[i]), method = "radix"), collapse = " ") + }, character(1L)) + expect_setequal(pairs, c("North north", "Ashfiel Ashfield", "G01 G0l")) +}) + +wordy <- function() { + # 26 labels, every pair one letter apart: 325 near-duplicate pairs. + data.frame(word = rep(paste0("word", letters), 2)) +} + +test_that("the printed near-duplicate report is capped at 20 pairs", { + msgs <- testthat::capture_messages(clean_table(wordy())) + pair_lines <- grep("~", msgs, value = TRUE, fixed = TRUE) + expect_equal(length(pair_lines), 20L) + expect_true(any(grepl("305 more", msgs, fixed = TRUE))) + expect_true(any(grepl("325", msgs, fixed = TRUE))) +}) + +test_that("the recipe keeps the count and 20 pairs as a plain data frame", { + d <- wordy() + d$y <- seq_len(nrow(d)) + r <- propose_roles(d, mode = "local", detect = FALSE) + rec <- recipe(mask(d, r, mode = "local", seed = 1)) + expect_identical(rec@cleaning$n_near_duplicates, 325L) + expect_identical(class(rec@cleaning$near_duplicates), "data.frame") + expect_equal(nrow(rec@cleaning$near_duplicates), 20L) + + path <- withr::local_tempfile(fileext = ".rds") + saveRDS(rec, path) + expect_identical(readRDS(path), rec) +}) + +test_that("conform_table never merges codes that differ inside a digit run", { + g <- factor(c( + rep("G001", 3), rep("G002", 2), rep("G010", 4), "G011", rep("G020", 2) + )) + cf <- conform_table(data.frame(genotype = g), merge_labels = "auto", + quiet = TRUE + ) + expect_identical(cf$data$genotype, g) + expect_equal(nrow(cf$merges), 0L) + + e <- c(rep("Entry10", 5), rep("Entry11", 2), rep("Entry12", 3), "Entry13") + ce <- conform_table(data.frame(entry = e), merge_labels = "auto", + types = "auto", quiet = TRUE + ) + expect_identical(as.character(ce$data$entry), e) +}) diff --git a/tests/testthat/test-fix0160-set-names.R b/tests/testthat/test-fix0160-set-names.R index 1bfbb67..d879aa2 100644 --- a/tests/testthat/test-fix0160-set-names.R +++ b/tests/testthat/test-fix0160-set-names.R @@ -1,6 +1,5 @@ -# Set layer with headers that need legalising: the guided path proposes on the -# cleaned names, roles keyed either way are accepted, and each table's recipe -# carries its cleaning record so the set round-trips like a single table. +# Sets whose headers need legalising: guided proposals, roles keyed either way, +# and the per-table cleaning record that makes the set round-trip. names_set <- function() { set.seed(11) @@ -155,7 +154,7 @@ collision_set <- function() { list(a = a, b = b) } -test_that("Site Name and Site.Name legalise without a duplicate, both keyings", { +test_that("Site Name beside Site.Name legalises without a duplicate", { tabs <- collision_set() cleaned <- clean_names_of(tabs) expect_named(cleaned$a, c("Site.Name", "Site.Name_1", "Yield")) From b2fa1d8729381ff9ccd66c8b816b913d4b97833f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:30:50 +1030 Subject: [PATCH 17/66] fix(detect): structural treatment hints need design evidence (F18, N5) A treatment hint from the design rules is auto-applied only for a treatment-pattern name, a user treatment, or a column crossed with a design column (design-pattern name, column roled design, or detected environment). On the scoped path, a column with one value per level of a design column and a column with an outcome-pattern name are no longer design or treatment candidates, a treatment-pattern name counts whatever its cardinality, and a split-plot names both factors as the treatment. DESIGN_PATTERN gains site_name and trial_site in snake, spaced and dotted forms (also environment candidates); numeric columns named yield, grain_yield, response or outcome are proposed as outcome. env = FALSE is unchanged. --- R/detect_candidates.R | 115 +++++++++++- R/detect_design.R | 86 ++++++++- R/detect_environment.R | 8 +- R/detect_print.R | 11 ++ R/propose_roles.R | 32 +++- tests/testthat/test-fix0160-blocks.R | 10 +- tests/testthat/test-fix0160-promotion.R | 230 ++++++++++++++++++++++++ tests/testthat/test-propose_roles.R | 26 ++- tests/testthat/test-roles-actions.R | 12 +- 9 files changed, 497 insertions(+), 33 deletions(-) create mode 100644 tests/testthat/test-fix0160-promotion.R diff --git a/R/detect_candidates.R b/R/detect_candidates.R index 0834f2e..eef9cae 100644 --- a/R/detect_candidates.R +++ b/R/detect_candidates.R @@ -4,11 +4,17 @@ #' @param roles Optional roles table from `propose_roles()`. Columns roled as #' outcome, id, text or unsupported, or dropped, are left out of the #' candidates; a column roled `treatment` goes into `trt_user`. +#' @param revise If `TRUE` (the scoped path of `detect_design()`), outcome- +#' pattern names and columns constant within a design unit leave the +#' factors, and a treatment-pattern name is a treatment candidate whatever +#' its cardinality. `FALSE` keeps the legacy candidate sets. #' @returns A named list: `n_rows`, `cols`, `kinds`, `cardinality`, and the #' candidate sets `factors`, `numerics`, `spatial` (row and column #' coordinates, or `NULL`), `trt_named`, `block_named` and `trt_user`. +#' With `revise = TRUE` it also holds `excluded`, the reason per column +#' left out. #' @noRd -.propose_candidates <- function(df, roles = NULL) { +.propose_candidates <- function(df, roles = NULL, revise = FALSE) { n <- nrow(df) cols <- names(df) kinds <- vapply(df, col_kind, character(1L)) @@ -47,6 +53,12 @@ is_factor_like <- is_factor_like & !(cols %in% drop_candidates) } + excluded <- character(0L) + if (revise) { + excluded <- .excluded_candidates(df, cols[is_factor_like], roles) + is_factor_like <- is_factor_like & !(cols %in% names(excluded)) + } + factors <- cols[is_factor_like] numerics <- cols[ kinds %in% c("numeric", "integer") & @@ -60,7 +72,15 @@ trt_named <- cols[vapply(cols, matches_pattern, logical(1L), pat = TREATMENT_PATTERN )] - trt_named <- intersect(trt_named, factors) + # Revised: a named treatment with many levels (an augmented trial) counts. + named_ok <- if (revise) { + cardinality >= 2L & cardinality < n & + kinds %in% c(.categorical_kinds(), .numeric_kinds()) & + !(cols %in% drop_candidates) + } else { + is_factor_like + } + trt_named <- intersect(trt_named, cols[named_ok]) # Reuse propose_roles()'s DESIGN_PATTERN -- covers rep, block, row, col, # range, plot, site, env, trial, year, season. @@ -74,7 +94,7 @@ trt_user <- roles$col[roles$role == "treatment"] } - list( + out <- list( n_rows = n, cols = cols, kinds = kinds, @@ -87,6 +107,95 @@ block_named = block_named, trt_user = trt_user ) + if (revise) { + out$excluded <- excluded + } + out +} + +# Factor candidates that are covariates, as a named vector of reasons: an +# outcome-pattern name, or one value per unit of a design column. +.excluded_candidates <- function(df, candidates, roles = NULL) { + units <- .design_unit_cols(df, roles) + user_treatment <- character(0L) + if (!is.null(roles) && all(c("col", "role") %in% names(roles))) { + user_treatment <- roles$col[roles$role == "treatment"] + } + protected <- union(units, user_treatment) + out <- character(0L) + for (col in setdiff(candidates, protected)) { + if (matches_pattern(col, TREATMENT_PATTERN) || + matches_pattern(col, DESIGN_PATTERN)) { + next + } + if (matches_pattern(col, OUTCOME_PATTERN)) { + out[[col]] <- "outcome-pattern name" + next + } + unit <- .constant_within_unit(df, col, setdiff(units, col)) + if (!is.na(unit)) { + out[[col]] <- sprintf("one value per level of design column %s", unit) + } + } + out +} + +# Design columns whose levels are replicated units: design-pattern names and +# columns roled design, unless the roles table says otherwise, with at least +# two levels and two rows per level on average. +.design_unit_cols <- function(df, roles = NULL, extra = character(0L)) { + cols <- names(df) + design <- cols[matches_pattern(cols, DESIGN_PATTERN)] + if (!is.null(roles) && all(c("col", "role") %in% names(roles))) { + dropped <- if ("action" %in% names(roles)) { + roles$col[!is.na(roles$action) & roles$action == "drop"] + } else { + character(0L) + } + design <- union( + setdiff(design, roles$col[roles$role != "design"]), + roles$col[roles$role == "design"] + ) + design <- setdiff(design, dropped) + } + design <- intersect(union(design, extra), cols) + replicated <- vapply(design, function(col) { + x <- df[[col]] + n_levels <- data.table::uniqueN(x, na.rm = TRUE) + n_levels >= 2L && sum(!is.na(x)) >= 2L * n_levels + }, logical(1L)) + design[replicated] +} + +# The first unit column within whose every level `col` takes at most one +# non-missing value, or NA when there is none. +.constant_within_unit <- function(df, col, units) { + for (unit in units) { + pairs <- .unit_value_pairs(df, col, unit) + if (nrow(pairs) > 0L && + data.table::uniqueN(pairs, by = c("unit", "value")) == + data.table::uniqueN(pairs, by = "unit")) { + return(unit) + } + } + NA_character_ +} + +# Crossed: some value of `col` in two or more units, and some unit holding two +# or more values. +.crossed_with_unit <- function(df, col, unit) { + pairs <- .unit_value_pairs(df, col, unit) + n_pairs <- data.table::uniqueN(pairs, by = c("unit", "value")) + n_pairs > data.table::uniqueN(pairs, by = "unit") && + n_pairs > data.table::uniqueN(pairs, by = "value") +} + +# Complete (unit, value) rows for one candidate and one unit column. +.unit_value_pairs <- function(df, col, unit) { + pairs <- data.frame( + unit = df[[unit]], value = df[[col]], stringsAsFactors = FALSE + ) + pairs[stats::complete.cases(pairs), , drop = FALSE] } # Look for an (integer-like, gridded) row + col / column / range pair. diff --git a/R/detect_design.R b/R/detect_design.R index 6076ae5..40ef32c 100644 --- a/R/detect_design.R +++ b/R/detect_design.R @@ -15,7 +15,7 @@ #' trial. Supply `env` to define the environment basis, or use `env = FALSE` #' to run the pre-0.9 whole-table path exactly. #' -#' After the scope step, the pooled legacy detector runs six independent +#' Before the scope step, the pooled legacy detector runs six independent #' design rules. Each returns a score in \eqn{[0, 1]}. The highest-scoring #' class above `threshold` is one of `"CRD"`, `"RCBD"`, #' `"IBD/alpha-lattice"`, `"row-column"`, `"split-plot"`, `"factorial"`, @@ -25,6 +25,15 @@ #' that exceed the package safety bound are reported as `not_computed`. #' None of these diagnostics proves the original randomisation protocol. #' +#' When scope resolves, `block_cols` is the block basis the environments +#' agree on, or empty when they disagree; an environment column is never a +#' block. `evidence$block_revision` records the change. +#' +#' A treatment hint found by the rules rather than by a treatment-pattern +#' name is auto-applied only when the column is crossed with a design +#' column. A column with one value per level of a design column, or with an +#' outcome-pattern name, is a covariate and not a candidate. +#' #' The detector never edits `df`. Its job is to recommend a role #' assignment, surface the evidence, and (optionally) draw a sanity #' check via [plot()]. @@ -112,7 +121,8 @@ detect_design <- function(df, interactive = interactive, threshold = threshold, tie_delta = tie_delta, - scope = .scope_unknown() + scope = .scope_unknown(), + revise = TRUE ) if (env_spec$mode == "explicit") { @@ -138,19 +148,23 @@ detect_design <- function(df, out@recommended_roles <- .normalise_role_recommendations( out@recommended_roles, df ) + out@recommended_roles <- .gate_treatment_hints(out, df, roles) out } # Legacy whole-table detector. Keep this path free of environment inference so -# `env = FALSE` remains a stable compatibility escape hatch. +# `env = FALSE` remains a stable compatibility escape hatch. `revise = TRUE` +# (scoped path only) applies the candidate revisions of .propose_candidates() +# and names both split-plot factors as the treatment. .detect_design_legacy <- function(df, roles = NULL, interactive = FALSE, threshold = 0.5, tie_delta = 0.02, - scope = .scope_unknown()) { + scope = .scope_unknown(), + revise = FALSE) { - cands <- .propose_candidates(df, roles = roles) + cands <- .propose_candidates(df, roles = roles, revise = revise) # Run every rule; collect score + result. results <- lapply(.rules_all, function(rule) rule(df, cands)) @@ -179,10 +193,70 @@ detect_design <- function(df, } result <- results[[picked]] - .build_design_summary( + out <- .build_design_summary( picked, result, scores, cands, warnings_msgs, scope = scope ) + if (revise && identical(picked, "split-plot")) { + out@treatment_col <- c(out@whole_plot_col, out@sub_plot_col) + } + out +} + +# A treatment hint from the design rules stays auto-applied only for a +# treatment-pattern name, a user treatment, or a column crossed with a design +# column; otherwise auto_apply becomes FALSE and the reason says why. +.gate_treatment_hints <- function(out, df, roles) { + rec <- out@recommended_roles + hints <- which( + rec$role == "treatment" & rec$source == "design_rule" & rec$auto_apply + ) + if (length(hints) == 0L) { + return(rec) + } + named <- union( + out@candidates$trt_user, + names(df)[matches_pattern(names(df), TREATMENT_PATTERN)] + ) + units <- .design_unit_cols(df, roles, extra = out@env_cols) + for (i in hints) { + col <- rec$col[i] + if (col %in% named) next + problem <- .treatment_evidence_problem(df, col, setdiff(units, col)) + if (!is.na(problem)) { + rec$auto_apply[i] <- FALSE + rec$reason[i] <- sprintf("Not applied: %s.", problem) + } + } + rec +} + +# Why a column found by the design rules is not a treatment, or NA. +.treatment_evidence_problem <- function(df, col, units) { + x <- df[[col]] + if (matches_pattern(col, DESIGN_PATTERN)) { + return("a design-pattern name") + } + if (is.numeric(x) && any(table(x) < 2L)) { + return("a numeric column with values seen once is a measurement") + } + if (length(units) == 0L) { + return("no design column in the table crosses it") + } + unit <- .constant_within_unit(df, col, units) + if (!is.na(unit)) { + return(sprintf( + "one value per level of design column %s (a unit-level covariate)", + unit + )) + } + crossed <- vapply(units, function(unit) { + .crossed_with_unit(df, col, unit) + }, logical(1L)) + if (!any(crossed)) { + return("not crossed with any design column") + } + NA_character_ } # Pick the top class label using a simpler-is-better tie-break. diff --git a/R/detect_environment.R b/R/detect_environment.R index d7af13e..6c39585 100644 --- a/R/detect_environment.R +++ b/R/detect_environment.R @@ -255,7 +255,10 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 ] site <- available[ grepl( - "^(site|location|loc|county|region|field|farm|place)$", + paste0( + "^(site|location|loc|county|region|field|farm|place|", + "site[ ._]?name|trial[ ._]?site)$" + ), available, ignore.case = TRUE, perl = TRUE @@ -679,7 +682,8 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 interactive = FALSE, threshold = threshold, tie_delta = tie_delta, - scope = .scope_disabled() + scope = .scope_disabled(), + revise = TRUE ) treatment_present <- intersect(treatment_cols, names(slice)) n_treatments <- if (length(treatment_present) == 0L) { diff --git a/R/detect_print.R b/R/detect_print.R index c1c01d9..a4345c0 100644 --- a/R/detect_print.R +++ b/R/detect_print.R @@ -133,6 +133,17 @@ local(S7::method(print, design_summary) <- function(x, ...) { "; review disclosure." )) } + held <- x@recommended_roles$col[ + x@recommended_roles$source == "design_rule" & + !x@recommended_roles$auto_apply + ] + if (length(held) > 0L) { + cli::cli_alert_info(paste0( + "Not applied: ", paste(held, collapse = ", "), + ". A structural hint needs a design column crossing it; see ", + "{.code x@recommended_roles$reason}." + )) + } } } diff --git a/R/propose_roles.R b/R/propose_roles.R index 2bd7766..d41f7ad 100644 --- a/R/propose_roles.R +++ b/R/propose_roles.R @@ -66,16 +66,21 @@ #' 3. ID-pattern names (`\\bid\\b`, `_id$`, `^id_`) -> role `id`; kept #' in local mode, dropped in collaborate mode. #' 4. Design-pattern names (`rep`, `block`, `row`, `col(umn)?`, -#' `range`, `plot(no)?`, `site`, `env(ironment)?`, `trial`, `year`, -#' `season`, `colrep`, `tos`) -> role `design`, action `keep`. +#' `range`, `plot(no)?`, `site`, `site_name`, `trial_site`, +#' `env(ironment)?`, `trial`, `year`, `season`, `colrep`, `tos`; the +#' two-word names also spaced or dotted, as in `Site Name` and +#' `Site.Name`) -> role `design`, action `keep`. #' 5. Treatment-pattern names (`treatment`, `variety`, `cultivar`, #' `genotype`, `^trt`, `^dose`) -> role `treatment`; kept in local #' mode, aliased in collaborate mode. -#' 6. Character columns with > 50% unique values on non-NA -> role +#' 6. Numeric columns with an outcome-pattern name (`yield`, `yield_t_ha`, +#' `Yield..t.ha.`, `grain_yield`, `response`, `outcome`) -> role +#' `outcome`, action `scramble`. +#' 7. Character columns with > 50% unique values on non-NA -> role #' `text`; kept in local mode, dropped in collaborate mode. -#' 7. Unsupported classes -> role `other`, action `keep`, with a note. -#' 8. Everything else -> role `covariate`, action `scramble`. Re-role -#' response variables as `outcome`. +#' 8. Unsupported classes -> role `other`, action `keep`, with a note. +#' 9. Everything else -> role `covariate`, action `scramble`. Re-role +#' other response variables as `outcome`. #' #' No outcome is required: with no column roled `outcome`, the copula #' simply re-simulates all scrambled numeric columns jointly. @@ -87,6 +92,14 @@ #' stashed as `attr(roles, "design")`. Pass `detect = FALSE` for the #' name-only heuristic. #' +#' A structural treatment hint (one found by the design rules, not by a +#' treatment-pattern name) is applied only when the column is crossed with a +#' design column: a design-pattern name, a column roled `design`, or a +#' detected environment. A column with one value per level of a design +#' column is a unit-level covariate and is never promoted. Hints that fail +#' are listed in `attr(roles, "design")@recommended_roles` with +#' `auto_apply = FALSE` and the reason. +#' #' @section Multi-environment trials: #' #' Since masque 0.9.1, high-confidence environment columns detected by @@ -297,6 +310,11 @@ classify_one_column <- function(nm, x, mode = "local") { } else { "Treatment-pattern name -> passes through in local mode." } + } else if (kind %in% .numeric_kinds() && + matches_pattern(nm, OUTCOME_PATTERN)) { + role <- "outcome" + action <- .default_action(role, kind, mode) + notes <- "Outcome-pattern name -> outcome (re-simulated)." } else if (kind == "character" && is_free_text(x)) { role <- "text" action <- .default_action(role, kind, mode) @@ -371,9 +389,11 @@ is_free_text <- function(x, threshold = 0.5) { ID_PATTERN <- "(\\bid\\b|_id$|^id_)" DESIGN_PATTERN <- paste0( "(^rep[0-9]*$|^block$|^row$|^col(umn)?$|^range$|^plot(no)?$|^site$|", + "^site[ ._]?name$|^trial[ ._]?site$|", "^env(ironment)?$|^trial(_?acronym)?$|^year$|^season$|^colrep$|^tos$)" ) TREATMENT_PATTERN <- "(treatment|^variety|^cultivar|genotype|^trt|^dose$)" +OUTCOME_PATTERN <- "(^(grain[ ._]?)?yield([ ._].*)?$|^response$|^outcome$)" PII_PATTERN <- paste0( "(contact|email|phone|address|gps|latitude|longitude|\\blat\\b|", "\\blon\\b|postcode|postal|^zip|ssn|dob|birthdate|password|secret|", diff --git a/tests/testthat/test-fix0160-blocks.R b/tests/testthat/test-fix0160-blocks.R index 0b2d274..806b9f9 100644 --- a/tests/testthat/test-fix0160-blocks.R +++ b/tests/testthat/test-fix0160-blocks.R @@ -93,12 +93,10 @@ test_that("the F6 fixtures have the structure they are built with", { d <- .f6_two_year_met() - per_rep <- data.table::as.data.table(d)[ - , .(n_blocks = data.table::uniqueN(block), n = .N), - by = .(year, site, rep) - ] - expect_true(all(per_rep$n_blocks == 6L) && all(per_rep$n == 24L)) - expect_true(all(table(paste(d$year, d$site, d$rep, d$block)) == 4L)) + site_year_rep <- paste(d$year, d$site, d$rep) + expect_true(all(tapply(d$block, site_year_rep, data.table::uniqueN) == 6L)) + expect_true(all(table(site_year_rep) == 24L)) + expect_true(all(table(paste(site_year_rep, d$block)) == 4L)) a <- .f6_plot_artefact() expect_true(all(table(a$site, a$block, a$genotype) == 1L)) diff --git a/tests/testthat/test-fix0160-promotion.R b/tests/testthat/test-fix0160-promotion.R new file mode 100644 index 0000000..ea927ca --- /dev/null +++ b/tests/testthat/test-fix0160-promotion.R @@ -0,0 +1,230 @@ +# F18 and N5: a column becomes a treatment by name, or by structure only when +# the table holds design evidence: a design column the candidate is crossed +# with. Expected roles follow from how each fixture is built. + +# 80 patients, eight ICD-10 codes of ten patients each, no design column. +.f18_clinical <- function() { + set.seed(80) + n <- 80L + dx <- c("C18.7", "C25.9", "C34.1", "C50.9", "C61", "C64.9", "C67.9", "C71.9") + data.frame( + patient_id = sprintf("P%03d", seq_len(n)), + age = round(rnorm(n, 64, 10)), + sex = factor(sample(c("F", "M"), n, replace = TRUE)), + diagnosis = factor(rep(dx, each = 10L)[sample(n)]), + stage = factor(sample(c("I", "II", "III", "IV"), n, replace = TRUE)), + biomarker = round(rlnorm(n, 2, 0.6), 2), + stringsAsFactors = FALSE + ) +} + +# Augmented trial: 3 blocks of 10 plots; 18 test entries once each and two +# checks twice in every block, so 18 of 20 genotypes are seen once. +.f18_augmented <- function(integer_yield = FALSE) { + set.seed(30) + tests <- sprintf("E%02d", 1:18) + blocks <- lapply(1:3, function(b) { + data.frame( + block = b, + genotype = sample(c(tests[(b - 1L) * 6L + 1:6], rep(c("CHK1", "CHK2"), 2))), + stringsAsFactors = FALSE + ) + }) + d <- do.call(rbind, blocks) + d$yield <- if (integer_yield) { + round(rnorm(30L, 5, 1)) + } else { + round(rnorm(30L, 5, 1), 2) + } + d +} + +# One site, 20 genotypes in 20 plots, no replication. +.f18_single_site_unreplicated <- function() { + set.seed(21) + data.frame( + row = 1:20, genotype = sprintf("G%02d", sample(20)), + yield = round(rnorm(20, 4, 0.5), 2), stringsAsFactors = FALSE + ) +} + +# Four sites, each of 15 genotypes once per site. +.f18_multi_site_unreplicated <- function() { + set.seed(22) + d <- expand.grid( + genotype = sprintf("G%02d", 1:15), site = sprintf("S%d", 1:4), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + d$yield <- round(rnorm(nrow(d), 4, 0.5), 2) + d[, c("site", "genotype", "yield")] +} + +# Three sites x 3 reps x 10 genotypes, the site column under a given name. +.f18_site_named <- function(site_col) { + set.seed(23) + d <- expand.grid( + genotype = sprintf("G%02d", 1:10), rep = 1:3, + site = c("Roseworthy", "Minnipa", "Loxton"), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + d$yield <- round(rnorm(nrow(d), 3, 0.5), 2) + d <- d[, c("site", "rep", "genotype", "yield")] + names(d)[1L] <- site_col + d +} + +# 6 trials x 20 genotypes x 3 reps, four blocks of five in each rep with +# labels restarted; aet (numeric) and soil (categorical) constant within trial. +.n5_trials <- function() { + set.seed(1) + trials <- sprintf("T%d", 1:6) + gens <- sprintf("G%02d", 1:20) + g <- expand.grid( + genotype = gens, rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$block <- NA_character_ + for (t in trials) for (r in 1:3) { + idx <- which(g$trial == t & g$rep == r) + g$block[idx] <- sprintf("B%d", sample(rep(1:4, each = 5))) + } + trial_eff <- c(0.8, -0.5, 0.3, 1.1, -0.2, 0.0) + g$yield <- round( + 3 + trial_eff[match(g$trial, trials)] + rnorm(nrow(g), 0, 0.35), 2 + ) + g$aet <- c(312.456, 287.901, 401.233, 355.678, 298.112, 376.543)[ + match(g$trial, trials) + ] + g$plot_cov <- round(rnorm(nrow(g), 50, 8), 2) + g$soil <- c("clay", "loam", "sand", "clay", "loam", "sand")[ + match(g$trial, trials) + ] + g[, c( + "trial", "rep", "block", "genotype", "yield", "aet", "plot_cov", "soil" + )] +} + +.role_of <- function(roles, col) roles$role[roles$col == col] +.action_of <- function(roles, col) roles$action[roles$col == col] + +test_that("a design-free clinical code is not promoted to treatment", { + d <- .f18_clinical() + for (mode in c("local", "collaborate")) { + r <- propose_roles(d, mode = mode) + expect_equal(.role_of(r, "diagnosis"), "covariate", info = mode) + expect_false("diagnosis" %in% masque:::.conditioning_cols(r), info = mode) + } + ds <- detect_design(d) + rec <- ds@recommended_roles + hint <- rec[rec$col == "diagnosis" & rec$role == "treatment", ] + expect_equal(nrow(hint), 1L) + expect_false(hint$auto_apply) + expect_match(hint$reason, "no design column") + printed <- paste(capture.output(print(ds), type = "message"), collapse = "\n") + expect_match(printed, "Not applied: diagnosis") +}) + +test_that("an augmented trial keeps genotype and never treats yield", { + for (integer_yield in c(FALSE, TRUE)) { + d <- .f18_augmented(integer_yield) + info <- paste("integer_yield =", integer_yield) + expect_equal(sum(table(d$genotype) == 1L), 18L, info = info) + r <- propose_roles(d) + expect_equal(.role_of(r, "genotype"), "treatment", info = info) + expect_equal(.role_of(r, "block"), "design", info = info) + expect_equal(.role_of(r, "yield"), "outcome", info = info) + ds <- detect_design(d) + # No rule models an augmented design; whatever it names, it is genotype. + expect_true(all(ds@treatment_col %in% "genotype"), info = info) + expect_false(any(c("yield", "block") %in% ds@treatment_col), info = info) + treatment_hints <- ds@recommended_roles$col[ + ds@recommended_roles$role == "treatment" + ] + expect_false("yield" %in% treatment_hints, info = info) + expect_false("block" %in% treatment_hints, info = info) + } +}) + +test_that("unreplicated genotypes found by name stay treatments", { + single <- propose_roles(.f18_single_site_unreplicated()) + expect_equal(.role_of(single, "genotype"), "treatment") + expect_equal(.role_of(single, "row"), "design") + + multi <- propose_roles(.f18_multi_site_unreplicated()) + expect_equal(.role_of(multi, "genotype"), "treatment") + expect_equal(.role_of(multi, "site"), "design") +}) + +test_that("site-name spellings are design columns, raw and legalised", { + spellings <- c( + "site_name", "Site Name", "Site.Name", "site.name", + "trial_site", "Trial Site", "Trial.Site" + ) + for (nm in spellings) { + d <- .f18_site_named(nm) + expect_identical(names(d)[1L], nm) + expect_equal(.role_of(propose_roles(d, detect = FALSE), nm), "design", + info = nm + ) + r <- propose_roles(d) + expect_equal(.role_of(r, nm), "design", info = nm) + expect_equal(.role_of(r, "genotype"), "treatment", info = nm) + } +}) + +test_that("yates.oats: varieties on whole plots, nitrogen on sub-plots", { + skip_if_not_installed("agridat") + # ?agridat::yates.oats: "Varieties were applied to the main plots. + # Manurial (nitrogen) treatments were applied to the sub-plots." 6 blocks. + d <- agridat::yates.oats + ds <- detect_design(d) + expect_identical(ds@class_label, "split-plot") + expect_identical(ds@whole_plot_col, "gen") + expect_identical(ds@sub_plot_col, "nitro") + expect_setequal(ds@treatment_col, c("gen", "nitro")) + expect_identical(ds@block_cols, "block") + + r <- propose_roles(d) + expect_equal(.role_of(r, "gen"), "treatment") + expect_equal(.role_of(r, "nitro"), "treatment") + expect_equal(.role_of(r, "block"), "design") + expect_equal(.role_of(r, "yield"), "outcome") +}) + +test_that("a trial-constant covariate is never a kept treatment (N5)", { + d <- .n5_trials() + # Construction: one aet and one soil value per trial. + expect_true(all(tapply(d$aet, d$trial, data.table::uniqueN) == 1L)) + expect_true(all(tapply(d$soil, d$trial, data.table::uniqueN) == 1L)) + + expected <- c( + trial = "design", rep = "design", block = "design", + genotype = "treatment", yield = "outcome", aet = "covariate", + plot_cov = "covariate", soil = "covariate" + ) + for (mode in c("local", "collaborate")) { + r <- propose_roles(d, mode = mode) + expect_identical( + stats::setNames(r$role, r$col)[names(expected)], expected, + info = mode + ) + expect_equal(.action_of(r, "aet"), "scramble", info = mode) + } +}) + +test_that("the trial table reads as a blocked MET, not a split-plot (N5)", { + d <- .n5_trials() + ds <- detect_design(d) + # Construction: every genotype three times in every trial and six times in + # every rep, so trials are complete blocks of the pooled table. + expect_identical(ds@class_label, "RCBD") + expect_identical(ds@treatment_col, "genotype") + expect_false("aet" %in% c(ds@whole_plot_col, ds@treatment_col)) + + # With trial as the environment, each trial is a resolvable incomplete + # block design: four blocks of five nested in each of three reps. + met <- detect_design(d, env = "trial") + expect_true(isTRUE(met@is_met)) + expect_identical(unique(met@per_env$class_label), "IBD/alpha-lattice") + expect_setequal(met@block_cols, c("rep", "block")) +}) diff --git a/tests/testthat/test-propose_roles.R b/tests/testthat/test-propose_roles.R index 70d7488..62217fb 100644 --- a/tests/testthat/test-propose_roles.R +++ b/tests/testthat/test-propose_roles.R @@ -30,13 +30,27 @@ test_that("propose_roles(detect = FALSE) classifies iris by name (v0.2)", { }) test_that("propose_roles(detect = TRUE) overlays design hints (default)", { + # `arm` has no treatment-pattern name; the RCBD on `block` promotes it. + d <- expand.grid( + block = factor(1:4), arm = factor(c("A", "B", "C")), + KEEP.OUT.ATTRS = FALSE + ) + d$response <- seq_len(nrow(d)) / 10 + r <- propose_roles(d) + expect_equal(r$role[r$col == "arm"], "treatment") + expect_match(r$notes[r$col == "arm"], "detect_design") + expect_equal(attr(r, "design")@class_label, "RCBD") +}) + +test_that("a CRD with no design column leaves its factor a covariate", { r <- propose_roles(iris) - # Species has 3 levels balanced 50/50/50 -> looks like CRD treatment. - expect_equal(r$role[r$col == "Species"], "treatment") - expect_match(r$notes[r$col == "Species"], "detect_design") + # Species is balanced 50/50/50, a CRD, but no design column crosses it. + expect_equal(r$role[r$col == "Species"], "covariate") ds <- attr(r, "design") expect_true(inherits(ds, "masque::design_summary")) expect_equal(ds@class_label, "CRD") + hint <- ds@recommended_roles[ds@recommended_roles$col == "Species", ] + expect_false(hint$auto_apply) }) test_that("propose_roles classifies mtcars (all numeric -> covariate)", { @@ -74,7 +88,7 @@ test_that("PII patterns flag obvious cases and set action = drop", { pii_cols, c("contact_name", "email_addr", "gps_lat", "farmer") ) expect_true(all(r$action[r$pii_suspected] == "drop")) - expect_equal(r$role[r$col == "yield"], "covariate") + expect_equal(r$role[r$col == "yield"], "outcome") }) test_that("Design patterns assign role = design (case-insensitive)", { @@ -95,7 +109,7 @@ test_that("Design patterns assign role = design (case-insensitive)", { expect_setequal( design_cols, c("Rep", "BLOCK", "Row", "col", "site", "year", "plot") ) - expect_equal(r$role[r$col == "yield"], "covariate") + expect_equal(r$role[r$col == "yield"], "outcome") }) test_that("Treatment patterns assign role = treatment", { @@ -129,7 +143,7 @@ test_that("ID patterns assign role = id", { expect_true(all(r$action[r$role == "id"] == "keep")) rc <- propose_roles(df, mode = "collaborate") expect_true(all(rc$action[rc$role == "id"] == "drop")) - # `something` is numeric -> covariate; `yield` is covariate too + # `something` is numeric -> covariate; `yield` is an outcome by name expect_equal(r$role[r$col == "something"], "covariate") }) diff --git a/tests/testthat/test-roles-actions.R b/tests/testthat/test-roles-actions.R index f1f51b5..25900eb 100644 --- a/tests/testthat/test-roles-actions.R +++ b/tests/testthat/test-roles-actions.R @@ -27,15 +27,18 @@ test_that("set_role validates inputs", { }) test_that("default actions differ between local and collaborate", { - rl <- propose_roles(iris, mode = "local") - rc <- propose_roles(iris, mode = "collaborate") - # Species detects as a CRD treatment: kept in local, aliased in collaborate. + # A CRD with no design column is not promoted, so set the treatment role: + # kept in local, aliased in collaborate. + rl <- set_role(propose_roles(iris, mode = "local"), "Species", "treatment") + rc <- set_role( + propose_roles(iris, mode = "collaborate"), "Species", "treatment" + ) expect_equal(rl$action[rl$col == "Species"], "keep") expect_equal(rc$action[rc$col == "Species"], "alias") }) test_that("validating a local table for collaborate re-resolves defaults", { - r <- propose_roles(iris, mode = "local") # Species -> treatment/keep + r <- set_role(propose_roles(iris, mode = "local"), "Species", "treatment") rv <- suppressMessages(roles_validate(r, iris, mode = "collaborate")) expect_equal(rv$action[rv$col == "Species"], "alias") }) @@ -43,6 +46,7 @@ test_that("validating a local table for collaborate re-resolves defaults", { test_that("mask(mode=) re-resolves a default-action table to that mode", { r <- propose_roles(iris, mode = "local") r <- set_role(r, "Sepal.Length", role = "outcome") + r <- set_role(r, "Species", role = "treatment") m <- suppressMessages(suppressWarnings( mask(iris, r, mode = "collaborate", seed = 1) )) From afec00d39114b9f8ff219ca752880559af437080 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:30:54 +1030 Subject: [PATCH 18/66] feat: synthesise unit-constant columns once per design unit (grain = TRUE) A covariate constant within each trial came back varying within trial in every mode, and a per-trial categorical came back with every level inside every trial. .detect_grain() finds, for each re-simulated column, the coarsest design column or pair of design columns within which it holds at most one distinct non-missing value (replicated units only). mask() synthesises those columns on the unit-level table, numerics jointly through the copula and categoricals permuted across units, and broadcasts each unit's value to its rows. Collaborate mode falls back to rows below five units with masque_grain_small_units. grain = FALSE gives the row-level clone, byte for byte. The recipe gains a grain property and one line per grain. A table with no grained column clones exactly as before. --- R/grain.R | 203 +++++++++++ R/mask.R | 65 +++- R/recipe.R | 4 + tests/testthat/test-fix0160-grain.R | 343 ++++++++++++++++++ tests/testthat/test-fix0160-reproducibility.R | 88 +++++ tests/testthat/test-masque-guided.R | 2 +- 6 files changed, 699 insertions(+), 6 deletions(-) create mode 100644 R/grain.R create mode 100644 tests/testthat/test-fix0160-grain.R create mode 100644 tests/testthat/test-fix0160-reproducibility.R diff --git a/R/grain.R b/R/grain.R new file mode 100644 index 0000000..126dd39 --- /dev/null +++ b/R/grain.R @@ -0,0 +1,203 @@ +# grain.R -- The grain of a column, and synthesis at that grain. +# +# A column whose value is fixed for a whole design unit (a trial's rainfall, a +# site's soil class) has the unit as its grain. mask() synthesises such columns +# once per unit and broadcasts each unit's new value to the unit's rows, as the +# geomask does for coordinates, so the clone carries one value per unit and +# averaging a unit's rows recovers nothing. + +#' Detect the grain of the columns mask() re-simulates +#' +#' Candidates are scrambled covariate and outcome columns of numeric, +#' categorical or date kind, and aliased categorical covariates, less +#' `exclude`. Units are the design columns of `df`, singly and in pairs. A +#' column's grain is the unit with the fewest groups within which every group +#' holds at most one distinct non-missing value, among units with at least two +#' groups and at most `nrow(df) / 2`; ties go to fewer columns, then column +#' order. A column constant over the whole table has no grain. +#' +#' @returns A list with one entry per unit holding grained columns: +#' `list(unit, cols, n_units, synthesised = "unit", reason = "")`, in the +#' column order of `df`. +#' @keywords internal +#' @noRd +.detect_grain <- function(df, roles, exclude = character()) { + role <- roles$role + action <- roles$action + kind <- roles$kind + cand <- roles$col[ + ((role %in% c("covariate", "outcome") & action == "scramble" & + kind %in% c(.numeric_kinds(), .categorical_kinds(), .date_kinds())) | + (role == "covariate" & action == "alias" & + kind %in% .categorical_kinds())) & + !roles$col %in% exclude + ] + cand <- intersect(names(df), cand) + design <- intersect(names(df), roles$col[role == "design"]) + n <- nrow(df) + if (!length(cand) || !length(design) || n < 4L) { + return(list()) + } + + varies <- vapply(cand, function(cl) { + data.table::uniqueN(df[[cl]], na.rm = TRUE) > 1L + }, logical(1L)) + cand <- cand[varies] + if (!length(cand)) { + return(list()) + } + + units <- as.list(design) + if (length(design) > 1L) { + units <- c(units, utils::combn(design, 2L, simplify = FALSE)) + } + + # Per unit: its group count and whether each candidate holds at most one + # distinct non-missing value in every group (an all-missing group counts). + fits <- lapply(units, function(u) { + gid <- .unit_id(df, u) + n_groups <- as.integer(max(gid)) + ok <- n_groups >= 2L && n_groups <= n / 2 + constant <- vapply(cand, function(cl) { + if (!ok) { + return(FALSE) + } + x <- df[[cl]] + obs <- !is.na(x) + pairs <- data.table::data.table(g = gid[obs], x = x[obs]) + data.table::uniqueN(pairs) == data.table::uniqueN(pairs$g) + }, logical(1L)) + list(n_groups = n_groups, constant = constant) + }) + n_groups <- vapply(fits, `[[`, integer(1L), "n_groups") + n_cols <- lengths(units) + ord <- order(n_groups, n_cols, seq_along(units)) + + grain_of <- vapply(cand, function(cl) { + hit <- ord[vapply(ord, function(i) fits[[i]]$constant[[cl]], logical(1L))] + if (length(hit)) hit[1L] else NA_integer_ + }, integer(1L)) + grain_of <- grain_of[!is.na(grain_of)] + if (!length(grain_of)) { + return(list()) + } + + lapply(unique(grain_of), function(i) { + list( + unit = units[[i]], + cols = names(grain_of)[grain_of == i], + n_units = n_groups[[i]], + synthesised = "unit", + reason = "" + ) + }) +} + +# Internal: decide per grain group whether it is synthesised per unit. With +# `grain = FALSE` every group is synthesised per row; in collaborate mode a +# group with fewer than .MIN_STRATUM units falls back to rows with a warning. +# Returns the groups and the recipe lines. +.resolve_grain <- function(groups, grain, mode) { + lines <- character() + for (i in seq_along(groups)) { + g <- groups[[i]] + cols <- paste(g$cols, collapse = ", ") + unit <- paste(g$unit, collapse = ":") + if (!isTRUE(grain)) { + g$synthesised <- "row" + g$reason <- "grain = FALSE" + lines <- c(lines, sprintf( + paste0( + "%s %s constant within %s (%d units) but %s synthesised row by row ", + "(grain = FALSE); the clone does not keep that grain." + ), + cols, if (length(g$cols) > 1L) "are" else "is", unit, g$n_units, + if (length(g$cols) > 1L) "were" else "was" + )) + } else if (identical(mode, "collaborate") && g$n_units < .MIN_STRATUM) { + g$synthesised <- "row" + g$reason <- sprintf("fewer than %d units", .MIN_STRATUM) + msg <- sprintf( + paste0( + "%s %s constant within %s, which has %d units, fewer than %d: ", + "resampling that few real values would disclose them, so %s ", + "synthesised row by row and the clone does not keep that grain." + ), + cols, if (length(g$cols) > 1L) "are" else "is", unit, g$n_units, + .MIN_STRATUM, if (length(g$cols) > 1L) "they were" else "it was" + ) + warning(warningCondition(msg, class = "masque_grain_small_units")) + lines <- c(lines, msg) + } else { + lines <- c(lines, sprintf( + paste0( + "%s synthesised once per %s (%d units); %s correlation with ", + "row-level columns is not preserved." + ), + cols, unit, g$n_units, + if (length(g$cols) > 1L) "their" else "its" + )) + } + groups[[i]] <- g + } + list(groups = groups, lines = lines) +} + +# Internal: dense group id (1, 2, ...) of each row over the unit columns; a +# missing design value is a group of its own. +.unit_id <- function(df, unit) { + data.table::frankv(df[, unit, drop = FALSE], ties.method = "dense") +} + +# Internal: synthesise one grain group once per unit and broadcast to rows. +# Numeric columns go through the copula jointly (with collaborate jitter +# against the unit-level values), categorical and date columns are permuted +# across units, and aliased covariates are permuted, then aliased. Rows that +# were missing stay missing. Returns the new columns and any level maps. +.synthesise_grain_group <- function(df, group, roles, jitter) { + cols <- group$cols + row_unit <- .unit_id(df, group$unit) + n_units <- max(row_unit) + # One value per unit: its first non-missing value, NA when it has none. + unit_value <- function(x) { + o <- which(!is.na(x)) + x[o[match(seq_len(n_units), row_unit[o])]] + } + ut <- lapply(df[cols], unit_value) + + kind <- roles$kind[match(cols, roles$col)] + action <- roles$action[match(cols, roles$col)] + new_unit <- list() + maps <- list() + + num <- cols[kind %in% .numeric_kinds()] + if (length(num)) { + x_unit <- as.data.frame(ut[num], stringsAsFactors = FALSE) + names(x_unit) <- num + x_new <- synthesise_numeric_local(x_unit) + for (cl in num) { + new_unit[[cl]] <- if (isTRUE(jitter)) { + synthesise_numeric_collaborate(x_obs = x_unit[[cl]], x_new = x_new[[cl]]) + } else { + x_new[[cl]] + } + } + } + for (cl in setdiff(cols, num)) { + new_unit[[cl]] <- synthesise_categorical_local(ut[[cl]]) + } + + out <- list() + for (cl in cols) { + x <- df[[cl]] + obs <- !is.na(x) + x[obs] <- new_unit[[cl]][row_unit[obs]] + if (identical(action[match(cl, cols)], "alias")) { + res <- alias_levels(x, prefix = paste0(cl, "_L")) + x <- res$x + maps[[cl]] <- res$map + } + out[[cl]] <- x + } + list(columns = out, level_maps = maps) +} diff --git a/R/mask.R b/R/mask.R index a202843..321814e 100644 --- a/R/mask.R +++ b/R/mask.R @@ -164,6 +164,30 @@ #' `masque_events_pooled` warning; when no other pattern has the gap free, #' `mask()` stops. No hazard model is fitted and no covariate-to-event #' relation is kept. +#' @param grain Single logical (default `TRUE`). Synthesise each column at +#' its **grain**: a scrambled covariate or outcome, or an aliased +#' categorical covariate, that holds at most one distinct non-missing value +#' within every unit of a design column, or of a pair of design columns +#' (`site:year`), is re-simulated once per unit and each unit's new value +#' is written to that unit's rows. The grain is the coarsest such unit with +#' at least two units and at least two rows per unit on average; a unit with +#' no value stays missing. Numeric columns of one grain are drawn jointly +#' through the copula on the unit-level table (with collaborate-mode +#' jitter against the unit-level values), and categorical and date columns +#' are permuted across units, then aliased where their action is `alias`. +#' The clone therefore keeps one value per unit, the frequencies of a +#' categorical across units, and the correlation among columns of the same +#' grain. It does **not** keep the correlation between a grained column +#' and row-level columns, and grained columns are not conditioned on +#' treatment when `conditional = TRUE`. In local mode the unit values are +#' drawn from the original unit values. In collaborate mode a grain with +#' fewer than five units is synthesised row by row instead, with a +#' `masque_grain_small_units` warning, because resampling that few real +#' values would disclose them. Each grain is recorded on the recipe +#' (`grain`) with a line in its warnings. `FALSE` synthesises every column +#' row by row, as masque 0.15 did; detected grains are still recorded. +#' Coordinates declared to `coords`, `events` columns and linked columns +#' are never grained. #' @param allow_unmasked_coords Single logical. A masked table must not carry #' a real coordinate, so by default `mask()` **stops** when a column whose #' name says coordinate (`gps`, `lat`, `lon`, `easting`, ...) would be @@ -206,6 +230,7 @@ mask <- function(df, ladder = c("hierarchy", "levels"), coords = NULL, events = NULL, + grain = TRUE, allow_unmasked_coords = FALSE, quiet = FALSE, .shared_maps = list(), @@ -246,6 +271,12 @@ mask <- function(df, class = c("masque_bad_conditional_refusal", "orchestra_refusal") ) } + if (!is.logical(grain) || length(grain) != 1L || is.na(grain)) { + cli::cli_abort( + "`grain` must be a single `TRUE` or `FALSE`.", + class = c("masque_bad_grain_refusal", "orchestra_refusal") + ) + } if (!is.character(ladder) || !length(ladder) || !all(ladder %in% c("hierarchy", "levels"))) { cli::cli_abort( @@ -348,6 +379,16 @@ mask <- function(df, .guard_unmasked_coords(df, roles, coord_cols, allow_unmasked_coords) warnings_acc <- c(warnings_acc, .check_row_floor(df, roles, mode, event_specs)) + # Columns constant within a design unit are synthesised once per unit. + event_cols <- unlist(lapply(event_specs, function(s) c(s$time, s$status))) + grain_res <- .resolve_grain( + .detect_grain( + df, roles, exclude = c(coord_cols, event_cols, names(.shared_maps)) + ), + grain, mode + ) + warnings_acc <- c(warnings_acc, grain_res$lines) + opts <- mode_defaults(mode) # With nothing to condition on, a conditional clone uses the global copula. @@ -370,7 +411,7 @@ mask <- function(df, seed, .mask_orchestrate( df, roles, mode, opts, .shared_maps, conditional, ladder, - events = event_specs + events = event_specs, grain = grain_res$groups ) ) synth <- result$synth @@ -539,6 +580,7 @@ mask <- function(df, cleaning = cleaning_rec, coords = coord_reports, events = event_reports, + grain = grain_res$groups, allow_unmasked_coords = isTRUE(allow_unmasked_coords), warnings = warnings_acc, integrity_fp = digest::digest(is.na(df), algo = "sha256") @@ -640,7 +682,7 @@ mask <- function(df, # column decides what happens to each column; mode only adds numeric jitter. .mask_orchestrate <- function(df, roles, mode, opts, shared_maps = list(), conditional = FALSE, ladder = "hierarchy", - events = list()) { + events = list(), grain = list()) { synth <- df level_maps <- list() warnings <- character() @@ -659,11 +701,13 @@ mask <- function(df, is_shared <- roles$col %in% shared_cols event_cols <- unlist(lapply(events, function(s) c(s$time, s$status))) is_event <- roles$col %in% event_cols + unit_groups <- Filter(function(g) identical(g$synthesised, "unit"), grain) + is_grain <- roles$col %in% unlist(lapply(unit_groups, `[[`, "cols")) # All scrambled numeric columns go through one Gaussian copula, fitted within # each treatment x design stratum when `conditional = TRUE`. num_idx <- which(action == "scramble" & kind %in% .numeric_kinds() & - !is_shared & !is_event) + !is_shared & !is_event & !is_grain) conditional_report <- NULL if (length(num_idx) >= 1L) { x_num <- df[, num_idx, drop = FALSE] @@ -741,7 +785,7 @@ mask <- function(df, perm_idx <- which( action == "scramble" & kind %in% c(.categorical_kinds(), .date_kinds()) & - role != "treatment" & !is_shared & !is_event + role != "treatment" & !is_shared & !is_event & !is_grain ) for (j in perm_idx) { col <- roles$col[j] @@ -788,7 +832,7 @@ mask <- function(df, } cov_alias_idx <- which(role == "covariate" & action == "alias" & - !is_shared) + !is_shared & !is_grain) for (j in cov_alias_idx) { col <- roles$col[j] perm <- synthesise_categorical_local(df[[col]]) @@ -797,6 +841,17 @@ mask <- function(df, level_maps[[col]] <- res$map } + # Grained columns: one draw per unit, broadcast to the unit's rows. + for (g in unit_groups) { + res <- .synthesise_grain_group( + df, g, roles, jitter = isTRUE(opts$jitter_numeric) + ) + for (col in names(res$columns)) { + synth[[col]] <- res$columns[[col]] + } + level_maps[names(res$level_maps)] <- res$level_maps + } + # Linked columns were aliased by the shared block and are never dropped. drop_idx <- which(action == "drop" & !is_shared) if (length(drop_idx)) { diff --git a/R/recipe.R b/R/recipe.R index fb5a371..dc71f36 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -71,6 +71,10 @@ masque_recipe <- S7::new_class( # One entry per linked-event spec: columns, stratification, pattern count # and the small patterns that were pooled. Empty before 0.16.0. events = S7::new_property(class = S7::class_list, default = list()), + # One entry per design unit holding grained columns: the unit, its + # columns, the unit count, and whether they were synthesised per "unit" + # or per "row" with the reason. Empty before 0.16.0. + grain = S7::new_property(class = S7::class_list, default = list()), # TRUE when the caller deliberately wrote a real coordinate through # unmasked. FALSE, and absent on a recipe written before 0.11.0. allow_unmasked_coords = S7::new_property( diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R new file mode 100644 index 0000000..0f5e53a --- /dev/null +++ b/tests/testthat/test-fix0160-grain.R @@ -0,0 +1,343 @@ +# A column constant within a design unit is synthesised once per unit and +# broadcast to the unit's rows, so the clone keeps its grain (F7). + +# 6 trials x 3 reps x 20 genotypes, blocks restarted within rep, a trial-level +# numeric (aet) and categorical (soil) covariate. +.fx_met <- function() { + withr::with_seed(1L, { + trials <- sprintf("T%d", 1:6) + gens <- sprintf("G%02d", 1:20) + g <- expand.grid( + genotype = gens, rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$block <- NA_character_ + for (t in trials) { + for (r in 1:3) { + idx <- which(g$trial == t & g$rep == r) + g$block[idx] <- sprintf("B%d", sample(rep(1:4, each = 5))) + } + } + trial_eff <- c(0.8, -0.5, 0.3, 1.1, -0.2, 0.0) + gen_eff <- stats::rnorm(20, 0, 0.3) + rep_eff <- c(0.1, 0, -0.1) + g$yield <- round(3 + trial_eff[match(g$trial, trials)] + + gen_eff[match(g$genotype, gens)] + rep_eff[g$rep] + + stats::rnorm(nrow(g), 0, 0.35), 2) + g$aet <- c(312.456, 287.901, 401.233, 355.678, 298.112, 376.543)[ + match(g$trial, trials) + ] + g$plot_cov <- round(stats::rnorm(nrow(g), 50, 8), 2) + g$soil <- c("clay", "loam", "sand", "clay", "loam", "sand")[ + match(g$trial, trials) + ] + g[, c("trial", "rep", "block", "genotype", "yield", "aet", "plot_cov", + "soil")] + }) +} + +.fx_met_roles <- function(df, mode) { + collab <- identical(mode, "collaborate") + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, c("trial", "rep", "block"), role = "design", + action = "keep") + r <- set_role(r, "genotype", role = "treatment", + action = if (collab) "alias" else "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, c("aet", "plot_cov"), role = "covariate", + action = "scramble") + r <- set_role(r, "soil", role = "covariate", + action = if (collab) "alias" else "scramble") + r +} + +# 3 sites x 2 years x 2 reps x 8 genotypes; rain is set per site-year, so it is +# constant within neither site nor year alone. +.fx_site_year <- function() { + withr::with_seed(9L, { + g <- expand.grid( + genotype = sprintf("G%d", 1:8), rep = 1:2, year = c(2024L, 2025L), + site = c("North", "South", "East"), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + sy <- paste(g$site, g$year) + g$rain <- c(410.5, 388.2, 502.7, 455.1, 361.9, 420.4)[ + match(sy, unique(sy)) + ] + g$yield <- round(stats::rnorm(nrow(g), 3, 0.4), 2) + g[, c("site", "year", "rep", "genotype", "yield", "rain")] + }) +} + +.fx_site_year_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, c("site", "year", "rep"), role = "design", + action = "keep") + r <- set_role(r, "genotype", role = "treatment", action = "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, "rain", role = "covariate", action = "scramble") + r +} + +# Text form of a clone: doubles to 15 significant digits, then the level maps. +.fx_canon <- function(m) { + syn <- as.data.frame(synthetic(m)) + cols <- vapply(names(syn), function(nm) { + x <- syn[[nm]] + v <- if (is.double(x) && !inherits(x, "Date")) { + sprintf("%.15g", x) + } else { + as.character(x) + } + paste(nm, paste(class(x), collapse = "/"), paste(v, collapse = ","), + sep = "|") + }, character(1)) + maps <- recipe(m)@level_maps + map_txt <- vapply(names(maps), function(nm) { + paste(nm, paste(names(maps[[nm]]), maps[[nm]], sep = "=", collapse = ","), + sep = "|") + }, character(1)) + digest::digest(paste(c(cols, map_txt), collapse = "\n"), algo = "sha256", + serialize = FALSE) +} + +.fx_distinct_by <- function(x, g) { + as.vector(tapply(x, g, function(v) length(unique(v[!is.na(v)])))) +} + +# One value per unit, in unit order. +.fx_unit_values <- function(x, g) { + as.vector(tapply(x, g, function(v) unique(v[!is.na(v)])[1L])) +} + +.fx_combos <- list( + list(mode = "local", cond = FALSE), list(mode = "local", cond = TRUE), + list(mode = "collaborate", cond = FALSE), + list(mode = "collaborate", cond = TRUE) +) + +# ---- detection ------------------------------------------------------------- + +test_that("the trial-level columns are found at the trial grain", { + df <- .fx_met() + g <- .detect_grain(df, .fx_met_roles(df, "local")) + expect_length(g, 1L) + expect_identical(g[[1]]$unit, "trial") + expect_identical(g[[1]]$cols, c("aet", "soil")) + expect_identical(g[[1]]$n_units, 6L) + # The alias action on a categorical covariate is a candidate too. + g_c <- .detect_grain(df, .fx_met_roles(df, "collaborate")) + expect_identical(g_c[[1]]$cols, c("aet", "soil")) +}) + +test_that("a nested design reports the coarsest unit", { + df <- .fx_met() + df$block <- paste(df$trial, df$rep, df$block, sep = "-") # unique labels + g <- .detect_grain(df, .fx_met_roles(df, "local")) + expect_identical(g[[1]]$unit, "trial") +}) + +test_that("a crossed site:year covariate is found on the interaction", { + df <- .fx_site_year() + roles <- .fx_site_year_roles(df, "local") + expect_false(all(.fx_distinct_by(df$rain, df$site) <= 1L)) + expect_false(all(.fx_distinct_by(df$rain, df$year) <= 1L)) + g <- .detect_grain(df, roles) + expect_length(g, 1L) + expect_identical(g[[1]]$unit, c("site", "year")) + expect_identical(g[[1]]$cols, "rain") + expect_identical(g[[1]]$n_units, 6L) +}) + +test_that("constant columns, unreplicated units and kept columns have no grain", { + df <- .fx_met() + df$const <- 7.5 + df$plot <- seq_len(nrow(df)) + roles <- .fx_met_roles(df, "local") + roles <- set_role(roles, "const", role = "covariate", action = "scramble") + roles <- set_role(roles, "plot", role = "design", action = "keep") + roles <- set_role(roles, "soil", action = "keep") + g <- .detect_grain(df, roles) + expect_identical(g[[1]]$cols, "aet") + # A unit with one row per level (plot) is not a grain even though every + # column is constant within it. + roles2 <- set_role(roles, c("trial", "rep", "block"), role = "covariate", + action = "keep") + expect_length(.detect_grain(df, roles2), 0L) +}) + +# ---- synthesis ------------------------------------------------------------- + +test_that("trial-level columns keep one value per trial in every mode", { + df <- .fx_met() + for (cb in .fx_combos) { + lab <- sprintf("%s cond=%s", cb$mode, cb$cond) + m <- suppressWarnings(mask( + df, .fx_met_roles(df, cb$mode), mode = cb$mode, seed = 1, + conditional = cb$cond + )) + syn <- synthetic(m) + expect_identical(.fx_distinct_by(syn$aet, df$trial), rep(1L, 6), label = lab) + expect_identical(.fx_distinct_by(syn$soil, df$trial), rep(1L, 6), + label = lab) + # Row-level columns still vary within trial. + expect_true(all(.fx_distinct_by(syn$plot_cov, df$trial) > 1L), label = lab) + # Soil's frequencies across trials are the original's. + expect_identical( + sort(as.vector(table(.fx_unit_values(syn$soil, df$trial)))), + sort(as.vector(table(.fx_unit_values(df$soil, df$trial)))), + label = lab + ) + # Design columns and the treatment incidence are untouched. + for (cn in c("trial", "rep", "block")) { + expect_identical(syn[[cn]], df[[cn]], label = paste(lab, cn)) + } + first_seen <- function(x) factor(x, levels = unique(x)) + expect_identical( + unname(unclass(table(first_seen(syn$genotype), syn$trial))), + unname(unclass(table(first_seen(df$genotype), df$trial))), + label = lab + ) + gr <- recipe(m)@grain + expect_length(gr, 1L) + expect_identical(gr[[1]]$synthesised, "unit", label = lab) + expect_identical(gr[[1]]$reason, "", label = lab) + expect_true(any(grepl("aet, soil synthesised once per trial (6 units)", + recipe(m)@warnings, fixed = TRUE)), label = lab) + } +}) + +test_that("local unit values are drawn from the original unit values", { + # Local numeric synthesis returns observed values (type 1 quantiles), so + # with six units the clone's trial values are the real ones, resampled. + df <- .fx_met() + orig <- .fx_unit_values(df$aet, df$trial) + for (s in 1:5) { + m <- suppressWarnings(mask( + df, .fx_met_roles(df, "local"), mode = "local", seed = s + )) + vals <- .fx_unit_values(synthetic(m)$aet, df$trial) + expect_true(all(vals %in% orig)) + } +}) + +test_that("collaborate unit values are not confined to the original values", { + df <- .fx_met() + orig <- .fx_unit_values(df$aet, df$trial) + m <- suppressWarnings(mask( + df, .fx_met_roles(df, "collaborate"), mode = "collaborate", seed = 1 + )) + vals <- .fx_unit_values(synthetic(m)$aet, df$trial) + expect_lt(mean(vals %in% orig), 1) + expect_true(all(vals >= min(orig) & vals <= max(orig))) + expect_identical(recipe(m)@grain[[1]]$n_units, 6L) + # Soil is aliased after the unit-level permutation. + expect_false(any(synthetic(m)$soil %in% df$soil)) + expect_identical(names(recipe(m)@level_maps$soil), sort(unique(df$soil))) +}) + +test_that("the NA mask and an all-missing unit survive", { + df <- .fx_met() + df$aet[df$trial == "T6"] <- NA # a unit with no value at all + df$aet[which(df$trial == "T1")[1:4]] <- NA + df$soil[which(df$trial == "T2")[1:3]] <- NA + roles <- .fx_met_roles(df, "collaborate") + g <- .detect_grain(df, roles) + expect_identical(g[[1]]$cols, c("aet", "soil")) + m <- suppressWarnings(mask(df, roles, mode = "collaborate", seed = 2)) + syn <- synthetic(m) + expect_identical(is.na(syn$aet), is.na(df$aet)) + expect_identical(is.na(syn$soil), is.na(df$soil)) + expect_true(all(.fx_distinct_by(syn$aet, df$trial) <= 1L)) + expect_identical(.fx_distinct_by(syn$aet, df$trial)[6], 0L) +}) + +test_that("a crossed site:year covariate keeps one value per site-year", { + df <- .fx_site_year() + sy <- paste(df$site, df$year) + for (mode in c("local", "collaborate")) { + m <- suppressWarnings(mask( + df, .fx_site_year_roles(df, mode), mode = mode, seed = 3 + )) + syn <- synthetic(m) + expect_identical(.fx_distinct_by(syn$rain, sy), rep(1L, 6), label = mode) + expect_identical(recipe(m)@grain[[1]]$unit, c("site", "year")) + expect_true(any(grepl("once per site:year", recipe(m)@warnings))) + } +}) + +test_that("fewer than five units in collaborate mode fall back to rows", { + df <- .fx_met() + df <- df[df$trial %in% c("T1", "T2", "T3", "T4"), ] + rownames(df) <- NULL + expect_warning( + m <- mask(df, .fx_met_roles(df, "collaborate"), mode = "collaborate", + seed = 1), + class = "masque_grain_small_units" + ) + gr <- recipe(m)@grain[[1]] + expect_identical(gr$synthesised, "row") + expect_identical(gr$n_units, 4L) + expect_match(gr$reason, "fewer than 5 units") + expect_true(any(.fx_distinct_by(synthetic(m)$aet, df$trial) > 1L)) + # Local mode proceeds at unit level and records the count. + expect_no_warning( + m_l <- mask(df, .fx_met_roles(df, "local"), mode = "local", seed = 1), + class = "masque_grain_small_units" + ) + expect_identical(recipe(m_l)@grain[[1]]$synthesised, "unit") + expect_identical(recipe(m_l)@grain[[1]]$n_units, 4L) + expect_identical(.fx_distinct_by(synthetic(m_l)$aet, df$trial), rep(1L, 4)) +}) + +test_that("grain = FALSE is the row-level clone of masque 0.15", { + # Digests of the same calls on the unmodified 0.16.0 base (12ead24). + base <- c( + local_FALSE = + "65154f4d80860124e54c67e3fcccc934d3757823dac158f5d2dda8ac98764601", + local_TRUE = + "789a764980d220b4924e1367dd6e47fe8db31c903d9b02196d5635390a689808", + collaborate_FALSE = + "bb8c3fa8180eb32a18887d517ca9e709b5049f8e5306bd6c5b6294b6fbbd2f39", + collaborate_TRUE = + "cf60a1627fc0b9aff9c81a6ffa1fe5c733e7605134d9b9ba0bd3e0e804fc106f" + ) + df <- .fx_met() + for (cb in .fx_combos) { + key <- sprintf("%s_%s", cb$mode, cb$cond) + m <- suppressWarnings(mask( + df, .fx_met_roles(df, cb$mode), mode = cb$mode, seed = 1L, + conditional = cb$cond, grain = FALSE + )) + gr <- recipe(m)@grain + expect_identical(gr[[1]]$synthesised, "row", label = key) + expect_identical(gr[[1]]$reason, "grain = FALSE", label = key) + skip_on_cran() + expect_identical(.fx_canon(m), base[[key]], label = key) + } +}) + +test_that("grain must be a single TRUE or FALSE", { + df <- .fx_met() + roles <- .fx_met_roles(df, "local") + for (bad in list(NA, "yes", c(TRUE, FALSE), 1)) { + expect_error( + suppressWarnings(mask(df, roles, seed = 1, grain = bad)), + class = "masque_bad_grain_refusal" + ) + } +}) + +test_that("a grained clone round-trips through apply_recipe() and unmask()", { + df <- .fx_met() + m <- suppressWarnings(mask( + df, .fx_met_roles(df, "collaborate"), mode = "collaborate", seed = 1 + )) + rec <- recipe(m) + fwd <- apply_recipe(df, rec) + expect_identical(sort(unique(fwd$soil)), sort(unique(synthetic(m)$soil))) + expect_identical(fwd$aet, df$aet) + back <- unmask(fwd, rec) + expect_identical(back$soil, df$soil) + expect_identical(back$genotype, df$genotype) +}) diff --git a/tests/testthat/test-fix0160-reproducibility.R b/tests/testthat/test-fix0160-reproducibility.R new file mode 100644 index 0000000..d8cfc89 --- /dev/null +++ b/tests/testthat/test-fix0160-reproducibility.R @@ -0,0 +1,88 @@ +# A table with no grained column must clone byte for byte as it did before +# grain synthesis: no extra draws, no reordering. + +.repro_fixture <- function() { + withr::with_seed(20261030L, { + n <- 72L + df <- data.frame( + trial = rep(sprintf("T%d", 1:4), each = 18L), + rep = rep(rep(1:3, each = 6L), 4L), + genotype = rep(sprintf("G%d", 1:6), 12L), + yield = round(stats::rnorm(n, 4, 0.6), 3), + height = round(stats::rnorm(n, 90, 7), 1), + count = stats::rpois(n, 12), + soil_n = round(stats::runif(n, 5, 40), 2), + texture = sample(c("clay", "loam", "sand"), n, TRUE), + sown = as.Date("2024-05-01") + sample(0:30, n, TRUE), + stringsAsFactors = FALSE + ) + df$height[c(3, 17, 40)] <- NA + df$texture[c(5, 60)] <- NA + df + }) +} + +.repro_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "genotype", role = "treatment", + action = if (mode == "collaborate") "alias" else "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, c("height", "count", "soil_n"), role = "covariate", + action = "scramble") + r <- set_role(r, "texture", role = "covariate", + action = if (mode == "collaborate") "alias" else "scramble") + r <- set_role(r, "sown", role = "date", action = "scramble") + r +} + +# Text form of a clone: 15 significant digits per double, so the digest is +# independent of serialisation format and of last-bit platform noise. +.repro_canon <- function(m) { + syn <- as.data.frame(synthetic(m)) + cols <- vapply(names(syn), function(nm) { + x <- syn[[nm]] + v <- if (is.double(x) && !inherits(x, "Date")) { + sprintf("%.15g", x) + } else { + as.character(x) + } + paste(nm, paste(class(x), collapse = "/"), paste(v, collapse = ","), + sep = "|") + }, character(1)) + maps <- recipe(m)@level_maps + map_txt <- vapply(names(maps), function(nm) { + paste(nm, paste(names(maps[[nm]]), maps[[nm]], sep = "=", collapse = ","), + sep = "|") + }, character(1)) + digest::digest(paste(c(cols, map_txt), collapse = "\n"), algo = "sha256", + serialize = FALSE) +} + +test_that("a table with no grained column clones as on the 0.16.0 base", { + # Digests of the same calls on the unmodified base (12ead24). + base <- c( + local_FALSE = + "ba60bae634f27d436a003f1ab22bc4146a77e64aea643d9716349d0325499097", + local_TRUE = + "dbecf2f681904bbef71d17f1169cddca7eb78ff67e85040611a038d0046c75c3", + collaborate_FALSE = + "f7e80803ea82d18e6647d2ef6d085eb1d3614ea74fe090aa7393007912b74ecc", + collaborate_TRUE = + "3290ab8bf490881ab9ec6fe683e1745b57ac9a76d3c082c51480a76cd07c3727" + ) + df <- .repro_fixture() + for (mode in c("local", "collaborate")) { + roles <- .repro_roles(df, mode) + expect_length(.detect_grain(df, roles), 0L) + for (cond in c(FALSE, TRUE)) { + key <- sprintf("%s_%s", mode, cond) + m <- suppressWarnings(suppressMessages(mask( + df, roles, mode = mode, seed = 1L, conditional = cond + ))) + expect_length(recipe(m)@grain, 0L) + skip_on_cran() + expect_identical(.repro_canon(m), base[[key]], label = key) + } + } +}) diff --git a/tests/testthat/test-masque-guided.R b/tests/testthat/test-masque-guided.R index e396656..220c075 100644 --- a/tests/testthat/test-masque-guided.R +++ b/tests/testthat/test-masque-guided.R @@ -43,7 +43,7 @@ test_that("masque() reads a folder as a set", { file.path(d, "t1.csv"), row.names = FALSE ) utils::write.csv( - data.frame(gen = c("x", "y"), w = c(1, 2)), + data.frame(gen = c("x", "y", "x", "y", "x"), w = c(1, 2, 4, 3, 5)), file.path(d, "t2.csv"), row.names = FALSE ) m <- masque(d, seed = 1, ask = FALSE, quiet = TRUE) From 9dba218aa2acba0443a3da069f4f894bff7e33c3 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:31:32 +1030 Subject: [PATCH 19/66] docs: several published coordinate jitters of one point average toward it One sentence in mask(coords =) and jitter_coordinates(seed =): keep the seed and publish one jittered release of a set of sites. Also shortens three internal comments. --- R/grain.R | 13 ++++--------- R/jitter_coordinates.R | 9 +++++---- R/mask.R | 4 +++- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/R/grain.R b/R/grain.R index 126dd39..9ad29e4 100644 --- a/R/grain.R +++ b/R/grain.R @@ -93,10 +93,8 @@ }) } -# Internal: decide per grain group whether it is synthesised per unit. With -# `grain = FALSE` every group is synthesised per row; in collaborate mode a -# group with fewer than .MIN_STRATUM units falls back to rows with a warning. -# Returns the groups and the recipe lines. +# Internal: mark each grain group "unit" or "row" (grain = FALSE, or a small +# collaborate grain, with a warning); returns the groups and recipe lines. .resolve_grain <- function(groups, grain, mode) { lines <- character() for (i in seq_along(groups)) { @@ -149,11 +147,8 @@ data.table::frankv(df[, unit, drop = FALSE], ties.method = "dense") } -# Internal: synthesise one grain group once per unit and broadcast to rows. -# Numeric columns go through the copula jointly (with collaborate jitter -# against the unit-level values), categorical and date columns are permuted -# across units, and aliased covariates are permuted, then aliased. Rows that -# were missing stay missing. Returns the new columns and any level maps. +# Internal: synthesise one grain group on its unit table and broadcast each +# unit's value to its non-missing rows; returns the columns and level maps. .synthesise_grain_group <- function(df, group, roles, jitter) { cols <- group$cols row_unit <- .unit_id(df, group$unit) diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index 8850a84..2e31ad0 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -111,7 +111,9 @@ #' A site that cannot be placed on land is set to `NA` on both axes, with a #' warning: masque never emits the true coordinate as though it were masked. #' @param seed Optional integer seed for reproducibility. The caller's RNG state -#' is preserved. +#' is preserved. Several published jitters of the same point average toward +#' it: keep the seed (a [mask()] recipe records it) and do not publish more +#' than one jittered release of the same sites. #' @param .group Internal. A pre-computed grouping vector of length #' `nrow(df)`, used by [mask()] to group by the *original* table's site #' structure. Takes precedence over `by`. Not for direct use. @@ -548,9 +550,8 @@ jitter_coordinates <- function(df, lat_col, lon_col, unique(unlist(lapply(specs, function(s) c(s$lat, s$lon)), use.names = FALSE)) } -# Declared coordinates are kept and coarsened. A column still on its proposed -# action yields to the declaration, with a note for the recipe; an explicit -# non-keep action, or one whose provenance is unknown, is refused. +# Declared coordinates get action keep; an explicit or unknown non-keep +# action is refused, a proposed default yields with a recipe note. .coords_take_keep <- function(roles, coord_cols) { idx <- which(roles$col %in% coord_cols & roles$action != "keep") if (!length(idx)) { diff --git a/R/mask.R b/R/mask.R index 321814e..8eb743c 100644 --- a/R/mask.R +++ b/R/mask.R @@ -141,7 +141,9 @@ #' identical input coordinate already identify one. A declared pair always #' survives masking, coarsened; the recipe records the parameters and the site #' grouping it was coarsened under, and [apply_recipe()] retargets to the real -#' coordinates. +#' coordinates. Several published jitters of the same point average toward +#' it: keep the seed, which the recipe records, and do not publish more than +#' one jittered release of the same sites. #' @param events Optional linked time-to-event outcomes. One spec per group of #' linked events, `list(time = c(...), status = c(...))`, with the time and #' status columns listed in the order the events occur; pass several specs From 504cdc3b20165ce49a448c27e05652082ff3c710 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:31:48 +1030 Subject: [PATCH 20/66] docs(detect): state the treatment-hint rule positively; shorter comments --- R/detect_candidates.R | 5 ++--- R/detect_design.R | 15 ++++++--------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/R/detect_candidates.R b/R/detect_candidates.R index eef9cae..d0a8eb8 100644 --- a/R/detect_candidates.R +++ b/R/detect_candidates.R @@ -140,9 +140,8 @@ out } -# Design columns whose levels are replicated units: design-pattern names and -# columns roled design, unless the roles table says otherwise, with at least -# two levels and two rows per level on average. +# Design-pattern names and columns roled design (the roles table wins) with at +# least two levels and two rows per level on average. .design_unit_cols <- function(df, roles = NULL, extra = character(0L)) { cols <- names(df) design <- cols[matches_pattern(cols, DESIGN_PATTERN)] diff --git a/R/detect_design.R b/R/detect_design.R index 40ef32c..9015524 100644 --- a/R/detect_design.R +++ b/R/detect_design.R @@ -29,10 +29,10 @@ #' agree on, or empty when they disagree; an environment column is never a #' block. `evidence$block_revision` records the change. #' -#' A treatment hint found by the rules rather than by a treatment-pattern -#' name is auto-applied only when the column is crossed with a design +#' For a column without a treatment-pattern name, a treatment hint from the +#' rules is auto-applied only when the column is crossed with a design #' column. A column with one value per level of a design column, or with an -#' outcome-pattern name, is a covariate and not a candidate. +#' outcome-pattern name, is a covariate and never a candidate. #' #' The detector never edits `df`. Its job is to recommend a role #' assignment, surface the evidence, and (optionally) draw a sanity @@ -153,9 +153,7 @@ detect_design <- function(df, } # Legacy whole-table detector. Keep this path free of environment inference so -# `env = FALSE` remains a stable compatibility escape hatch. `revise = TRUE` -# (scoped path only) applies the candidate revisions of .propose_candidates() -# and names both split-plot factors as the treatment. +# `env = FALSE` remains a stable compatibility escape hatch. .detect_design_legacy <- function(df, roles = NULL, interactive = FALSE, @@ -203,9 +201,8 @@ detect_design <- function(df, out } -# A treatment hint from the design rules stays auto-applied only for a -# treatment-pattern name, a user treatment, or a column crossed with a design -# column; otherwise auto_apply becomes FALSE and the reason says why. +# A rule's treatment hint keeps auto_apply for a treatment name, a user +# treatment or a column crossed with a design column; else the reason says why. .gate_treatment_hints <- function(out, df, roles) { rec <- out@recommended_roles hints <- which( From e3540e2c904461d1b3d4efbce125db0775135e9e Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:31:48 +1030 Subject: [PATCH 21/66] fix(roles): name the remedy for a numeric environment or design column (F22) The masque_environment_disclosure warning and the roles_validate() refusal of alias on a numeric design column both say: convert the column to a factor in the data and re-run propose_roles() (alias), or drop it. --- R/propose_roles.R | 13 +++++--- R/roles_actions.R | 4 ++- tests/testthat/test-fix0160-env-remedy.R | 39 ++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 tests/testthat/test-fix0160-env-remedy.R diff --git a/R/propose_roles.R b/R/propose_roles.R index d41f7ad..5a87733 100644 --- a/R/propose_roles.R +++ b/R/propose_roles.R @@ -92,10 +92,10 @@ #' stashed as `attr(roles, "design")`. Pass `detect = FALSE` for the #' name-only heuristic. #' -#' A structural treatment hint (one found by the design rules, not by a -#' treatment-pattern name) is applied only when the column is crossed with a -#' design column: a design-pattern name, a column roled `design`, or a -#' detected environment. A column with one value per level of a design +#' For a column without a treatment-pattern name, a treatment hint from the +#' design rules is applied only when the column is crossed with a design +#' column: a design-pattern name, a column roled `design`, or a detected +#' environment. A column with one value per level of a design #' column is a unit-level covariate and is never promoted. Hints that fail #' are listed in `attr(roles, "design")@recommended_roles` with #' `auto_apply = FALSE` and the reason. @@ -257,6 +257,11 @@ propose_roles <- function(df, mode = c("local", "collaborate"), "i" = paste0( "This preserves environment structure but may disclose year or other ", "numeric labels; review before release." + ), + "i" = paste0( + "To hide the values, convert the column to a factor in the data and ", + "re-run {.fun propose_roles}, which proposes {.val alias} for a ", + "categorical environment; or set its action to {.val drop}." ) ), class = "masque_environment_disclosure") invisible(NULL) diff --git a/R/roles_actions.R b/R/roles_actions.R index 7ae033a..38046e1 100644 --- a/R/roles_actions.R +++ b/R/roles_actions.R @@ -194,7 +194,9 @@ set_role <- function(roles, cols, role = NULL, action = NULL) { if (!(kind %in% .categorical_kinds())) { return(paste0( "design label aliasing requires a factor / character / logical ", - "column; numeric design columns can only be kept or dropped" + "column; numeric design columns can only be kept or dropped. To ", + "hide the labels, convert the column to a factor in the data, re-run ", + "propose_roles() and set action \"alias\", or drop it" )) } } diff --git a/tests/testthat/test-fix0160-env-remedy.R b/tests/testthat/test-fix0160-env-remedy.R new file mode 100644 index 0000000..fb2e327 --- /dev/null +++ b/tests/testthat/test-fix0160-env-remedy.R @@ -0,0 +1,39 @@ +# F22: the numeric-environment disclosure warning and the numeric design +# alias refusal both name the remedy. + +test_that("the numeric-environment warning names factor, alias and drop", { + d <- met_site_year_fixture() + cnd <- expect_warning( + propose_roles(d, mode = "collaborate"), + class = "masque_environment_disclosure" + ) + msg <- conditionMessage(cnd) + expect_match(msg, "year") + expect_match(msg, "convert") + expect_match(msg, "factor") + expect_match(msg, "propose_roles") + expect_match(msg, "alias") + expect_match(msg, "drop") +}) + +test_that("following the remedy removes the warning and aliases the year", { + d <- met_site_year_fixture() + d$year <- factor(d$year) + expect_no_warning(r <- propose_roles(d, mode = "collaborate")) + expect_equal(r$role[r$col == "year"], "design") + expect_equal(r$action[r$col == "year"], "alias") +}) + +test_that("aliasing a numeric design column is refused with the remedy", { + d <- met_site_year_fixture() + r <- suppressWarnings(propose_roles(d, mode = "collaborate")) + r$action[r$col == "year"] <- "alias" + err <- expect_error(roles_validate(r, d), "year") + msg <- conditionMessage(err) + # The old refusal already said "factor" and "dropped"; the remedy is new. + expect_match(msg, "convert") + expect_match(msg, "factor") + expect_match(msg, "propose_roles") + expect_match(msg, "alias") + expect_match(msg, "drop") +}) From 90916441a7987de874a9eaec780d91dc3f68f57f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:32:40 +1030 Subject: [PATCH 22/66] fix: grain covers per-unit date columns and never uses a dropped design unit A scrambled date-role column (a per-trial sowing date) is a grain candidate. Only retained design columns form units, so a dropped trial column is not broadcast back into the clone through its covariates. --- R/grain.R | 9 ++++++--- R/mask.R | 11 ++++++----- tests/testthat/test-fix0160-grain.R | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/R/grain.R b/R/grain.R index 9ad29e4..6f409c0 100644 --- a/R/grain.R +++ b/R/grain.R @@ -9,8 +9,10 @@ #' Detect the grain of the columns mask() re-simulates #' #' Candidates are scrambled covariate and outcome columns of numeric, -#' categorical or date kind, and aliased categorical covariates, less -#' `exclude`. Units are the design columns of `df`, singly and in pairs. A +#' categorical or date kind, scrambled date columns, and aliased categorical +#' covariates, less `exclude`. Units are the retained (not dropped) design +#' columns of `df`, singly and in pairs, since a dropped unit broadcast into +#' the clone would publish the grouping the plan removed. A #' column's grain is the unit with the fewest groups within which every group #' holds at most one distinct non-missing value, among units with at least two #' groups and at most `nrow(df) / 2`; ties go to fewer columns, then column @@ -28,12 +30,13 @@ cand <- roles$col[ ((role %in% c("covariate", "outcome") & action == "scramble" & kind %in% c(.numeric_kinds(), .categorical_kinds(), .date_kinds())) | + (role == "date" & action == "scramble" & kind %in% .date_kinds()) | (role == "covariate" & action == "alias" & kind %in% .categorical_kinds())) & !roles$col %in% exclude ] cand <- intersect(names(df), cand) - design <- intersect(names(df), roles$col[role == "design"]) + design <- intersect(names(df), roles$col[role == "design" & action != "drop"]) n <- nrow(df) if (!length(cand) || !length(design) || n < 4L) { return(list()) diff --git a/R/mask.R b/R/mask.R index 8eb743c..6ca0ebb 100644 --- a/R/mask.R +++ b/R/mask.R @@ -167,11 +167,12 @@ #' `mask()` stops. No hazard model is fitted and no covariate-to-event #' relation is kept. #' @param grain Single logical (default `TRUE`). Synthesise each column at -#' its **grain**: a scrambled covariate or outcome, or an aliased -#' categorical covariate, that holds at most one distinct non-missing value -#' within every unit of a design column, or of a pair of design columns -#' (`site:year`), is re-simulated once per unit and each unit's new value -#' is written to that unit's rows. The grain is the coarsest such unit with +#' its **grain**: a scrambled covariate, outcome or date column, or an +#' aliased categorical covariate, that holds at most one distinct +#' non-missing value within every unit of a retained design column, or of a +#' pair of them (`site:year`), is re-simulated once per unit and each unit's +#' new value is written to that unit's rows. The grain is the coarsest such +#' unit with #' at least two units and at least two rows per unit on average; a unit with #' no value stays missing. Numeric columns of one grain are drawn jointly #' through the copula on the unit-level table (with collaborate-mode diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R index 0f5e53a..e1116d0 100644 --- a/tests/testthat/test-fix0160-grain.R +++ b/tests/testthat/test-fix0160-grain.R @@ -341,3 +341,25 @@ test_that("a grained clone round-trips through apply_recipe() and unmask()", { expect_identical(back$soil, df$soil) expect_identical(back$genotype, df$genotype) }) + +test_that("a per-trial date is synthesised once per trial", { + df <- .fx_met() + sown <- as.Date("2025-05-01") + c(3L, 11L, 0L, 20L, 7L, 15L) + df$sown <- sown[match(df$trial, sprintf("T%d", 1:6))] + roles <- set_role(.fx_met_roles(df, "local"), "sown", role = "date", + action = "scramble") + expect_identical(.detect_grain(df, roles)[[1]]$cols, c("aet", "soil", "sown")) + m <- suppressWarnings(mask(df, roles, mode = "local", seed = 4)) + syn <- synthetic(m) + expect_s3_class(syn$sown, "Date") + expect_identical(.fx_distinct_by(syn$sown, df$trial), rep(1L, 6)) + expect_setequal(syn$sown, sown) +}) + +test_that("a dropped design column is not a unit", { + df <- .fx_met() + roles <- set_role(.fx_met_roles(df, "local"), "trial", action = "drop") + expect_length(.detect_grain(df, roles), 0L) + m <- suppressWarnings(mask(df, roles, mode = "local", seed = 1)) + expect_length(recipe(m)@grain, 0L) +}) From 6041b23d13433bd46081a85a5d5d3a35869a91c5 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:33:02 +1030 Subject: [PATCH 23/66] style: wrap four lines past 80 characters --- R/grain.R | 4 +++- R/mask.R | 4 +++- tests/testthat/test-fix0160-grain.R | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/R/grain.R b/R/grain.R index 6f409c0..4a35b75 100644 --- a/R/grain.R +++ b/R/grain.R @@ -175,7 +175,9 @@ x_new <- synthesise_numeric_local(x_unit) for (cl in num) { new_unit[[cl]] <- if (isTRUE(jitter)) { - synthesise_numeric_collaborate(x_obs = x_unit[[cl]], x_new = x_new[[cl]]) + synthesise_numeric_collaborate( + x_obs = x_unit[[cl]], x_new = x_new[[cl]] + ) } else { x_new[[cl]] } diff --git a/R/mask.R b/R/mask.R index 6ca0ebb..258dea4 100644 --- a/R/mask.R +++ b/R/mask.R @@ -380,7 +380,9 @@ mask <- function(df, ) } .guard_unmasked_coords(df, roles, coord_cols, allow_unmasked_coords) - warnings_acc <- c(warnings_acc, .check_row_floor(df, roles, mode, event_specs)) + warnings_acc <- c( + warnings_acc, .check_row_floor(df, roles, mode, event_specs) + ) # Columns constant within a design unit are synthesised once per unit. event_cols <- unlist(lapply(event_specs, function(s) c(s$time, s$status))) diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R index e1116d0..adcefb9 100644 --- a/tests/testthat/test-fix0160-grain.R +++ b/tests/testthat/test-fix0160-grain.R @@ -149,7 +149,7 @@ test_that("a crossed site:year covariate is found on the interaction", { expect_identical(g[[1]]$n_units, 6L) }) -test_that("constant columns, unreplicated units and kept columns have no grain", { +test_that("constant, unreplicated and kept columns have no grain", { df <- .fx_met() df$const <- 7.5 df$plot <- seq_len(nrow(df)) @@ -177,7 +177,8 @@ test_that("trial-level columns keep one value per trial in every mode", { conditional = cb$cond )) syn <- synthetic(m) - expect_identical(.fx_distinct_by(syn$aet, df$trial), rep(1L, 6), label = lab) + expect_identical(.fx_distinct_by(syn$aet, df$trial), rep(1L, 6), + label = lab) expect_identical(.fx_distinct_by(syn$soil, df$trial), rep(1L, 6), label = lab) # Row-level columns still vary within trial. From 54674b3dc7558545231401fe65b26101c227424b Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:33:43 +1030 Subject: [PATCH 24/66] test: a lone design column is no longer a kept treatment (F18 follow-up) On base, a table whose only factor was Rep had Rep promoted to a kept treatment by the CRD rule, and the audit graded that passthrough HIGH. Rep now stays a design column (graded low): the jitter tests no longer expect a HIGH warning and the blocked summary counts 1 HIGH. --- tests/testthat/test-fix0160-promotion.R | 4 +++- tests/testthat/test-safety-gate.R | 6 +++--- .../testthat/test-synthesise_numeric_jitter.R | 20 ++++--------------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/testthat/test-fix0160-promotion.R b/tests/testthat/test-fix0160-promotion.R index ea927ca..ab5c899 100644 --- a/tests/testthat/test-fix0160-promotion.R +++ b/tests/testthat/test-fix0160-promotion.R @@ -26,7 +26,9 @@ blocks <- lapply(1:3, function(b) { data.frame( block = b, - genotype = sample(c(tests[(b - 1L) * 6L + 1:6], rep(c("CHK1", "CHK2"), 2))), + genotype = sample( + c(tests[(b - 1L) * 6L + 1:6], rep(c("CHK1", "CHK2"), 2)) + ), stringsAsFactors = FALSE ) }) diff --git a/tests/testthat/test-safety-gate.R b/tests/testthat/test-safety-gate.R index 988c1bd..07b95d4 100644 --- a/tests/testthat/test-safety-gate.R +++ b/tests/testthat/test-safety-gate.R @@ -134,9 +134,9 @@ test_that("blocked summary is status-first and avoids release language", { )) ) expect_true(any(grepl("BLOCKED", msgs))) - # The fixture trips two HIGH rules: the retained PII-pattern column and - # the byte-identical Rep design column. - expect_true(any(grepl("2 HIGH", msgs))) + # The fixture trips one HIGH rule, the retained PII-pattern column; Rep is + # a kept design column, graded low. + expect_true(any(grepl("1 HIGH", msgs))) expect_false(any(grepl("\\bshare\\b|\\bsend\\b|\\bsafe\\b", msgs, ignore.case = TRUE ))) diff --git a/tests/testthat/test-synthesise_numeric_jitter.R b/tests/testthat/test-synthesise_numeric_jitter.R index 549a7e4..61af4fb 100644 --- a/tests/testthat/test-synthesise_numeric_jitter.R +++ b/tests/testthat/test-synthesise_numeric_jitter.R @@ -73,12 +73,8 @@ test_that("Collaborate mode lowers numeric exact-match in mask()", { r <- propose_roles(df) r$role[r$col == "yield"] <- "outcome" - # audit_mask() flags Rep (integer design column) HIGH in collaborate mode. - # The assignment sits inside expect_warning() so `m` is the masque object. - expect_warning( - m <- mask(df, r, mode = "collaborate", seed = 1), - "HIGH leakage" - ) + # Rep stays a kept design column (graded low), so no HIGH warning. + m <- mask(df, r, mode = "collaborate", seed = 1) audit <- audit_mask(m, print = FALSE) # Continuous outcome: exact_match_pct < 1 @@ -122,11 +118,7 @@ test_that("Mask() RNG hygiene holds with jitter in collaborate", { # initialised internal state); the test is specifically about mask(). set.seed(999) before <- .Random.seed - # Expected HIGH-leakage warning on Rep; see jitter test above. - expect_warning( - m <- mask(df, r, mode = "collaborate", seed = 42), - "HIGH leakage" - ) + m <- mask(df, r, mode = "collaborate", seed = 42) after <- .Random.seed expect_identical(before, after) }) @@ -144,11 +136,7 @@ test_that("Collaborate mode preserves integer storage on int outcome columns", { r <- propose_roles(df) r$role[r$col == "grain_count"] <- "outcome" - # Expected HIGH-leakage warning on Rep; see jitter test above. - expect_warning( - m <- mask(df, r, mode = "collaborate", seed = 1), - "HIGH leakage" - ) + m <- mask(df, r, mode = "collaborate", seed = 1) expect_type(synthetic(m)$grain_count, "integer") rng <- range(synthetic(m)$grain_count, na.rm = TRUE) expect_gte(rng[1], min(df$grain_count)) From d0eead0df334dc350d901813cb80d201ed4f60d9 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:33:58 +1030 Subject: [PATCH 25/66] fix: front door inherits the plan's mode, forwards arguments, sets info aside masque() without mode lets mask() and mask_set() inherit the mode of the supplied roles, so a collaborate plan is no longer downgraded to local. masque() gains detect, sheets, info and ... (forwarded to mask() or mask_set()); mask_set() gains sheets, info and refuses unused arguments with masque_unused_arg_refusal. Tables named in info are set aside before masking, never written, and only their names are kept on the recipe bundle; a table that looks like documentation raises a masque_info_hint message. read_set() refuses sheets on anything but a workbook. The guided summary counts every planned column, as the audit does, and names the dropped ones; the non-interactive notice no longer advises ask = TRUE when it was passed. --- R/mask_set.R | 132 ++++++++++++++- R/masque.R | 161 +++++++++++++------ R/read_set.R | 15 +- R/recipe_set.R | 3 +- tests/testthat/test-fix0160-front-door.R | 146 +++++++++++++++++ tests/testthat/test-fix0160-info-sheets.R | 186 ++++++++++++++++++++++ 6 files changed, 585 insertions(+), 58 deletions(-) create mode 100644 tests/testthat/test-fix0160-info-sheets.R diff --git a/R/mask_set.R b/R/mask_set.R index d472a6a..e27cfb2 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -47,7 +47,20 @@ #' [mask()] for the full account. #' @param ladder Drop order of the conditioning ladder, `"hierarchy"` (default) #' or `"levels"`; passed through to each [mask()] call. -#' @param quiet Logical; suppress the link / hygiene report. +#' @param quiet Logical; suppress the link / hygiene report, including the +#' hint that a table looks like documentation. +#' @param sheets For an Excel workbook, the sheets to read (default: all); +#' passed to [read_set()]. +#' @param info Optional character vector naming tables (sheet names or file +#' names without extension) that hold documentation rather than data, +#' such as a data dictionary or a read-me. They are set aside before +#' masking: not masked, not in the synthetic set, not written by +#' [write_set()], and their content is not kept. Their names are +#' recorded on the recipe bundle. A table that looks like documentation +#' and is not named here raises a hint (a message of class +#' `masque_info_hint`); the hint decides nothing. +#' @param ... Must be empty. A misspelled argument is refused +#' (`masque_unused_arg_refusal`) rather than ignored. #' #' @return A `masque_set` S7 object. Use [synthetic()] for the named list #' of synthetic tables and [recipe()] for the private recipe bundle. @@ -79,7 +92,11 @@ mask_set <- function(input, alias_names = FALSE, conditional = FALSE, ladder = c("hierarchy", "levels"), - quiet = FALSE) { + quiet = FALSE, + sheets = NULL, + info = NULL, + ...) { + .refuse_unused_dots("mask_set", ...) withr::local_preserve_seed() ladder <- match.arg(ladder) # Check the shape of `roles` before reading its mode, so a wrong argument @@ -122,9 +139,20 @@ mask_set <- function(input, # A table given as a conform_table() result was cleaned and conformed # already; its own record is used and its decisions go on its recipe. split <- .split_conformance(input) - conformed <- split$conformed - tables <- read_set(split$input) + tables <- read_set(split$input, sheets = sheets) + + # Documentation tables are set aside before anything reads their content; + # only their names are kept. + info <- .check_info(info, names(tables)) + tables <- tables[setdiff(names(tables), info)] + conformed <- split$conformed[setdiff(names(split$conformed), info)] table_names <- names(tables) + if (!quiet) { + if (length(info)) { + cli::cli_alert_info("Set aside, not masked: {.val {info}}.") + } + .hint_info(tables) + } # Hygiene per table (names legalised, whitespace trimmed) up front, so # link detection and roles see the cleaned schema. Each table's record goes @@ -203,7 +231,8 @@ mask_set <- function(input, created_at = Sys.time(), mode = mode, recipes = recipes, - links = link_record + links = link_record, + info = info ) masque_set( @@ -214,6 +243,99 @@ mask_set <- function(input, ) } +# The same refusal mask() gives: a misspelled argument must not vanish. +.refuse_unused_dots <- function(fn, ...) { + if (!...length()) { + return(invisible()) + } + dot_names <- names(list(...)) + dot_names <- dot_names[nzchar(dot_names)] + unused <- if (length(dot_names)) { + paste0(" ", paste0("`", dot_names, "`", collapse = ", ")) + } else { + "" + } + label <- if (...length() == 1L) "argument" else "arguments" + cli::cli_abort(c( + paste0("Unused ", label, unused, " passed to `", fn, "()`."), + i = "Check for misspelled argument names." + ), class = c("masque_unused_arg_refusal", "orchestra_refusal")) +} + +# `info` names tables to set aside: each must be in the set, and at least one +# table must remain to mask. +.check_info <- function(info, table_names) { + if (is.null(info)) { + return(character()) + } + if (!is.character(info) || anyNA(info)) { + cli::cli_abort( + "{.arg info} must be a character vector of table names.", + class = c("masque_bad_info_refusal", "orchestra_refusal") + ) + } + unknown <- setdiff(info, table_names) + if (length(unknown)) { + cli::cli_abort(c( + paste0( + "{.arg info} names {cli::qty(length(unknown))}table{?s} not in ", + "the set: {.val {unknown}}." + ), + i = "Tables: {.val {table_names}}." + ), class = c("masque_bad_info_refusal", "orchestra_refusal")) + } + if (!length(setdiff(table_names, info))) { + cli::cli_abort( + "{.arg info} sets aside every table; nothing is left to mask.", + class = c("masque_bad_info_refusal", "orchestra_refusal") + ) + } + unique(info) +} + +# Hygiene hint, never a decision: a table named like documentation, or of at +# most three columns with one of long free text, is offered to `info =`. +.INFO_NAME_PATTERN <- paste0( + "dictionar|read[ ._-]?me|notes|info|meta[ ._-]?data|code[ ._-]?book" +) + +.hint_info <- function(tables) { + looks <- vapply(names(tables), function(nm) { + .looks_like_info(nm, tables[[nm]]) + }, logical(1L)) + hits <- names(tables)[looks] + if (!length(hits)) { + return(invisible()) + } + cli::cli_inform(c( + i = paste0( + "{cli::qty(length(hits))}Table{?s} {.val {hits}} look{?s/} like ", + "documentation rather than data." + ), + " " = paste0( + "Name {cli::qty(length(hits))}{?it/them} in {.arg info} to set ", + "{?it/them} aside: not masked and not written." + ) + ), class = "masque_info_hint") +} + +.looks_like_info <- function(nm, tab) { + if (grepl(.INFO_NAME_PATTERN, nm, ignore.case = TRUE)) { + return(TRUE) + } + if (ncol(tab) > 3L) { + return(FALSE) + } + long_text <- vapply(tab, function(x) { + if (!(is.character(x) || is.factor(x))) { + return(FALSE) + } + x <- as.character(x[!is.na(x)]) + length(x) > 0L && stats::median(nchar(x)) >= 25 + }, logical(1L)) + any(long_text) +} + # Pull conform_table() results out of a list input: `input` with each one # replaced by its data, and the conformance objects by table name. .split_conformance <- function(input) { diff --git a/R/masque.R b/R/masque.R index 9f90fd0..24aa2bf 100644 --- a/R/masque.R +++ b/R/masque.R @@ -49,7 +49,13 @@ #' @param out Optional output path. For a single table, a `.csv` file (or #' `.xlsx` with `writexl`). For a set, a folder (one CSV per table) or #' an `.xlsx` workbook. When `NULL` (default) nothing is written. -#' @param mode Either `"local"` (default) or `"collaborate"`. +#' @param mode Either `"local"` or `"collaborate"`. `"local"` makes an +#' owner-only development copy (real labels may remain); `"collaborate"` +#' makes a copy meant to be shared, with labels aliased, numbers jittered, +#' free text dropped and the leakage audit run. When omitted and `roles` +#' are supplied, the mode the roles were proposed for is used, exactly as +#' in [mask()] and [mask_set()]; with no `roles`, the proposal is made for +#' `"local"`. #' @param seed Optional integer for reproducibility. #' @param clean Hygiene mode passed to [clean_table()] (`"auto"`, #' `"report"`, `"off"`). @@ -72,6 +78,18 @@ #' refused. Pass `TRUE` to write anyway after your own review; the #' override raises a `masque_high_override` warning and is recorded in #' the recipe's warnings, so the exception stays auditable. +#' @param detect Logical (default `TRUE`). Passed to [propose_roles()] when +#' roles are proposed: `FALSE` skips design detection and proposes from +#' column names and types alone. +#' @param sheets For an Excel workbook, the sheets to read (default: all). +#' See [mask_set()]. +#' @param info For a set, tables that hold documentation rather than data +#' (a data dictionary, a read-me): set aside, not masked and not written. +#' See [mask_set()]. +#' @param ... Further arguments for the masking verb: [mask()] for one +#' table (for example `coords`, `events`, `allow_unmasked_coords`), +#' [mask_set()] for a set (for example `links`). An argument the verb +#' does not take is refused (`masque_unused_arg_refusal`). #' #' @return A `masque` object (single-table input) or a `masque_set` #' object (set input), invisibly. Use [synthetic()] and [recipe()]. @@ -84,7 +102,8 @@ #' head(synthetic(m)) #' #' @seealso [mask()], [mask_set()], [propose_roles()], [set_role()], -#' [write_set()], [save_recipe()]. +#' [apply_recipe()] and [unmask()] for the round trip, [save_recipe()] to +#' keep the recipe, [write_set()]. #' @export masque <- function(input, roles = NULL, @@ -98,22 +117,33 @@ masque <- function(input, ask = interactive(), overwrite = FALSE, quiet = FALSE, - allow_high = FALSE) { - mode <- match.arg(mode) + allow_high = FALSE, + detect = TRUE, + sheets = NULL, + info = NULL, + ...) { + # NULL: let the verb inherit the mode of the supplied roles. + mode <- if (missing(mode)) NULL else match.arg(mode) clean <- match.arg(clean) ladder <- match.arg(ladder) is_set <- .masque_is_set(input) + if (!is_set && (!is.null(sheets) || !is.null(info))) { + cli::cli_abort(c( + "{.arg sheets} and {.arg info} apply to a workbook, folder or list.", + i = "A single table has no sheets or tables to set aside." + ), class = c("masque_bad_info_refusal", "orchestra_refusal")) + } if (is_set) { m <- .masque_guided_set( input, roles, mode, seed, clean, alias_names, conditional, ladder, - ask, quiet + ask, quiet, detect, sheets, info, ... ) } else { m <- .masque_guided_one( input, roles, mode, seed, clean, alias_names, conditional, ladder, - ask, quiet + ask, quiet, detect, ... ) } @@ -148,59 +178,60 @@ masque <- function(input, )) } +# Call a masking verb with `mode` only when one was chosen, so the verb's own +# inheritance from the roles applies otherwise. +.with_mode <- function(verb, mode, ...) { + if (is.null(mode)) verb(...) else verb(..., mode = mode) +} + .masque_guided_one <- function(input, roles, mode, seed, clean, - alias_names, conditional, ladder, ask, quiet) { + alias_names, conditional, ladder, ask, quiet, + detect, ...) { df <- if (is.data.frame(input) || inherits(input, "masque_conformance")) { input } else { .read_one_file(input) } - if (!is.null(roles)) { - return(mask( - df, roles, mode = mode, seed = seed, clean = clean, - alias_names = alias_names, conditional = conditional, ladder = ladder - )) + if (is.null(roles)) { + tab <- if (inherits(df, "masque_conformance")) df$data else df + roles <- propose_roles(tab, mode = mode %||% "local", detect = detect) + roles <- .masque_review(roles, ask, quiet, label = NULL) } - - proposed <- propose_roles( - if (inherits(df, "masque_conformance")) df$data else df, mode = mode - ) - proposed <- .masque_review(proposed, ask, quiet, label = NULL) - mask( - df, proposed, mode = mode, seed = seed, clean = clean, - alias_names = alias_names, conditional = conditional, ladder = ladder + .with_mode( + mask, mode, df, roles, seed = seed, clean = clean, + alias_names = alias_names, conditional = conditional, ladder = ladder, + ... ) } .masque_guided_set <- function(input, roles, mode, seed, clean, - alias_names, conditional, ladder, ask, quiet) { - if (!is.null(roles)) { - return(mask_set( - input, roles = roles, mode = mode, seed = seed, clean = clean, - alias_names = alias_names, conditional = conditional, ladder = ladder, - quiet = quiet - )) + alias_names, conditional, ladder, ask, quiet, + detect, sheets, info, ...) { + if (is.null(roles)) { + # Propose on the names mask_set() will mask under. It cleans again and + # raises the rename warning once, so the warning is held back here. + split <- .split_conformance(input) + tables <- read_set(split$input, sheets = sheets) + to_mask <- setdiff(names(tables), info) + roles <- lapply(to_mask, function(nm) { + cl <- withCallingHandlers( + clean_table(tables[[nm]], clean = clean, quiet = TRUE), + masque_name_repaired = function(w) invokeRestart("muffleWarning") + ) + r <- propose_roles(cl$data, mode = mode %||% "local", detect = detect) + .masque_review(r, ask, quiet, label = nm) + }) + names(roles) <- to_mask + tables[names(split$conformed)] <- split$conformed + # Read once: the tables go on, not the path. + input <- tables + sheets <- NULL } - # Propose on the names mask_set() will mask under. It cleans again and - # raises the rename warning once, so the warning is held back here. - split <- .split_conformance(input) - tables <- read_set(split$input) - proposed <- lapply(names(tables), function(nm) { - cl <- withCallingHandlers( - clean_table(tables[[nm]], clean = clean, quiet = TRUE), - masque_name_repaired = function(w) invokeRestart("muffleWarning") - ) - r <- propose_roles(cl$data, mode = mode) - .masque_review(r, ask, quiet, label = nm) - }) - names(proposed) <- names(tables) - tables[names(split$conformed)] <- split$conformed - - mask_set( - tables, roles = proposed, mode = mode, seed = seed, clean = clean, + .with_mode( + mask_set, mode, input, roles = roles, seed = seed, clean = clean, alias_names = alias_names, conditional = conditional, ladder = ladder, - quiet = quiet + quiet = quiet, sheets = sheets, info = info, ... ) } @@ -210,10 +241,15 @@ masque <- function(input, if (!isTRUE(ask) || !interactive()) { if (!quiet) { where <- if (is.null(label)) "" else sprintf(" for {.field {label}}") - cli::cli_alert_info(paste0( - "Using the proposed masking plan", where, + how <- if (isTRUE(ask)) { + paste0( + " ({.code ask = TRUE} needs an interactive session; pass ", + "{.arg roles} to review the plan in a script)." + ) + } else { " (pass {.arg roles} or set {.code ask = TRUE} to review)." - )) + } + cli::cli_alert_info(paste0("Using the proposed masking plan", where, how)) } return(roles) } @@ -391,11 +427,24 @@ masque <- function(input, } .masque_summary <- function(m) { + # Count every column of the plan, as the audit does, and name the dropped. if (S7::S7_inherits(m, masque_set)) { - n <- length(m@synthetic) - headline <- "Masked {n} table{?s} in {.val {m@mode}} mode" + recs <- m@recipe@recipes + n <- length(recs) + n_cols <- sum(vapply(recs, function(r) nrow(r@roles), integer(1L))) + dropped <- unlist(lapply(names(recs), function(nm) { + cols <- .recipe_dropped_cols(recs[[nm]]) + if (length(cols)) paste0(nm, ":", cols) else character() + })) + info <- m@recipe@info + headline <- paste0( + "Masked {n} table{?s} ({n_cols} column{?s}) in {.val {m@mode}} mode" + ) } else { - headline <- "Masked {ncol(synthetic(m))} column{?s} in {.val {m@mode}} mode" + n_cols <- nrow(recipe(m)@roles) + dropped <- .recipe_dropped_cols(recipe(m)) + info <- character() + headline <- "Masked {n_cols} column{?s} in {.val {m@mode}} mode" } status <- .release_status(m) @@ -411,6 +460,16 @@ masque <- function(input, } else { cli::cli_alert_success(paste0(headline, ".")) } + if (length(dropped)) { + cli::cli_alert_info( + "{length(dropped)} dropped, not in the clone: {.field {dropped}}." + ) + } + if (length(info)) { + cli::cli_alert_info( + "Set aside, not masked and not written: {.val {info}}." + ) + } if (identical(status, "blocked")) { flagged <- paste(.audit_high_cols(m), collapse = ", ") diff --git a/R/read_set.R b/R/read_set.R index 0b10483..19ac41b 100644 --- a/R/read_set.R +++ b/R/read_set.R @@ -23,7 +23,8 @@ #' @param input A folder path, an `.xlsx` / `.xls` path, or a named list #' of data frames. #' @param sheets For an Excel workbook, an optional character vector of -#' sheet names to read (default: all sheets). +#' sheet names to read (default: all sheets). Any other input with +#' `sheets` set is refused (`masque_bad_sheets_refusal`). #' @param pattern For a folder, an optional regular expression to select #' files (default: all `.csv` / `.tsv` / `.fst`). #' @@ -39,6 +40,18 @@ #' @seealso [mask_set()], [write_set()]. #' @export read_set <- function(input, sheets = NULL, pattern = NULL) { + is_workbook <- is.character(input) && length(input) == 1L && + file.exists(input) && !dir.exists(input) && + tolower(tools::file_ext(input)) %in% c("xlsx", "xls") + if (!is.null(sheets) && !is_workbook) { + cli::cli_abort(c( + "{.arg sheets} selects sheets of an Excel workbook.", + i = paste0( + "For a folder use {.arg pattern}; to leave a table out of the ", + "masking, name it in {.arg info}." + ) + ), class = c("masque_bad_sheets_refusal", "orchestra_refusal")) + } if (is.list(input) && !is.data.frame(input)) { return(.validate_table_list(input)) } diff --git a/R/recipe_set.R b/R/recipe_set.R index 365e5c0..f2b948d 100644 --- a/R/recipe_set.R +++ b/R/recipe_set.R @@ -14,7 +14,8 @@ masque_recipe_set <- S7::new_class( created_at = S7::class_POSIXct, mode = S7::class_character, recipes = S7::new_property(class = S7::class_list, default = list()), - links = S7::new_property(class = S7::class_list, default = list()) + links = S7::new_property(class = S7::class_list, default = list()), + info = S7::new_property(class = S7::class_character, default = character()) ) ) diff --git a/tests/testthat/test-fix0160-front-door.R b/tests/testthat/test-fix0160-front-door.R index 043a427..c42609d 100644 --- a/tests/testthat/test-fix0160-front-door.R +++ b/tests/testthat/test-fix0160-front-door.R @@ -40,3 +40,149 @@ test_that("duplicated input column names give one classed refusal", { err <- expect_error(clean_table(d, quiet = TRUE), "\"a\"") expect_false(grepl("a_1", conditionMessage(err), fixed = TRUE)) }) + +door_trial <- function() { + set.seed(8) + g <- expand.grid( + genotype = c("Scope", "Compass", "Spartacus", "Trojan"), rep = 1:3, + site = c("north", "south", "east", "west"), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$plot_id <- sprintf("P%03d", seq_len(nrow(g))) + g$yield <- round(3 + stats::rnorm(nrow(g), 0, 0.4), 2) + g +} + +door_roles <- function(df, mode) { + r <- propose_roles(df, mode = mode, detect = FALSE) + r <- set_role(r, "genotype", role = "treatment") + r <- set_role(r, "rep", role = "design") + r <- set_role(r, "site", role = "design") + r <- set_role(r, "plot_id", role = "id", action = "drop") + set_role(r, "yield", role = "outcome") +} + +door_set <- function() { + df <- door_trial() + lines <- data.frame( + genotype = c("Scope", "Compass", "Spartacus", "Trojan"), + height = c(80, 95, 102, 88), stringsAsFactors = FALSE + ) + list(trial = df, lines = lines) +} + +door_set_roles <- function(tabs, mode) { + rl <- propose_roles(tabs$lines, mode = mode, detect = FALSE) + rl <- set_role(rl, "genotype", role = "treatment") + list(trial = door_roles(tabs$trial, mode), lines = rl) +} + +test_that("masque() inherits a collaborate plan's mode without `mode`", { + df <- door_trial() + r <- door_roles(df, "collaborate") + for (q in c(TRUE, FALSE)) { + expect_no_warning( + m <- suppressMessages(masque(df, roles = r, seed = 1, quiet = q)), + class = "masque_mode_downgrade" + ) + expect_identical(recipe(m)@mode, "collaborate", info = as.character(q)) + expect_false(any(df$genotype %in% as.character(synthetic(m)$genotype))) + } +}) + +test_that("masque() on a set inherits the plans' mode without `mode`", { + tabs <- door_set() + rl <- door_set_roles(tabs, "collaborate") + for (q in c(TRUE, FALSE)) { + expect_no_warning( + m <- suppressMessages(masque(tabs, roles = rl, seed = 1, quiet = q)), + class = "masque_mode_downgrade" + ) + expect_identical(m@mode, "collaborate", info = as.character(q)) + expect_false(any( + tabs$trial$genotype %in% as.character(synthetic(m)$trial$genotype) + )) + } +}) + +test_that("masque() passes `detect` to propose_roles()", { + real <- propose_roles + seen <- logical() + local_mocked_bindings(propose_roles = function(df, mode = "local", + detect = TRUE) { + seen <<- c(seen, detect) + real(df, mode = mode, detect = detect) + }) + masque(door_trial(), detect = FALSE, seed = 1, ask = FALSE, quiet = TRUE) + masque(door_set(), detect = FALSE, seed = 1, ask = FALSE, quiet = TRUE) + expect_length(seen, 3L) + expect_false(any(seen)) +}) + +test_that("masque() forwards the verb's arguments and refuses unknown ones", { + df <- door_trial() + r <- door_roles(df, "local") + m <- masque(df, roles = r, mode = "local", seed = 1, quiet = TRUE, + allow_unmasked_coords = TRUE + ) + expect_true(recipe(m)@allow_unmasked_coords) + + tabs <- door_set() + rl <- door_set_roles(tabs, "collaborate") + linked <- masque(tabs, roles = rl, seed = 1, quiet = TRUE) + expect_gt(length(recipe(linked)@links), 0L) + unlinked <- masque(tabs, roles = rl, seed = 1, quiet = TRUE, links = FALSE) + expect_length(recipe(unlinked)@links, 0L) + + expect_error( + masque(df, roles = r, mode = "local", quiet = TRUE, bogus = 1), + class = "masque_unused_arg_refusal" + ) + expect_error( + masque(tabs, roles = rl, quiet = TRUE, bogus = 1), + class = "masque_unused_arg_refusal" + ) + expect_error( + mask_set(tabs, roles = rl, quiet = TRUE, bogus = 1), + class = "masque_unused_arg_refusal" + ) +}) + +summary_counts <- function(msgs) { + head_line <- grep("Masked [0-9]+ column", msgs, value = TRUE) + audit <- regmatches(head_line, regexec( + "audit: ([0-9]+) HIGH, ([0-9]+) medium, ([0-9]+) low", head_line + ))[[1L]] + list( + masked = as.integer(sub(".*Masked ([0-9]+) column.*", "\\1", head_line)), + audited = sum(as.integer(audit[-1L])) + ) +} + +test_that("the guided summary counts columns as the audit does", { + df <- door_trial() + r <- door_roles(df, "collaborate") + msgs <- testthat::capture_messages( + masque(df, roles = r, mode = "collaborate", seed = 1) + ) + n <- summary_counts(msgs) + expect_identical(n$masked, n$audited) + expect_identical(n$masked, ncol(df)) + expect_true(any(grepl("dropped", msgs) & grepl("plot_id", msgs))) +}) + +test_that("the non-interactive notice does not advise the argument passed", { + skip_if(interactive()) + msgs <- testthat::capture_messages( + masque(door_trial(), mode = "local", seed = 1, ask = TRUE) + ) + notice <- grep("proposed masking plan", msgs, value = TRUE) + expect_length(notice, 1L) + expect_false(grepl("set `ask = TRUE`", notice, fixed = TRUE)) + expect_match(notice, "interactive") + + msgs_f <- testthat::capture_messages( + masque(door_trial(), mode = "local", seed = 1, ask = FALSE) + ) + expect_true(any(grepl("ask = TRUE", msgs_f, fixed = TRUE))) +}) diff --git a/tests/testthat/test-fix0160-info-sheets.R b/tests/testthat/test-fix0160-info-sheets.R new file mode 100644 index 0000000..4e7cf65 --- /dev/null +++ b/tests/testthat/test-fix0160-info-sheets.R @@ -0,0 +1,186 @@ +# Documentation sheets and files: `sheets =` chooses what is read, `info =` +# sets tables aside (not masked, not written), and a hint names likely ones. + +CONTACT <- "Contact: J. Smith, site agronomist" + +info_tables <- function() { + set.seed(4) + trial <- data.frame( + site = rep(c("north", "south", "east", "west"), each = 6), + genotype = rep(c("Scope", "Compass", "Mace"), 8), + yield = round(stats::rnorm(24, 3, 0.5), 2), + stringsAsFactors = FALSE + ) + dictionary <- data.frame( + Column = c("site", "genotype", "yield"), + Description = c( + CONTACT, "Entry name as sown in the trial plan", + "Grain yield, tonnes per hectare at 12.5% moisture" + ), + stringsAsFactors = FALSE + ) + list(trial = trial, Dictionary = dictionary) +} + +info_roles <- function(tabs, mode = "local") { + r <- propose_roles(tabs$trial, mode = mode, detect = FALSE) + r <- set_role(r, "site", role = "design") + r <- set_role(r, "genotype", role = "treatment") + list(trial = set_role(r, "yield", role = "outcome")) +} + +info_workbook <- function(tabs = info_tables()) { + path <- file.path(withr::local_tempdir(.local_envir = parent.frame()), + "trial.xlsx") + writexl::write_xlsx(tabs, path = path) + path +} + +info_folder <- function(tabs = info_tables()) { + d <- withr::local_tempdir(.local_envir = parent.frame()) + for (nm in names(tabs)) { + data.table::fwrite(tabs[[nm]], file.path(d, paste0(nm, ".csv"))) + } + d +} + +has_contact <- function(tables) { + any(vapply(tables, function(t) { + CONTACT %in% unlist(lapply(t, as.character)) + }, logical(1L))) +} + +test_that("info = sets a sheet aside on every set path", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + tabs <- info_tables() + path <- info_workbook(tabs) + d <- info_folder(tabs) + for (mode in c("local", "collaborate")) { + runs <- list( + masque(path, roles = info_roles(tabs, mode), mode = mode, seed = 1, + info = "Dictionary", quiet = TRUE + ), + masque(d, mode = mode, seed = 1, info = "Dictionary", ask = FALSE, + quiet = TRUE + ), + mask_set(tabs, roles = info_roles(tabs, mode), mode = mode, seed = 1, + info = "Dictionary", quiet = TRUE + ) + ) + for (m in runs) { + expect_named(synthetic(m), "trial") + expect_false(has_contact(synthetic(m)), info = mode) + expect_identical(recipe(m)@info, "Dictionary") + expect_named(recipe(m)@recipes, "trial") + } + } +}) + +test_that("a set-aside sheet is not written, and the raw set still applies", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + tabs <- info_tables() + path <- info_workbook(tabs) + out_dir <- file.path(withr::local_tempdir(), "masked") + m <- masque(path, roles = info_roles(tabs), mode = "local", seed = 1, + info = "Dictionary", out = out_dir, quiet = TRUE + ) + expect_identical(list.files(out_dir), "trial.csv") + + out_xl <- file.path(withr::local_tempdir(), "masked.xlsx") + masque(path, roles = info_roles(tabs), mode = "local", seed = 1, + info = "Dictionary", out = out_xl, quiet = TRUE + ) + expect_identical(readxl::excel_sheets(out_xl), "trial") + + fwd <- apply_recipe(read_set(path), recipe(m)) + expect_named(fwd, "trial") +}) + +test_that("sheets = reads only the named sheets", { + skip_if_not_installed("writexl") + skip_if_not_installed("readxl") + tabs <- info_tables() + path <- info_workbook(tabs) + m1 <- masque(path, roles = info_roles(tabs), mode = "local", seed = 1, + sheets = "trial", quiet = TRUE + ) + expect_named(synthetic(m1), "trial") + m2 <- mask_set(path, roles = info_roles(tabs), mode = "local", seed = 1, + sheets = "trial", quiet = TRUE + ) + expect_named(synthetic(m2), "trial") + expect_length(recipe(m2)@info, 0L) +}) + +test_that("info and sheets refuse what they cannot mean", { + tabs <- info_tables() + expect_error( + mask_set(tabs, info = "Readme", mode = "local", quiet = TRUE), + class = "masque_bad_info_refusal" + ) + expect_error( + mask_set(tabs, info = names(tabs), mode = "local", quiet = TRUE), + class = "masque_bad_info_refusal" + ) + expect_error( + masque(tabs$trial, info = "Dictionary", mode = "local", quiet = TRUE), + class = "masque_bad_info_refusal" + ) + expect_error( + mask_set(info_folder(tabs), sheets = "trial", mode = "local", + quiet = TRUE + ), + class = "masque_bad_sheets_refusal" + ) +}) + +# The Dictionary masked in local mode keeps every column, which mask() notes. +keep_all_ok <- function(expr) { + withCallingHandlers(expr, warning = function(w) { + if (grepl("nothing will be masked", conditionMessage(w), fixed = TRUE)) { + invokeRestart("muffleWarning") + } + }) +} + +test_that("a sheet that looks like documentation gets a hint, not a decision", { + tabs <- info_tables() + expect_message( + m <- keep_all_ok(mask_set(tabs, mode = "local", seed = 1)), + class = "masque_info_hint" + ) + # The hint decides nothing: the sheet was still masked. + expect_true("Dictionary" %in% names(synthetic(m))) + + # Named like documentation. + notes <- list(trial = tabs$trial, notes = data.frame(a = 1:6, b = 6:1)) + expect_message( + keep_all_ok(mask_set(notes, mode = "local", seed = 1)), + class = "masque_info_hint" + ) + + # No hint once the sheet is set aside, or for plain data tables. + msgs <- testthat::capture_messages( + mask_set(tabs, info = "Dictionary", mode = "local", seed = 1) + ) + expect_false(any(grepl("documentation", msgs))) + plain <- list(trial = tabs$trial, lines = data.frame( + genotype = c("Scope", "Compass", "Mace"), height = c(80, 95, 102) + )) + msgs <- testthat::capture_messages( + mask_set(plain, mode = "local", seed = 1) + ) + expect_false(any(grepl("documentation", msgs))) +}) + +test_that("the summary names what was set aside", { + tabs <- info_tables() + msgs <- testthat::capture_messages( + masque(tabs, roles = info_roles(tabs), mode = "local", seed = 1, + info = "Dictionary" + ) + ) + expect_true(any(grepl("Set aside", msgs) & grepl("Dictionary", msgs))) +}) From 7be67f9ed812a121b394143633fd812332493fb5 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:34:21 +1030 Subject: [PATCH 26/66] docs: plainer wording for info, dots and three internal comments --- R/clean_table.R | 5 ++--- R/conform_table.R | 5 ++--- R/mask_set.R | 16 ++++++++-------- R/masque.R | 5 ++--- R/recipe.R | 5 ++--- 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/R/clean_table.R b/R/clean_table.R index ad6403b..3be9715 100644 --- a/R/clean_table.R +++ b/R/clean_table.R @@ -347,9 +347,8 @@ trimws_keep_class <- function(x) { roles } -# Line a roles table up with the cleaned names. A table already keyed by the -# cleaned names is left alone: remapping it would move `Site.Name` onto -# `Site.Name_1` when `Site Name` and `Site.Name` both exist. +# Line a roles table up with the cleaned names. One keyed by them already is +# left alone: with `Site Name` beside `Site.Name` a remap would move it. .align_roles_cols <- function(roles, name_map, clean_names) { if (!length(name_map) || !("col" %in% names(roles))) { return(roles) diff --git a/R/conform_table.R b/R/conform_table.R index ce19a77..ca519f5 100644 --- a/R/conform_table.R +++ b/R/conform_table.R @@ -247,9 +247,8 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing as.data.frame(out) } -# The effective merge map: each source follows its merges until it reaches a -# label nothing is merged away from. A source that reaches two such labels has -# no single target, so none of its merges is applied. +# The effective merge map: each source follows its merges to a label nothing +# is merged away from. A source that reaches two such labels is not merged. .resolve_merges <- function(merges) { .datatable.aware <- TRUE none <- list( diff --git a/R/mask_set.R b/R/mask_set.R index e27cfb2..ddce820 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -51,16 +51,16 @@ #' hint that a table looks like documentation. #' @param sheets For an Excel workbook, the sheets to read (default: all); #' passed to [read_set()]. -#' @param info Optional character vector naming tables (sheet names or file -#' names without extension) that hold documentation rather than data, -#' such as a data dictionary or a read-me. They are set aside before -#' masking: not masked, not in the synthetic set, not written by -#' [write_set()], and their content is not kept. Their names are +#' @param info Optional character vector of table names (sheet names, or +#' file names without extension) to set aside as documentation, for +#' example a data dictionary. They are set aside before masking: not +#' masked, not in the synthetic set, not written by [write_set()], and +#' their content is not kept. Their names are #' recorded on the recipe bundle. A table that looks like documentation #' and is not named here raises a hint (a message of class #' `masque_info_hint`); the hint decides nothing. -#' @param ... Must be empty. A misspelled argument is refused -#' (`masque_unused_arg_refusal`) rather than ignored. +#' @param ... Must be empty: any argument here is refused with +#' `masque_unused_arg_refusal`. #' #' @return A `masque_set` S7 object. Use [synthetic()] for the named list #' of synthetic tables and [recipe()] for the private recipe bundle. @@ -310,7 +310,7 @@ mask_set <- function(input, cli::cli_inform(c( i = paste0( "{cli::qty(length(hits))}Table{?s} {.val {hits}} look{?s/} like ", - "documentation rather than data." + "documentation." ), " " = paste0( "Name {cli::qty(length(hits))}{?it/them} in {.arg info} to set ", diff --git a/R/masque.R b/R/masque.R index 24aa2bf..eae4d12 100644 --- a/R/masque.R +++ b/R/masque.R @@ -83,9 +83,8 @@ #' column names and types alone. #' @param sheets For an Excel workbook, the sheets to read (default: all). #' See [mask_set()]. -#' @param info For a set, tables that hold documentation rather than data -#' (a data dictionary, a read-me): set aside, not masked and not written. -#' See [mask_set()]. +#' @param info For a set, names of documentation tables to set aside: not +#' masked and not written. See [mask_set()]. #' @param ... Further arguments for the masking verb: [mask()] for one #' table (for example `coords`, `events`, `allow_unmasked_coords`), #' [mask_set()] for a set (for example `links`). An argument the verb diff --git a/R/recipe.R b/R/recipe.R index 1c08ba5..54a0d9a 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -65,9 +65,8 @@ masque_recipe <- S7::new_class( class = S7::new_union(S7::class_list, NULL), default = NULL ), - # The conform_table() decisions mask() was given: the effective merge - # map, the applied type changes, the modes and max_levels. NULL when the - # input was not a conformance object, and absent before 0.16.0. + # conform_table() decisions (merge map, type changes, modes, max_levels); + # NULL unless mask() was given a conformance object. Absent before 0.16.0. conform = S7::new_property( class = S7::new_union(S7::class_list, NULL), default = NULL From f8049f148069497024a9c77a75200d8c515232b2 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:34:49 +1030 Subject: [PATCH 27/66] docs: document the internal .cleaning argument of mask() --- R/mask.R | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R/mask.R b/R/mask.R index 8e1da29..45d55a0 100644 --- a/R/mask.R +++ b/R/mask.R @@ -163,6 +163,9 @@ #' @param .shared_maps Internal. A named list of pre-computed #' `original -> alias` level maps for cross-table linked columns, set #' by [mask_set()]. Not for direct use. +#' @param .cleaning Internal. The cleaning record of a table [mask_set()] +#' has already cleaned; `mask()` then skips its own cleaning and stores +#' this record on the recipe. Not for direct use. #' @param ... Must be empty. An unused argument, for example a misspelled #' name, is an error. #' From 3a86f28722ecdc5ed7e6461a4638d368369fae95 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:34:58 +1030 Subject: [PATCH 28/66] docs: say what mask() keeps from a conformance object --- R/mask.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/mask.R b/R/mask.R index 45d55a0..b43705a 100644 --- a/R/mask.R +++ b/R/mask.R @@ -46,7 +46,9 @@ #' state is preserved across the call. #' #' @param df A data frame, or the `masque_conformance` object returned by -#' [conform_table()], whose `data` element is then masked. +#' [conform_table()], whose `data` element is then masked. Its cleaning +#' record and its merges and type changes go on the recipe, so +#' [apply_recipe()] replays them; `clean` is not applied a second time. #' @param roles A roles table from [propose_roles()] (possibly edited). #' Tables from masque <= 0.5.0 are upgraded with a deprecation #' warning; see [roles_validate()]. From e1a7c8972cd688c0f1257c892a78195f2f9d73c5 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:35:33 +1030 Subject: [PATCH 29/66] refactor: near-duplicate case pairs without a second full label matrix --- R/clean_table.R | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/R/clean_table.R b/R/clean_table.R index 3be9715..bdf748d 100644 --- a/R/clean_table.R +++ b/R/clean_table.R @@ -225,12 +225,16 @@ print.masque_cleaning <- function(x, ...) { return(.empty_near_dups()) } lower <- tolower(vals) - d <- utils::adist(vals) - # Upper triangle in column order: pairs listed by `b`, then by `a`. - idx <- which( - upper.tri(d) & (d == 1L | outer(lower, lower, "==")), - arr.ind = TRUE - ) + idx <- which(utils::adist(vals) == 1L, arr.ind = TRUE) + # Case variants can be several edits apart ("NSW", "nsw"). + shared <- which(duplicated(lower) | duplicated(lower, fromLast = TRUE)) + if (length(shared)) { + same <- which(outer(lower[shared], lower[shared], "=="), arr.ind = TRUE) + idx <- rbind(idx, cbind(shared[same[, 1L]], shared[same[, 2L]])) + } + # Each pair once, listed by `b`, then by `a`. + idx <- unique(idx[idx[, 1L] < idx[, 2L], , drop = FALSE]) + idx <- idx[order(idx[, 2L], idx[, 1L]), , drop = FALSE] if (!nrow(idx)) { return(.empty_near_dups()) } From d3419a32043df02d8e5d8cae16af5013b44203e0 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:41:44 +1030 Subject: [PATCH 30/66] feat: grade the audit on excess over a permutation baseline, per column and jointly The audit graded single columns on raw match rates: it could not see a joint or categorical leak, and a 0/1 outcome sat at medium by construction. Each categorical, logical, date and two-valued column now reports its same-row share in the original label space against the exact permutation chance (sum of p_k^2, within the conditioning strata for a scrambled numeric of a conditional clone) and is graded on the excess. Two-valued columns leave the exact-match rules. Table-level checks in attr(audit, 'joint') add a same-row joint match and a whole-row match over the content columns and a rare-cross count over visible categorical labels, each against permutations drawn under a fixed internal seed, plus an ungraded unit-size fingerprint note. In collaborate mode the joint verdicts raise masque_high_leakage, count in the tally and block package-managed writers; local mode reports them. --- R/audit.R | 80 +++- R/leakage.R | 493 +++++++++++++++++++++- R/mask.R | 31 +- R/release_gate.R | 49 ++- tests/testthat/test-audit_mask.R | 3 +- tests/testthat/test-fix0160-audit-joint.R | 388 +++++++++++++++++ tests/testthat/test-leakage.R | 66 +++ tests/testthat/test-safety-gate.R | 46 ++ 8 files changed, 1108 insertions(+), 48 deletions(-) create mode 100644 tests/testthat/test-fix0160-audit-joint.R diff --git a/R/audit.R b/R/audit.R index 941e6a0..4b00ba6 100644 --- a/R/audit.R +++ b/R/audit.R @@ -1,6 +1,7 @@ #' Audit a masque object for leakage and shareability risks #' -#' Returns a per-column audit data.table and prints a severity-grouped report. +#' Returns a per-column audit data.table, with table-level checks attached, and +#' prints a severity-grouped report. #' Auto-runs in `mode = "collaborate"` at [mask()] time and stores the #' result on the masque object (`m@audit`); for local-mode audits or #' explicit re-audits, pass the original data frame via `original`. @@ -20,8 +21,38 @@ #' \item `na_pattern_uniqueness`: fraction of rows in the original with #' a globally unique NA pattern (one number per data frame, repeated #' on every row). -#' \item `alias_status`: `aliased`, `passthrough`, or `dropped`. +#' \item `alias_status`: `aliased`, `permuted`, `passthrough`, or +#' `dropped`. #' \item `notes`: short human summary. +#' \item `same_row_pct`: percentage of rows where the clone value equals +#' the original value in the same row, read back through the level map +#' (categorical, logical, date and two-valued numeric columns). +#' \item `baseline_pct`: the same share expected by chance, under a random +#' permutation of the original column: \eqn{100 \sum_k p_k^2} over its +#' level frequencies (within the conditioning strata for a scrambled +#' numeric column of a conditional clone). +#' \item `excess_pct`: `same_row_pct - baseline_pct`. +#' } +#' +#' The table-level checks sit in `attr(audit, "joint")`, a data.table with +#' one row per check (`check`, `cols`, `observed`, `baseline`, `excess`, +#' `sd`, `leakage_class`, `notes`). Each baseline comes from independent +#' permutations of the original content columns (outcomes, covariates, +#' dates, kept text), drawn under a fixed internal seed that leaves the +#' caller's random number stream untouched; design, treatment, id, aliased +#' text and linked columns are held fixed: +#' +#' \itemize{ +#' \item `same_row_joint`: rows whose every content column equals the +#' original row. +#' \item `whole_row`: clone rows equal to some original row on every +#' retained column. +#' \item `rare_cross`: in the cross of visible (not aliased) categorical +#' covariates and outcomes, clone cells holding one row whose +#' combination is also a single row in the original. +#' \item `unit_sizes`: a note, never graded, when design units differ in +#' size, since the multiset of sizes can link a clone to a public trial +#' list. #' } #' #' Classification heuristics: @@ -34,6 +65,16 @@ #' \item Outcome with exact-match-pct > 1\% in collaborate -> `medium`. #' \item Numeric covariate with exact-match-pct > 5\% in collaborate -> #' `medium`. +#' \item Two-valued numeric columns skip the two exact-match rules: a 0/1 +#' column matches its own cell about half the time by chance. +#' \item Content column whose `excess_pct` is at least 10 points and four +#' binomial standard deviations in collaborate -> `medium`. +#' \item Joint match excess of at least 20 points and four permutation +#' standard deviations -> `high` (5 points -> `medium`); rare-cross +#' excess of at least 3 cells and four standard deviations -> `high` +#' (1 cell and three -> `medium`). In collaborate mode these verdicts +#' raise `masque_high_leakage` and block package-managed writers; in +#' local mode they are reported in the notes and graded `low`. #' \item Ignore column retained in local -> `low` (informational). #' } #' @@ -126,8 +167,12 @@ audit_mask <- function(m, original = NULL, print = TRUE) { ) } audits <- lapply(tables, function(nm) { + linked <- unlist(lapply(m@recipe@links, function(lk) { + if (nm %in% lk$tables) lk$name + }), use.names = FALSE) .audit_recompute( - original[[nm]], m@synthetic[[nm]], m@recipe@recipes[[nm]], m@mode + original[[nm]], m@synthetic[[nm]], m@recipe@recipes[[nm]], m@mode, + linked = as.character(linked) ) }) names(audits) <- tables @@ -141,10 +186,10 @@ audit_mask <- function(m, original = NULL, print = TRUE) { invisible(audits) } -# Internal: recompute an audit from the caller's original. The recipe's -# cleaning renames the original as mask() did, and aliased column names in -# the clone are read back to the names the recipe's roles use. -.audit_recompute <- function(original, synth, rec, mode) { +# Internal: recompute an audit from the caller's original, renamed as mask() +# cleaned it, against the clone under the names the recipe's roles use. +.audit_recompute <- function(original, synth, rec, mode, + linked = character()) { if (!is.data.frame(original)) { cli::cli_abort( "`original` must be a data frame; got {.cls {class(original)[1]}}.", @@ -161,7 +206,7 @@ audit_mask <- function(m, original = NULL, print = TRUE) { synth <- data.table::copy(synth) data.table::setnames(synth, nm) } - .compute_audit(original, synth, rec, mode) + .compute_audit(original, synth, rec, mode, linked = linked) } .print_audit <- function(audit, mode, table = NULL) { @@ -205,5 +250,24 @@ audit_mask <- function(m, original = NULL, print = TRUE) { ) } } + + # Table-level checks: joint matches, rare crosses, then informational notes. + joint <- attr(audit, "joint") + if (!is.null(joint) && nrow(joint)) { + cli::cli_h2("Table-level checks") + for (i in seq_len(nrow(joint))) { + line <- sprintf( + " %-9s %s", + if (joint$leakage_class[i] == "info") "note" else + toupper(joint$leakage_class[i]), + joint$notes[i] + ) + switch(joint$leakage_class[i], + high = cli::cli_alert_danger(line), + medium = cli::cli_alert_warning(line), + cli::cli_alert_info(line) + ) + } + } invisible(NULL) } diff --git a/R/leakage.R b/R/leakage.R index 31bd839..9a47503 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -1,11 +1,12 @@ #' Internal: build the audit data.table for a (df, synth, recipe, mode) tuple #' #' Called by `mask()` in `collaborate` mode and by `audit_mask()` on demand. -#' Returns a data.table with one row per column of the original data frame. +#' Returns a data.table with one row per column of the original data frame; +#' the table-level checks sit in its `"joint"` attribute. #' #' @keywords internal #' @noRd -.compute_audit <- function(df, synth, rec, mode) { +.compute_audit <- function(df, synth, rec, mode, linked = character()) { cols <- rec@roles$col has_action <- "action" %in% names(rec@roles) # Columns coarsened in place by the geomask. Retained, but displaced and @@ -14,6 +15,10 @@ use.names = FALSE) if (is.null(coarsened)) coarsened <- character() na_pattern_uniq <- .global_na_pattern_uniqueness(df) + strata <- .audit_strata(df, rec) + design_cols <- intersect( + rec@roles$col[rec@roles$role == "design"], names(df) + ) rows <- lapply(cols, function(col) { i <- which(rec@roles$col == col) @@ -38,6 +43,8 @@ n <- nrow(df) x <- df[[col]] + two_valued <- kind %in% .numeric_kinds() && + length(unique(x[!is.na(x)])) == 2L exact_match_pct <- NA_real_ comparable_n <- NA_integer_ @@ -51,6 +58,21 @@ } } + # Same-row match against a permutation baseline: categorical, date and + # two-valued columns, compared in the original label space. + same <- NULL + if (in_synth && (two_valued || + kind %in% c(.categorical_kinds(), .date_kinds()))) { + groups <- if (two_valued && .audit_numeric_resimulated(action)) { + strata + } + same <- .same_row_stats( + x, synth[[col]], map, + groups = groups, + units = .constant_unit(x, df, setdiff(design_cols, col)) + ) + } + n_unique_levels <- NA_integer_ freq_min <- NA_integer_ if (kind %in% c("factor", "character", "logical")) { @@ -68,15 +90,19 @@ in_synth = in_synth, alias_status = alias_status, exact_match_pct = exact_match_pct, freq_min = freq_min, n_unique_levels = n_unique_levels, n_rows = n, - coarsened = is_coarsened + coarsened = is_coarsened, + same_row_excess = if (is.null(same)) NA_real_ else same$excess, + same_row_sd = if (is.null(same)) NA_real_ else same$sd, + two_valued = two_valued, linked = col %in% linked ) notes <- .audit_notes( role, kind, action, pii, alias_status, mode, leakage_class, - exact_match_pct, freq_min, coarsened = is_coarsened + exact_match_pct, freq_min, coarsened = is_coarsened, + same = if (col %in% linked) NULL else same, two_valued = two_valued ) - data.frame( + data.table::data.table( col = col, role = role, action = if (is.na(action)) "" else action, @@ -90,11 +116,419 @@ na_pattern_uniqueness = na_pattern_uniq, alias_status = alias_status, notes = notes, - stringsAsFactors = FALSE + same_row_pct = if (is.null(same)) NA_real_ else same$observed, + baseline_pct = if (is.null(same)) NA_real_ else same$baseline, + excess_pct = if (is.null(same)) NA_real_ else same$excess ) }) - data.table::as.data.table(do.call(rbind, rows)) + audit <- data.table::rbindlist(rows) + joint <- .joint_audit(df, synth, rec, mode, audit, linked, strata) + data.table::setattr(audit, "joint", joint) + audit +} + +# Thresholds of the excess-over-chance rules, in percentage points of rows +# (cells for the rarity check). A finding needs both the absolute excess and +# `z` standard deviations of the permutation null, so a small table's chance +# swings are not graded. Calibrated on 20 seeds, conditional and not, of +# collaborate clones of agridat::john.alpha, a 6-trial MET and a 200-row +# clinical table: ordinary clones peaked at 9.5 points (z 3.1) per column, 0 +# points jointly and 2.8 cells (z 1.9) for rare crosses; leaky clones (kept +# outcome, kept binary outcome, kept visible crosses) reached 42.7 points +# (z 15.7) jointly and 8.8 cells (z 6.0). +.AUDIT_THRESHOLDS <- list( + same_row_medium = list(excess = 10, z = 4), + joint_medium = list(excess = 5, z = 4), + joint_high = list(excess = 20, z = 4), + rare_medium = list(excess = 1, z = 3), + rare_high = list(excess = 3, z = 4) +) + +# Permutations drawn for each table-level baseline, and the fixed seed they +# are drawn under (the caller's RNG state is restored afterwards). +.AUDIT_PERMUTATIONS <- 20L +.AUDIT_SEED <- 160L + +# TRUE when an excess clears a threshold pair; NA never does. +.excess_clears <- function(excess, sd, rule) { + isTRUE(is.finite(excess) && excess >= rule$excess && + excess >= rule$z * sd) +} + +# A scrambled numeric column is re-simulated through the copula, within the +# conditioning strata when the clone is conditional. +.audit_numeric_resimulated <- function(action) { + is.na(action) || !nzchar(action) || action == "scramble" +} + +# Row strata of a conditional clone (an integer id per row), else NULL. +.audit_strata <- function(df, rec) { + if (!isTRUE(rec@conditional)) { + return(NULL) + } + used <- intersect(rec@conditioning_used, names(df)) + if (!length(used)) { + return(NULL) + } + .row_group_id(df[used]) +} + +# Dense integer id of each row's combination of the columns in `cols` (a list +# or data frame); NA is a value like any other. +.row_group_id <- function(cols) { + codes <- lapply(cols, function(x) match(x, unique(x))) + data.table::frankv(codes, ties.method = "dense") +} + +# For each row of `cand`, whether the same combination occurs in `ref`; both +# hold integer codes in the same columns. +.rows_found <- function(cand, ref) { + n_ref <- nrow(ref) + ids <- data.table::frankv( + data.table::rbindlist(list(ref, cand)), ties.method = "dense" + ) + ids[n_ref + seq_len(nrow(cand))] %in% ids[seq_len(n_ref)] +} + +# Combinations held by exactly one row of `cand` and exactly one row of `ref`. +.singleton_overlap <- function(cand, ref) { + n_ref <- nrow(ref) + ids <- data.table::frankv( + data.table::rbindlist(list(ref, cand)), ties.method = "dense" + ) + k <- max(ids) + in_ref <- tabulate(ids[seq_len(n_ref)], k) + in_cand <- tabulate(ids[n_ref + seq_len(nrow(cand))], k) + sum(in_cand == 1L & in_ref == 1L) +} + +# The coarsest replicated design unit within which `x` takes one value, as an +# integer id per row; NULL when `x` varies within every design column. Such a +# column carries one value per unit, so its chance matches move unit by unit. +.constant_unit <- function(x, df, design_cols) { + best <- NULL + best_units <- Inf + for (dc in design_cols) { + u <- .row_group_id(list(df[[dc]])) + n_units <- max(u) + if (n_units >= best_units || n_units * 2L > length(x)) next + distinct <- tapply(x, u, function(v) length(unique(v[!is.na(v)]))) + if (all(distinct <= 1L)) { + best <- u + best_units <- n_units + } + } + best +} + +# Same-row match share of a clone column, its chance share under a random +# permutation of the original (within `groups` when given), the excess and a +# binomial SD of the chance share, counted in `units` when the column is +# constant within them. +.same_row_stats <- function(x, y, map, groups = NULL, units = NULL) { + x_cmp <- if (is.null(map)) .audit_values(x) else as.character(x) + y_cmp <- .audit_values(.audit_original_space(y, map)) + comparable <- !is.na(x_cmp) & !is.na(y) + m <- sum(comparable) + if (m == 0L) { + return(NULL) + } + matched <- comparable & !is.na(y_cmp) & x_cmp == y_cmp + observed <- 100 * sum(matched) / m + + # Chance share: sum over groups of n_gk^2 / n_g, over the comparable rows. + g <- if (is.null(groups)) { + rep(1L, m) + } else { + .row_group_id(list(groups[comparable])) + } + pair <- .row_group_id(list(g, x_cmp[comparable])) + pair_n <- tabulate(pair) + pair_group <- g[match(seq_along(pair_n), pair)] + chance <- sum(pair_n^2 / tabulate(g)[pair_group]) / m + + m_eff <- if (is.null(units)) m else length(unique(units[comparable])) + sd <- 100 * sqrt(chance * (1 - chance) / m_eff) + list( + observed = observed, baseline = 100 * chance, + excess = observed - 100 * chance, sd = sd + ) +} + +# Values comparable by equality: numbers stay numbers, the rest characters. +.audit_values <- function(x) { + if (is.numeric(x) && !is.factor(x)) { + return(as.numeric(x)) + } + as.character(x) +} + +# A clone column read back into the original label space through its map. +.audit_original_space <- function(y, map) { + if (is.null(map)) { + return(y) + } + inv <- stats::setNames(names(map), unname(map)) + unname(inv[as.character(y)]) +} + +# Table-level checks an adversary would run, each against a permutation +# baseline: same-row and whole-row joint matches over the retained content +# columns, singleton crosses of visible categorical labels, and the unit-size +# fingerprint (a note, never graded). Design, treatment, id, aliased text and +# linked columns are preserved by construction and only held fixed. +.joint_audit <- function(df, synth, rec, mode, audit, linked, strata) { + collab <- identical(mode, "collaborate") + coarsened <- unlist(lapply(rec@coords, function(cr) c(cr$lat, cr$lon)), + use.names = FALSE) + retained <- audit$alias_status != "dropped" & !(audit$col %in% coarsened) + structural <- audit$role %in% c("design", "treatment", "id") | + (audit$role == "text" & audit$action == "alias") | + audit$col %in% linked + fixed_cols <- audit$col[retained & structural] + content_cols <- audit$col[retained & !structural] + n <- nrow(df) + out <- list() + + if (length(content_cols) && n > 0L) { + codes <- .joint_codes(df, synth, rec, c(fixed_cols, content_cols)) + groups <- lapply(content_cols, function(col) { + i <- match(col, audit$col) + if (audit$kind[i] %in% .numeric_kinds() && + .audit_numeric_resimulated(audit$action[i])) { + strata + } + }) + names(groups) <- content_cols + st <- with_rng_state( + .AUDIT_SEED, + .joint_match_stats( + codes$orig, codes$clone, content_cols, groups, fixed_cols + ) + ) + # One row is the smallest step a share can take. + st$sd <- pmax(st$sd, 100 / n) + labels <- c(same_row_joint = "same-row joint match", + whole_row = "whole-row match") + for (k in seq_along(labels)) { + out[[length(out) + 1L]] <- .joint_row( + check = names(labels)[k], + cols = if (k == 1L) content_cols else c(fixed_cols, content_cols), + observed = st$observed[k], baseline = st$baseline[k], + sd = st$sd[k], label = labels[[k]], unit = "%", + high = .AUDIT_THRESHOLDS$joint_high, + medium = .AUDIT_THRESHOLDS$joint_medium, collab = collab + ) + } + } + + visible <- audit$col[retained & + audit$role %in% c("covariate", "outcome") & + audit$kind %in% .categorical_kinds() & + audit$alias_status %in% c("passthrough", "permuted")] + if (length(visible) >= 2L && n > 0L) { + rare <- with_rng_state( + .AUDIT_SEED, .rare_cross_stats(df, synth, visible) + ) + out[[length(out) + 1L]] <- .joint_row( + check = "rare_cross", cols = visible, + observed = rare$observed, baseline = rare$baseline, + # Half a cell: the null of a rare count can have no spread at all. + sd = max(rare$sd, 0.5), + label = "clone singleton cells also unique in the original", + unit = " cells", + high = .AUDIT_THRESHOLDS$rare_high, + medium = .AUDIT_THRESHOLDS$rare_medium, collab = collab + ) + } + + # Labels cannot tell a trial from a rep restarted within it, so the note + # names every top-level design column whose units differ in size. + unequal <- .unequal_unit_cols( + df, audit$col[retained & audit$role == "design"] + ) + if (length(unequal)) { + out[[length(out) + 1L]] <- data.table::data.table( + check = "unit_sizes", cols = paste(unequal, collapse = ", "), + observed = NA_real_, baseline = NA_real_, excess = NA_real_, + sd = NA_real_, leakage_class = "info", + notes = sprintf( + paste0( + "unit sizes differ across %s; the multiset of sizes can link the ", + "clone to a public trial list" + ), + paste(unequal, collapse = ", ") + ) + ) + } + + if (!length(out)) { + return(.joint_row_template()[0L]) + } + data.table::rbindlist(out) +} + +.joint_row_template <- function() { + data.table::data.table( + check = character(), cols = character(), observed = numeric(), + baseline = numeric(), excess = numeric(), sd = numeric(), + leakage_class = character(), notes = character() + ) +} + +# One graded table-level row. Local mode reports the verdict in the notes but +# grades it low. +.joint_row <- function(check, cols, observed, baseline, sd, label, unit, + high, medium, collab) { + excess <- observed - baseline + verdict <- if (.excess_clears(excess, sd, high)) { + "high" + } else if (.excess_clears(excess, sd, medium)) { + "medium" + } else { + "low" + } + fmt <- if (unit == "%") "%.1f%%" else paste0("%.1f", unit) + notes <- sprintf( + paste0("%s ", fmt, " vs chance ", fmt, " (sd %.1f)"), + label, observed, baseline, sd + ) + if (!collab && verdict != "low") { + notes <- sprintf("%s; %s in collaborate mode", notes, toupper(verdict)) + } + data.table::data.table( + check = check, cols = paste(cols, collapse = ", "), + observed = observed, baseline = baseline, excess = excess, sd = sd, + leakage_class = if (collab) verdict else "low", notes = notes + ) +} + +# Integer codes of the original and the clone on a shared code space, the +# clone read back into the original labels; NA is a code of its own. +.joint_codes <- function(df, synth, rec, cols) { + orig <- vector("list", length(cols)) + clone <- vector("list", length(cols)) + for (j in seq_along(cols)) { + map <- rec@level_maps[[cols[j]]] + x <- if (is.null(map)) .audit_values(df[[cols[j]]]) else + as.character(df[[cols[j]]]) + y <- .audit_values(.audit_original_space(synth[[cols[j]]], map)) + if (is.character(x) != is.character(y)) { + x <- as.character(x) + y <- as.character(y) + } + u <- unique(c(x, y)) + orig[[j]] <- match(x, u) + clone[[j]] <- match(y, u) + } + names(orig) <- names(clone) <- cols + list( + orig = data.table::as.data.table(orig), + clone = data.table::as.data.table(clone) + ) +} + +# Observed, mean and SD (in percent of rows) of the same-row joint match over +# `content` and the whole-row match over every column, against clones whose +# content columns are independent permutations of the original's. +.joint_match_stats <- function(orig, clone, content, groups, fixed) { + n <- nrow(orig) + same_row <- function(cand) { + ok <- rep(TRUE, n) + for (col in content) ok <- ok & cand[[col]] == orig[[col]] + mean(ok) + } + # Fixed columns that identify every row leave a clone row one candidate, + # its own original row: the whole-row share is then the same-row share. + unique_key <- length(fixed) > 0L && + anyDuplicated(.row_group_id(lapply(fixed, function(f) orig[[f]]))) == 0L + whole_row <- function(cand) { + same_fixed <- all(vapply(fixed, function(f) { + identical(cand[[f]], orig[[f]]) + }, logical(1L))) + if (unique_key && same_fixed) { + return(same_row(cand)) + } + mean(.rows_found(cand, orig)) + } + observed <- c(same_row(clone), whole_row(clone)) + draws <- vapply(seq_len(.AUDIT_PERMUTATIONS), function(b) { + cand <- data.table::copy(orig) + for (col in content) { + data.table::set( + cand, j = col, + value = orig[[col]][.permute_rows(n, groups[[col]])] + ) + } + c(same_row(cand), whole_row(cand)) + }, numeric(2L)) + list( + observed = 100 * observed, + baseline = 100 * rowMeans(draws), + sd = 100 * apply(draws, 1L, stats::sd) + ) +} + +# A random row order, within `groups` when given. +.permute_rows <- function(n, groups = NULL) { + idx <- seq_len(n) + if (is.null(groups)) { + return(idx[sample.int(n)]) + } + as.integer(stats::ave(idx, groups, FUN = function(g) { + g[sample.int(length(g))] + })) +} + +# Clone cells (combinations of the visible categorical labels) holding exactly +# one row whose combination is also a single row in the original, against +# the same count for independent permutations of the original columns. +.rare_cross_stats <- function(df, synth, cols) { + orig <- vector("list", length(cols)) + clone <- vector("list", length(cols)) + for (j in seq_along(cols)) { + x <- as.character(df[[cols[j]]]) + y <- as.character(synth[[cols[j]]]) + u <- unique(c(x, y)) + orig[[j]] <- match(x, u) + clone[[j]] <- match(y, u) + } + names(orig) <- names(clone) <- sprintf("v%d", seq_along(cols)) + orig <- data.table::as.data.table(orig) + clone <- data.table::as.data.table(clone) + overlap <- function(cand) .singleton_overlap(cand, orig) + n <- nrow(orig) + draws <- vapply(seq_len(.AUDIT_PERMUTATIONS), function(b) { + cand <- data.table::copy(orig) + for (col in names(cand)) { + data.table::set(cand, j = col, value = orig[[col]][sample.int(n)]) + } + as.numeric(overlap(cand)) + }, numeric(1L)) + list( + observed = as.numeric(overlap(clone)), + baseline = mean(draws), sd = stats::sd(draws) + ) +} + +# Retained design columns, not nested in another, whose units differ in size. +.unequal_unit_cols <- function(df, design_cols) { + ids <- lapply(design_cols, function(dc) .row_group_id(list(df[[dc]]))) + names(ids) <- design_cols + out <- character() + for (dc in design_cols) { + sizes <- tabulate(ids[[dc]]) + if (length(sizes) < 2L || length(unique(sizes)) == 1L) next + nested <- vapply(setdiff(design_cols, dc), function(other) { + length(unique(ids[[other]])) < length(sizes) && + all(tapply(ids[[other]], ids[[dc]], function(v) { + length(unique(v)) + }) == 1L) + }, logical(1L)) + if (!any(nested)) out <- c(out, dc) + } + out } # Share of rows whose NA pattern no other row shares, grouped on the NA matrix. @@ -106,15 +540,15 @@ if (ncol(df) == 0L) { return(if (n == 1L) 1 else 0) } - na_dt <- data.table::as.data.table(lapply(df, is.na)) - data.table::setnames(na_dt, sprintf("v%d", seq_along(na_dt))) - counts <- na_dt[, .N, by = c(names(na_dt))]$N - sum(counts == 1L) / n + pattern <- data.table::frankv(lapply(df, is.na), ties.method = "dense") + sum(tabulate(pattern) == 1L) / n } .classify_leakage <- function( role, kind, action, pii, mode, in_synth, alias_status, - exact_match_pct, freq_min, n_unique_levels, n_rows, coarsened = FALSE + exact_match_pct, freq_min, n_unique_levels, n_rows, coarsened = FALSE, + same_row_excess = NA_real_, same_row_sd = NA_real_, two_valued = FALSE, + linked = FALSE ) { collab <- identical(mode, "collaborate") @@ -142,15 +576,28 @@ return("high") } - # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM - if (role == "outcome" && collab && + # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM. A two-valued + # column matches its own cell by chance about half the time; rule 5b + # grades it on the excess over that chance instead. + if (role == "outcome" && collab && !isTRUE(two_valued) && !is.na(exact_match_pct) && exact_match_pct > 1) { return("medium") } # 5. Numeric covariate with exact-match-pct > 5% in collaborate: MEDIUM if (role == "covariate" && kind %in% c("numeric", "integer") && - collab && !is.na(exact_match_pct) && exact_match_pct > 5) { + collab && !isTRUE(two_valued) && + !is.na(exact_match_pct) && exact_match_pct > 5) { + return("medium") + } + + # 5b. Content column returned to its own row beyond chance in collaborate: + # MEDIUM. Structural and linked columns match by construction. + if (collab && in_synth && !isTRUE(linked) && + !(role %in% c("design", "treatment", "id")) && + .excess_clears( + same_row_excess, same_row_sd, .AUDIT_THRESHOLDS$same_row_medium + )) { return("medium") } @@ -166,7 +613,7 @@ .audit_notes <- function(role, kind, action, pii, alias_status, mode, leakage_class, exact_match_pct, freq_min, - coarsened = FALSE) { + coarsened = FALSE, same = NULL, two_valued = FALSE) { collab <- identical(mode, "collaborate") bits <- character() if (pii && !isTRUE(coarsened)) bits <- c(bits, "PII-pattern column name") @@ -193,14 +640,16 @@ if (!is.na(freq_min) && freq_min == 1L && role == "covariate" && collab) { bits <- c(bits, "rare level (freq = 1)") } - if (!is.na(exact_match_pct) && exact_match_pct > 1 && collab) { + if (!is.na(exact_match_pct) && exact_match_pct > 1 && collab && + !isTRUE(two_valued)) { bits <- c(bits, sprintf("exact-match %.1f%%", exact_match_pct)) } + if (!is.null(same) && !(role %in% c("design", "treatment", "id")) && + isTRUE(same$excess >= .AUDIT_THRESHOLDS$same_row_medium$excess)) { + bits <- c(bits, sprintf( + "same-row %.1f%% vs chance %.1f%%", same$observed, same$baseline + )) + } if (length(bits) == 0L) bits <- "ok" paste(bits, collapse = "; ") } - -# data.table's `[` uses data.table semantics only in a namespace that declares -# itself aware; masque imports data.table through `::` alone. -.datatable.aware <- TRUE -utils::globalVariables(c(".N", ".I")) diff --git a/R/mask.R b/R/mask.R index a9fe5de..a7092c5 100644 --- a/R/mask.R +++ b/R/mask.R @@ -454,6 +454,8 @@ mask <- function(df, mode = mode, seed = if (is.null(seed)) NULL else as.integer(seed), roles = .strip_roles_provenance(as.data.frame(roles)), + conditional = conditional, + conditioning_used = conditioning_used, column_name_map = NULL, level_maps = level_maps, storage_classes = list(), @@ -462,13 +464,28 @@ mask <- function(df, warnings = character(), integrity_fp = "" ) - audit_tbl <- .compute_audit(df, synth, .tmp_rec, mode) - high_leaks <- audit_tbl$col[audit_tbl$leakage_class == "high"] - if (length(high_leaks)) { - msg <- sprintf( - "audit_mask() flagged HIGH leakage on %s: %s", - ngettext(length(high_leaks), "column", "columns"), - paste(high_leaks, collapse = ", ") + audit_tbl <- .compute_audit( + df, synth, .tmp_rec, mode, linked = names(.shared_maps) + ) + high_cols <- audit_tbl$col[audit_tbl$leakage_class == "high"] + high_checks <- setdiff(.audit_high_findings(audit_tbl), high_cols) + if (length(high_cols) || length(high_checks)) { + parts <- c( + if (length(high_cols)) { + sprintf( + "%s: %s", ngettext(length(high_cols), "column", "columns"), + paste(high_cols, collapse = ", ") + ) + }, + if (length(high_checks)) { + sprintf( + "table %s: %s", ngettext(length(high_checks), "check", "checks"), + paste(high_checks, collapse = ", ") + ) + } + ) + msg <- paste0( + "audit_mask() flagged HIGH leakage on ", paste(parts, collapse = "; ") ) # Classed so callers can handle it programmatically; safety # findings are never suppressed by package code. diff --git a/R/release_gate.R b/R/release_gate.R index e4a4f18..53d2dee 100644 --- a/R/release_gate.R +++ b/R/release_gate.R @@ -3,7 +3,8 @@ # Generation never implies release: package writers refuse a collaborate-mode # object with unresolved HIGH findings unless `allow_high = TRUE`. -# Columns flagged HIGH at mask time; for a masque_set, as "$". +# Findings flagged HIGH at mask time: columns, then table-level checks named +# "[]". For a masque_set, each is prefixed "
$". .audit_high_cols <- function(m) { audit <- m@audit if (is.null(audit)) { @@ -12,20 +13,45 @@ if (S7::S7_inherits(m, masque_set)) { out <- character() for (nm in names(audit)) { - tbl <- audit[[nm]] - if (is.null(tbl)) next - hits <- tbl$col[tbl$leakage_class == "high"] + hits <- .audit_high_findings(audit[[nm]]) if (length(hits)) { out <- c(out, paste0(nm, "$", hits)) } } return(out) } - audit$col[audit$leakage_class == "high"] + .audit_high_findings(audit) +} + +# HIGH findings of one audit table. +.audit_high_findings <- function(audit) { + if (is.null(audit)) { + return(character()) + } + joint <- attr(audit, "joint") + checks <- if (is.null(joint)) { + character() + } else { + high <- joint$check[joint$leakage_class == "high"] + if (length(high)) sprintf("[%s]", .joint_check_label(high)) else character() + } + c(audit$col[audit$leakage_class == "high"], checks) +} + +# Reader-facing names of the table-level checks. +.joint_check_label <- function(check) { + labels <- c( + same_row_joint = "same-row joint match", + whole_row = "whole-row match", + rare_cross = "rare category cross", + unit_sizes = "unit sizes" + ) + unname(labels[check]) } # HIGH / medium / low counts across the whole object (summed over tables -# for a set). NULL when no audit was run (local mode). +# for a set). Graded table-level checks add to the high and medium counts. +# NULL when no audit was run (local mode). .audit_tally <- function(m) { audit <- m@audit if (is.null(audit)) { @@ -35,9 +61,12 @@ cls <- unlist(lapply(tbls, function(tbl) { if (is.null(tbl)) character() else tbl$leakage_class }), use.names = FALSE) + joint <- unlist(lapply(tbls, function(tbl) { + attr(tbl, "joint")$leakage_class + }), use.names = FALSE) c( - high = sum(cls == "high"), - medium = sum(cls == "medium"), + high = sum(cls == "high") + sum(joint == "high"), + medium = sum(cls == "medium") + sum(joint == "medium"), low = sum(cls == "low") ) } @@ -78,7 +107,7 @@ c( paste0( "Write blocked: the audit flagged HIGH leakage on ", - "{length(high)} column{?s}." + "{length(high)} finding{?s}." ), x = "Flagged: {flagged}.", i = "Re-role, alias, or drop each flagged column, then mask again.", @@ -93,7 +122,7 @@ warning(warningCondition( sprintf( "allow_high = TRUE: writing despite HIGH leakage on %s: %s.", - ngettext(length(high), "column", "columns"), flagged + ngettext(length(high), "finding", "findings"), flagged ), class = "masque_high_override" )) diff --git a/tests/testthat/test-audit_mask.R b/tests/testthat/test-audit_mask.R index 15ebce2..5199b58 100644 --- a/tests/testthat/test-audit_mask.R +++ b/tests/testthat/test-audit_mask.R @@ -29,7 +29,8 @@ test_that("audit_mask returns a data.table with the spec'd schema", { "n_unique_levels", "freq_min", "exact_match_pct", "comparable_n", "na_pct", "na_pattern_uniqueness", - "alias_status", "notes" + "alias_status", "notes", + "same_row_pct", "baseline_pct", "excess_pct" )) expect_equal(nrow(out), ncol(df)) expect_true(all(out$leakage_class %in% c("low", "medium", "high"))) diff --git a/tests/testthat/test-fix0160-audit-joint.R b/tests/testthat/test-fix0160-audit-joint.R new file mode 100644 index 0000000..de1c111 --- /dev/null +++ b/tests/testthat/test-fix0160-audit-joint.R @@ -0,0 +1,388 @@ +# F10 and F20: the audit grades re-identification on excess over a +# permutation baseline, per column and jointly, and the joint verdicts feed the +# collaborate HIGH gate. + +.joint_roles <- function(df, spec, mode = "collaborate") { + r <- propose_roles(df, mode = mode) + for (col in names(spec)) { + parts <- strsplit(spec[[col]], "/", fixed = TRUE)[[1L]] + r <- if (length(parts) == 2L) { + set_role(r, col, role = parts[1L], action = parts[2L]) + } else { + set_role(r, col, role = parts[1L]) + } + } + r +} + +.joint_mask <- function(df, spec, mode = "collaborate", ...) { + mask(df, .joint_roles(df, spec, mode), mode = mode, ...) +} + +.alpha_spec <- c( + plot = "id", rep = "design/keep", block = "design/keep", + row = "design/keep", col = "design/keep", gen = "treatment", + yield = "outcome" +) + +# Six trials x 20 genotypes x 3 reps, with a trial-level soil and aet. +.joint_met <- function(sizes = NULL) { + set.seed(1L) + trials <- sprintf("T%d", 1:6) + gens <- sprintf("G%02d", 1:20) + g <- expand.grid( + genotype = gens, rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$block <- sprintf("B%d", (seq_len(nrow(g)) - 1L) %/% 5L %% 4L + 1L) + g$yield <- round(3 + rnorm(nrow(g), 0, 0.5), 2) + g$aet <- c(312.456, 287.901, 401.233, 355.678, 298.112, 376.543)[ + match(g$trial, trials) + ] + g$plot_cov <- round(rnorm(nrow(g), 50, 8), 2) + g$soil <- c("clay", "loam", "sand", "clay", "loam", "sand")[ + match(g$trial, trials) + ] + if (!is.null(sizes)) { + keep <- unlist(lapply(seq_along(trials), function(i) { + utils::head(which(g$trial == trials[i]), sizes[i]) + })) + g <- g[keep, , drop = FALSE] + } + g +} + +.met_spec <- c( + trial = "design/keep", rep = "design/keep", block = "design/keep", + genotype = "treatment", yield = "outcome", aet = "covariate", + plot_cov = "covariate", soil = "covariate" +) + +.joint_clinical <- function(n = 200L) { + set.seed(7L) + data.frame( + arm = factor(sample(c("A", "B"), n, TRUE)), + age = round(rnorm(n, 64, 10)), + sex = factor(sample(c("F", "M"), n, TRUE)), + diagnosis = factor(sample(c("C50", "C34", "C18", "C61", "C25"), n, TRUE)), + stage = factor(sample(c("I", "II", "III", "IV"), n, TRUE)), + biomarker = round(rlnorm(n, 2, 0.6), 2), + time_to_death = round(rexp(n, 1 / 9), 1), + progressed = rbinom(n, 1L, 0.65), + died = rbinom(n, 1L, 0.5) + ) +} + +.clinical_spec <- c( + arm = "treatment", age = "covariate", sex = "covariate", + diagnosis = "covariate", stage = "covariate", biomarker = "covariate", + time_to_death = "outcome", progressed = "outcome", died = "outcome" +) + +# Two visible categorical covariates with 13 single-row crosses and no +# single-row level. +.rare_table <- function() { + set.seed(99L) + n <- 200L + region <- sprintf("R%02d", rep(1:10, each = 20L)) + occupation <- sprintf("O%02d", rep(1:10, 20L)) + for (k in 1:15) occupation[k * 13L] <- sprintf("O%02d", (k * 7L) %% 10L + 1L) + data.frame( + arm = factor(sample(c("A", "B"), n, TRUE)), + region = region, occupation = occupation, + score = round(rnorm(n), 2) + ) +} + +# Variable names never match a column name: under data.table semantics a +# clash would compare the column with itself. +.joint_row_of <- function(audit, which_check) { + j <- attr(audit, "joint") + j[j[["check"]] == which_check, , drop = FALSE] +} + +test_that("the audit keeps its columns, adds the same-row figures and a joint table", { + m <- .joint_mask(.joint_clinical(), .clinical_spec, seed = 1L) + a <- audit_mask(m, print = FALSE) + expect_identical(names(a)[1:13], c( + "col", "role", "action", "kind", "leakage_class", + "n_unique_levels", "freq_min", "exact_match_pct", "comparable_n", + "na_pct", "na_pattern_uniqueness", "alias_status", "notes" + )) + expect_identical( + names(a)[14:16], c("same_row_pct", "baseline_pct", "excess_pct") + ) + joint <- attr(a, "joint") + expect_s3_class(joint, "data.table") + expect_named(joint, c( + "check", "cols", "observed", "baseline", "excess", "sd", + "leakage_class", "notes" + )) + expect_true(all(c("same_row_joint", "whole_row") %in% joint$check)) + printed <- capture_full(audit_mask(m)) + expect_match(printed, "Table-level checks", fixed = TRUE) +}) + +test_that("same-row figures compare in the original label space against sum(p^2)", { + df <- .joint_clinical() + m <- .joint_mask(df, .clinical_spec, seed = 2L) + a <- audit_mask(m, print = FALSE) + map <- recipe(m)@level_maps$stage + expect_false(is.null(map)) + back <- names(map)[match(as.character(synthetic(m)$stage), unname(map))] + row <- a[a$col == "stage", ] + expect_equal(row$same_row_pct, 100 * mean(back == as.character(df$stage))) + p <- prop.table(table(df$stage)) + expect_equal(row$baseline_pct, 100 * sum(p^2)) + expect_equal(row$excess_pct, row$same_row_pct - row$baseline_pct) + # Continuous numerics keep exact_match_pct and carry no same-row figure. + expect_true(is.na(a$same_row_pct[a$col == "biomarker"])) +}) + +test_that("a 0/1 outcome is graded on its excess over chance, not its raw match", { + df <- .joint_clinical() + for (seed in 1:5) { + m <- .joint_mask(df, .clinical_spec, seed = seed) + a <- audit_mask(m, print = FALSE) + for (column in c("progressed", "died")) { + row <- a[a[["col"]] == column, ] + # The raw match sits near one half, as it does for any 0/1 column. + expect_gt(row$exact_match_pct, 30) + expect_equal(row$leakage_class, "low", info = paste(column, seed)) + expect_false(grepl("exact-match", row$notes, fixed = TRUE)) + } + } +}) + +test_that("ordinary collaborate clones raise no HIGH from the new rules", { + skip_if_not_installed("agridat") + alpha <- agridat::john.alpha + met <- .joint_met() + clinical <- .joint_clinical() + visible <- .clinical_spec + visible[c("sex", "diagnosis", "stage")] <- "covariate/scramble" + cases <- list( + list(alpha, .alpha_spec), list(met, .met_spec), + list(clinical, .clinical_spec), list(clinical, visible) + ) + for (case in cases) { + for (conditional in c(FALSE, TRUE)) { + for (seed in 1:3) { + expect_no_warning( + m <- .joint_mask( + case[[1L]], case[[2L]], seed = seed, conditional = conditional + ), + class = "masque_high_leakage" + ) + joint <- attr(m@audit, "joint") + expect_false(any(joint$leakage_class == "high")) + expect_identical(sum(m@audit$leakage_class == "high"), 0L) + } + } + } +}) + +test_that("an outcome kept as-is trips the joint checks and blocks the write", { + skip_if_not_installed("agridat") + spec <- .alpha_spec + spec["yield"] <- "outcome/keep" + w <- expect_warning( + m <- .joint_mask(agridat::john.alpha, spec, seed = 1L), + class = "masque_high_leakage" + ) + expect_match(conditionMessage(w), "[same-row joint match]", fixed = TRUE) + expect_match(conditionMessage(w), "[whole-row match]", fixed = TRUE) + srj <- .joint_row_of(m@audit, "same_row_joint") + expect_identical(srj$leakage_class, "high") + expect_gt(srj$excess, 90) + expect_identical(.joint_row_of(m@audit, "whole_row")$leakage_class, "high") + # No column rule reaches HIGH: the joint verdict alone blocks the write. + expect_false(any(m@audit$leakage_class == "high")) + expect_gt(masque:::.audit_tally(m)[["high"]], 0L) + expect_identical(masque:::.release_status(m), "blocked") + expect_error(masque:::.gate_release(m), class = "masque_blocked_write") + + out <- file.path(tempdir(), "masque-joint-blocked.csv") + on.exit(unlink(out), add = TRUE) + expect_error( + suppressWarnings(masque( + agridat::john.alpha, + roles = .joint_roles(agridat::john.alpha, spec), mode = "collaborate", + seed = 1L, ask = FALSE, quiet = TRUE, out = out + )), + class = "masque_blocked_write" + ) + expect_false(file.exists(out)) +}) + +test_that("a kept 0/1 outcome trips the same-row joint check", { + df <- .joint_clinical()[, c("arm", "progressed")] + expect_warning( + m <- .joint_mask( + df, c(arm = "treatment", progressed = "outcome/keep"), seed = 1L + ), + class = "masque_high_leakage" + ) + expect_identical( + .joint_row_of(m@audit, "same_row_joint")$leakage_class, "high" + ) +}) + +test_that("visible covariates with single-row crosses trip the rarity check", { + df <- .rare_table() + # No single level is rare, so the single-column rule cannot see it. + expect_gt(min(table(df$region)), 1L) + expect_gt(min(table(df$occupation)), 1L) + spec <- c( + arm = "treatment", region = "covariate/keep", + occupation = "covariate/keep", score = "outcome" + ) + w <- expect_warning( + m <- .joint_mask(df, spec, seed = 1L), + class = "masque_high_leakage" + ) + expect_match(conditionMessage(w), "[rare category cross]", fixed = TRUE) + rare <- .joint_row_of(m@audit, "rare_cross") + expect_identical(rare$leakage_class, "high") + expect_identical(rare$observed, 13) + expect_lt(rare$baseline, rare$observed) + + # Scrambled, the same columns sit at chance. + spec[c("region", "occupation")] <- "covariate/scramble" + for (seed in 1:3) { + expect_no_warning( + m2 <- .joint_mask(df, spec, seed = seed), + class = "masque_high_leakage" + ) + expect_false( + .joint_row_of(m2@audit, "rare_cross")$leakage_class == "high" + ) + } + # Aliased labels are not visible and are left out of the cross. + spec[c("region", "occupation")] <- "covariate/alias" + m3 <- .joint_mask(df, spec, seed = 1L) + expect_identical(nrow(.joint_row_of(m3@audit, "rare_cross")), 0L) +}) + +test_that("local mode reports the joint verdicts without grading them", { + skip_if_not_installed("agridat") + spec <- .alpha_spec + spec["yield"] <- "outcome/keep" + roles <- .joint_roles(agridat::john.alpha, spec, mode = "local") + m <- mask(agridat::john.alpha, roles, mode = "local", seed = 1L) + a <- audit_mask(m, original = agridat::john.alpha, print = FALSE) + srj <- .joint_row_of(a, "same_row_joint") + expect_identical(srj$leakage_class, "low") + expect_gt(srj$excess, 90) + expect_match(srj$notes, "HIGH in collaborate mode", fixed = TRUE) + expect_no_error(masque:::.gate_release(m)) +}) + +test_that("unequal unit sizes add a note that is never graded", { + met <- .joint_met(sizes = c(60L, 60L, 45L, 60L, 30L, 60L)) + m <- .joint_mask(met, .met_spec, seed = 1L) + note <- .joint_row_of(m@audit, "unit_sizes") + expect_identical(nrow(note), 1L) + expect_true("trial" %in% strsplit(note$cols, ", ", fixed = TRUE)[[1L]]) + expect_identical(note$leakage_class, "info") + expect_match(note$notes, "public trial list", fixed = TRUE) + expect_identical(masque:::.release_status(m) == "blocked", FALSE) + + balanced <- .joint_mask(.joint_met(), .met_spec, seed = 1L) + expect_identical(nrow(.joint_row_of(balanced@audit, "unit_sizes")), 0L) +}) + +test_that("the joint baselines leave the caller's RNG alone and repeat exactly", { + df <- .joint_clinical() + m <- .joint_mask(df, .clinical_spec, mode = "local", seed = 3L) + set.seed(42L) + before <- .Random.seed + a1 <- audit_mask(m, original = df, print = FALSE) + expect_identical(.Random.seed, before) + a2 <- audit_mask(m, original = df, print = FALSE) + expect_identical(attr(a1, "joint"), attr(a2, "joint")) +}) + +test_that("a linked column is held fixed like a design column", { + set.seed(5L) + tables <- list( + plots = data.frame( + site = rep(c("S1", "S2", "S3"), each = 20L), + gen = rep(sprintf("G%02d", 1:10), 6L), + yield = round(rnorm(60L, 4, 0.5), 2) + ), + soils = data.frame( + site = rep(c("S1", "S2", "S3"), 10L), + ph = round(rnorm(30L, 6.5, 0.4), 2) + ) + ) + roles <- list( + plots = .joint_roles(tables$plots, c( + site = "design/alias", gen = "treatment", yield = "outcome" + )), + soils = .joint_roles(tables$soils, c( + site = "covariate/alias", ph = "outcome" + )) + ) + expect_no_warning( + ms <- mask_set( + tables, roles = roles, mode = "collaborate", seed = 1L, quiet = TRUE + ), + class = "masque_high_leakage" + ) + soils <- ms@audit$soils + # The shared map relabels in place: the link returns to its row by design. + expect_identical(soils$same_row_pct[soils[["col"]] == "site"], 100) + expect_identical(soils$leakage_class[soils[["col"]] == "site"], "low") + expect_identical(.joint_row_of(soils, "same_row_joint")$cols, "ph") + local_audit <- audit_mask( + mask_set(tables, roles = lapply(roles, function(r) { + attr(r, "mode") <- "local" + r + }), mode = "local", seed = 1L, quiet = TRUE), + original = tables, print = FALSE + ) + expect_identical( + .joint_row_of(local_audit$soils, "same_row_joint")$cols, "ph" + ) +}) + +test_that("observed joint shares equal a brute-force count in label space", { + skip_if_not_installed("agridat") + # Row keys pasted from the original labels: an independent route to the + # same two shares, for a unique design key and for a shared one. + brute <- function(df, m, fixed, content) { + syn <- as.data.frame(synthetic(m)) + for (col in names(recipe(m)@level_maps)) { + map <- recipe(m)@level_maps[[col]] + syn[[col]] <- names(map)[match(as.character(syn[[col]]), unname(map))] + } + key <- function(d, cols) { + do.call(paste, c(lapply(d[cols], as.character), sep = "\r")) + } + all_cols <- c(fixed, content) + c( + same_row_joint = 100 * mean(key(syn, content) == key(df, content)), + whole_row = 100 * mean(key(syn, all_cols) %in% key(df, all_cols)) + ) + } + spec <- .alpha_spec + spec["yield"] <- "outcome/keep" + alpha <- agridat::john.alpha + m1 <- suppressWarnings(.joint_mask(alpha, spec, seed = 4L)) + visible <- .clinical_spec + visible[c("sex", "diagnosis", "stage")] <- "covariate/scramble" + clinical <- .joint_clinical()[, c("arm", "sex", "diagnosis", "stage")] + m2 <- .joint_mask(clinical, visible[names(clinical)], seed = 4L) + cases <- list( + list(alpha, m1, c("rep", "block", "gen", "row", "col"), "yield"), + list(clinical, m2, "arm", c("sex", "diagnosis", "stage")) + ) + for (case in cases) { + expected <- brute(case[[1L]], case[[2L]], case[[3L]], case[[4L]]) + joint <- attr(case[[2L]]@audit, "joint") + got <- stats::setNames(joint$observed, joint$check) + expect_equal(got[["same_row_joint"]], expected[["same_row_joint"]]) + expect_equal(got[["whole_row"]], expected[["whole_row"]]) + } +}) diff --git a/tests/testthat/test-leakage.R b/tests/testthat/test-leakage.R index 4140de8..c471b65 100644 --- a/tests/testthat/test-leakage.R +++ b/tests/testthat/test-leakage.R @@ -87,3 +87,69 @@ test_that(".global_na_pattern_uniqueness on per-row-unique NA = 1", { u <- masque:::.global_na_pattern_uniqueness(df2) expect_gt(u, 0) }) + +test_that(".classify_leakage: content column beyond chance on its own row is MEDIUM", { + args <- list( + role = "covariate", kind = "factor", action = "alias", pii = FALSE, + mode = "collaborate", in_synth = TRUE, alias_status = "aliased", + exact_match_pct = NA_real_, freq_min = 5L, n_unique_levels = 3L, + n_rows = 100L + ) + expect_equal( + do.call(masque:::.classify_leakage, c(args, list( + same_row_excess = 40, same_row_sd = 5 + ))), + "medium" + ) + # Within four SDs of chance, or below ten points, it is not graded. + expect_equal( + do.call(masque:::.classify_leakage, c(args, list( + same_row_excess = 15, same_row_sd = 5 + ))), + "low" + ) + expect_equal( + do.call(masque:::.classify_leakage, c(args, list( + same_row_excess = 9, same_row_sd = 1 + ))), + "low" + ) + # Structural and linked columns match by construction. + design <- modifyList(args, list(role = "design")) + expect_equal( + do.call(masque:::.classify_leakage, c(design, list( + same_row_excess = 60, same_row_sd = 2 + ))), + "low" + ) + expect_equal( + do.call(masque:::.classify_leakage, c(args, list( + same_row_excess = 60, same_row_sd = 2, linked = TRUE + ))), + "low" + ) + # Local mode reports only. + local <- modifyList(args, list(mode = "local")) + expect_equal( + do.call(masque:::.classify_leakage, c(local, list( + same_row_excess = 60, same_row_sd = 2 + ))), + "low" + ) +}) + +test_that(".classify_leakage: a two-valued outcome skips the exact-match rule", { + args <- list( + role = "outcome", kind = "integer", action = "scramble", pii = FALSE, + mode = "collaborate", in_synth = TRUE, alias_status = "passthrough", + exact_match_pct = 52, freq_min = NA_integer_, + n_unique_levels = NA_integer_, n_rows = 200L + ) + expect_equal(do.call(masque:::.classify_leakage, args), "medium") + expect_equal( + do.call(masque:::.classify_leakage, c(args, list( + two_valued = TRUE, same_row_excess = 1.5, same_row_sd = 3.5 + ))), + "low" + ) +}) diff --git a/tests/testthat/test-safety-gate.R b/tests/testthat/test-safety-gate.R index 988c1bd..4c84df8 100644 --- a/tests/testthat/test-safety-gate.R +++ b/tests/testthat/test-safety-gate.R @@ -178,3 +178,49 @@ test_that("local mode is never gated", { ) expect_true(file.exists(out)) }) + +test_that("a joint HIGH finding blocks write_set() and names the check", { + set.seed(3L) + tables <- list( + plots = data.frame( + site = rep(c("S1", "S2", "S3"), each = 20L), + gen = rep(sprintf("G%02d", 1:10), 6L), + yield = round(rnorm(60L, 4, 0.5), 2) + ), + sites = data.frame( + site = rep(c("S1", "S2", "S3"), 2L), + year = rep(2019:2020, each = 3L), + rain = c(410, 520, 300, 455, 610, 395) + ) + ) + roles <- list( + plots = set_role( + set_role( + set_role(propose_roles(tables$plots, mode = "collaborate"), "site", + role = "design", action = "alias" + ), + "gen", role = "treatment" + ), + "yield", role = "outcome", action = "keep" + ), + sites = set_role( + set_role( + set_role(propose_roles(tables$sites, mode = "collaborate"), "site", + role = "design", action = "alias" + ), + "year", role = "design", action = "keep" + ), + "rain", role = "covariate", action = "scramble" + ) + ) + m <- suppressWarnings(mask_set( + tables, roles = roles, mode = "collaborate", seed = 1L, quiet = TRUE + )) + dir <- file.path(tempdir(), "masque-joint-blocked-set") + on.exit(unlink(dir, recursive = TRUE), add = TRUE) + err <- expect_error(write_set(m, dir), class = "masque_blocked_write") + expect_match( + conditionMessage(err), "plots$[same-row joint match]", fixed = TRUE + ) + expect_false(dir.exists(dir)) +}) From 13b5714bdff8419da81e53c00ea980cd04f8e88f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:42:05 +1030 Subject: [PATCH 31/66] style: one- and two-line comments on the audit helpers --- R/leakage.R | 33 +++++++++-------------- tests/testthat/test-fix0160-audit-joint.R | 5 ++-- 2 files changed, 14 insertions(+), 24 deletions(-) diff --git a/R/leakage.R b/R/leakage.R index 9a47503..3c2f791 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -203,9 +203,8 @@ sum(in_cand == 1L & in_ref == 1L) } -# The coarsest replicated design unit within which `x` takes one value, as an -# integer id per row; NULL when `x` varies within every design column. Such a -# column carries one value per unit, so its chance matches move unit by unit. +# Row ids of the coarsest replicated design unit within which `x` takes one +# value (its chance matches move unit by unit); NULL when there is none. .constant_unit <- function(x, df, design_cols) { best <- NULL best_units <- Inf @@ -222,10 +221,8 @@ best } -# Same-row match share of a clone column, its chance share under a random -# permutation of the original (within `groups` when given), the excess and a -# binomial SD of the chance share, counted in `units` when the column is -# constant within them. +# Same-row share, permutation chance share (within `groups`), excess and a +# binomial SD counted in `units`, for one clone column. .same_row_stats <- function(x, y, map, groups = NULL, units = NULL) { x_cmp <- if (is.null(map)) .audit_values(x) else as.character(x) y_cmp <- .audit_values(.audit_original_space(y, map)) @@ -273,11 +270,8 @@ unname(inv[as.character(y)]) } -# Table-level checks an adversary would run, each against a permutation -# baseline: same-row and whole-row joint matches over the retained content -# columns, singleton crosses of visible categorical labels, and the unit-size -# fingerprint (a note, never graded). Design, treatment, id, aliased text and -# linked columns are preserved by construction and only held fixed. +# Table-level checks against permutation baselines. Design, treatment, id, +# aliased text and linked columns match by construction and are held fixed. .joint_audit <- function(df, synth, rec, mode, audit, linked, strata) { collab <- identical(mode, "collaborate") coarsened <- unlist(lapply(rec@coords, function(cr) c(cr$lat, cr$lon)), @@ -429,9 +423,8 @@ ) } -# Observed, mean and SD (in percent of rows) of the same-row joint match over -# `content` and the whole-row match over every column, against clones whose -# content columns are independent permutations of the original's. +# Same-row joint and whole-row shares (percent of rows) of the clone and of +# clones whose content columns are independent permutations of the original. .joint_match_stats <- function(orig, clone, content, groups, fixed) { n <- nrow(orig) same_row <- function(cand) { @@ -481,9 +474,8 @@ })) } -# Clone cells (combinations of the visible categorical labels) holding exactly -# one row whose combination is also a single row in the original, against -# the same count for independent permutations of the original columns. +# Clone singleton cells of the visible labels that are also singletons in the +# original, for the clone and for independent permutations of the columns. .rare_cross_stats <- function(df, synth, cols) { orig <- vector("list", length(cols)) clone <- vector("list", length(cols)) @@ -576,9 +568,8 @@ return("high") } - # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM. A two-valued - # column matches its own cell by chance about half the time; rule 5b - # grades it on the excess over that chance instead. + # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM. Two-valued + # columns match by chance about half the time; rule 5b grades them. if (role == "outcome" && collab && !isTRUE(two_valued) && !is.na(exact_match_pct) && exact_match_pct > 1) { return("medium") diff --git a/tests/testthat/test-fix0160-audit-joint.R b/tests/testthat/test-fix0160-audit-joint.R index de1c111..6f6c768 100644 --- a/tests/testthat/test-fix0160-audit-joint.R +++ b/tests/testthat/test-fix0160-audit-joint.R @@ -1,6 +1,5 @@ -# F10 and F20: the audit grades re-identification on excess over a -# permutation baseline, per column and jointly, and the joint verdicts feed the -# collaborate HIGH gate. +# F10 and F20: excess over a permutation baseline, per column and jointly; the +# joint verdicts feed the collaborate HIGH gate. .joint_roles <- function(df, spec, mode = "collaborate") { r <- propose_roles(df, mode = mode) From c408e2556d39c5ef8d785c15fdb4beceb63c9473 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 11:43:39 +1030 Subject: [PATCH 32/66] test: expect the ladder and all-keep warnings in the audit fixtures by class --- tests/testthat/test-fix0160-audit-joint.R | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/testthat/test-fix0160-audit-joint.R b/tests/testthat/test-fix0160-audit-joint.R index 6f6c768..6a61aab 100644 --- a/tests/testthat/test-fix0160-audit-joint.R +++ b/tests/testthat/test-fix0160-audit-joint.R @@ -167,9 +167,15 @@ test_that("ordinary collaborate clones raise no HIGH from the new rules", { for (case in cases) { for (conditional in c(FALSE, TRUE)) { for (seed in 1:3) { + # A coarsened conditional ladder warns on its own; not under test. expect_no_warning( - m <- .joint_mask( - case[[1L]], case[[2L]], seed = seed, conditional = conditional + m <- withCallingHandlers( + .joint_mask( + case[[1L]], case[[2L]], seed = seed, conditional = conditional + ), + masque_conditional_degraded = function(w) { + invokeRestart("muffleWarning") + } ), class = "masque_high_leakage" ) @@ -268,7 +274,10 @@ test_that("local mode reports the joint verdicts without grading them", { spec <- .alpha_spec spec["yield"] <- "outcome/keep" roles <- .joint_roles(agridat::john.alpha, spec, mode = "local") - m <- mask(agridat::john.alpha, roles, mode = "local", seed = 1L) + expect_warning( + m <- mask(agridat::john.alpha, roles, mode = "local", seed = 1L), + "Every action" + ) a <- audit_mask(m, original = agridat::john.alpha, print = FALSE) srj <- .joint_row_of(a, "same_row_joint") expect_identical(srj$leakage_class, "low") From 6e966bb91fbbc9605c7676045b30d9c1635f03d9 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:05:31 +1030 Subject: [PATCH 33/66] docs: regenerate Rd after merging the four 0.16.0 fix branches --- man/apply_recipe.Rd | 7 ++++- man/audit_mask.Rd | 55 ++++++++++++++++++++++++++++++--- man/clean_table.Rd | 10 +++++- man/conform_table.Rd | 22 ++++++++++++- man/detect_design.Rd | 11 ++++++- man/jitter_coordinates.Rd | 4 ++- man/mask.Rd | 65 +++++++++++++++++++++++++++++++++++---- man/mask_set.Rd | 27 ++++++++++++++-- man/masque.Rd | 32 +++++++++++++++++-- man/propose_roles.Rd | 19 ++++++++++-- man/read_recipe.Rd | 8 ++--- man/read_set.Rd | 3 +- man/reveal_maps.Rd | 13 ++++++-- man/save_recipe.Rd | 6 ++-- 14 files changed, 247 insertions(+), 35 deletions(-) diff --git a/man/apply_recipe.Rd b/man/apply_recipe.Rd index bea837e..e98a88c 100644 --- a/man/apply_recipe.Rd +++ b/man/apply_recipe.Rd @@ -29,11 +29,16 @@ synthetic) against the original data without modifying pipeline code. \details{ Operations applied (in order): \enumerate{ +\item \strong{Re-apply cleaning and conform decisions}: the legalised names and +trimmed labels from \code{\link[=clean_table]{clean_table()}}, then the label merges and type +changes from \code{\link[=conform_table]{conform_table()}} when the clone was built on a +conformance object. \item \strong{Verify integrity} by comparing the NA mask of \code{original} to the SHA-256 fingerprint stored on the recipe (controlled by \code{check_integrity}). \item \strong{Drop} every column whose resolved \code{action} was \code{"drop"} on the -roles table \code{mask()} masked with, in either mode. +roles table \code{mask()} masked with, in either mode. A linked join key +of a set is aliased in the clone and is kept. \item \strong{Subset and reorder} to the columns the recipe knows about. \item \strong{Re-label factors / characters} for any column with a level map held by the recipe (i.e., treatment and categorical covariates in diff --git a/man/audit_mask.Rd b/man/audit_mask.Rd index 5d51f74..1e9df43 100644 --- a/man/audit_mask.Rd +++ b/man/audit_mask.Rd @@ -7,18 +7,23 @@ audit_mask(m, original = NULL, print = TRUE) } \arguments{ -\item{m}{A \code{masque} object from \code{\link[=mask]{mask()}}.} +\item{m}{A \code{masque} object from \code{\link[=mask]{mask()}}, or a \code{masque_set} from +\code{\link[=mask_set]{mask_set()}}.} \item{original}{Optional. Required when \code{m@audit} is NULL (typically -in local mode). Used to recompute exact-match-pct etc. on demand.} +in local mode). Used to recompute exact-match-pct etc. on demand. +The original data frame for a \code{masque}; for a \code{masque_set}, a named +list holding the original table of every table in the set.} \item{print}{Logical; if TRUE (default), print a styled report.} } \value{ -The audit data.table, returned invisibly. +The audit data.table, returned invisibly. For a \code{masque_set}, a +named list of per-table audit data.tables, printed table by table. } \description{ -Returns a per-column audit data.table and prints a severity-grouped report. +Returns a per-column audit data.table, with table-level checks attached, and +prints a severity-grouped report. Auto-runs in \code{mode = "collaborate"} at \code{\link[=mask]{mask()}} time and stores the result on the masque object (\code{m@audit}); for local-mode audits or explicit re-audits, pass the original data frame via \code{original}. @@ -39,8 +44,38 @@ original cell (numeric only; cell-by-cell). \item \code{na_pattern_uniqueness}: fraction of rows in the original with a globally unique NA pattern (one number per data frame, repeated on every row). -\item \code{alias_status}: \code{aliased}, \code{passthrough}, or \code{dropped}. +\item \code{alias_status}: \code{aliased}, \code{permuted}, \code{passthrough}, or +\code{dropped}. \item \code{notes}: short human summary. +\item \code{same_row_pct}: percentage of rows where the clone value equals +the original value in the same row, read back through the level map +(categorical, logical, date and two-valued numeric columns). +\item \code{baseline_pct}: the same share expected by chance, under a random +permutation of the original column: \eqn{100 \sum_k p_k^2} over its +level frequencies (within the conditioning strata for a scrambled +numeric column of a conditional clone). +\item \code{excess_pct}: \code{same_row_pct - baseline_pct}. +} + +The table-level checks sit in \code{attr(audit, "joint")}, a data.table with +one row per check (\code{check}, \code{cols}, \code{observed}, \code{baseline}, \code{excess}, +\code{sd}, \code{leakage_class}, \code{notes}). Each baseline comes from independent +permutations of the original content columns (outcomes, covariates, +dates, kept text), drawn under a fixed internal seed that leaves the +caller's random number stream untouched; design, treatment, id, aliased +text and linked columns are held fixed: + +\itemize{ +\item \code{same_row_joint}: rows whose every content column equals the +original row. +\item \code{whole_row}: clone rows equal to some original row on every +retained column. +\item \code{rare_cross}: in the cross of visible (not aliased) categorical +covariates and outcomes, clone cells holding one row whose +combination is also a single row in the original. +\item \code{unit_sizes}: a note, never graded, when design units differ in +size, since the multiset of sizes can link a clone to a public trial +list. } Classification heuristics: @@ -53,6 +88,16 @@ Classification heuristics: \item Outcome with exact-match-pct > 1\\% in collaborate -> \code{medium}. \item Numeric covariate with exact-match-pct > 5\\% in collaborate -> \code{medium}. +\item Two-valued numeric columns skip the two exact-match rules: a 0/1 +column matches its own cell about half the time by chance. +\item Content column whose \code{excess_pct} is at least 10 points and four +binomial standard deviations in collaborate -> \code{medium}. +\item Joint match excess of at least 20 points and four permutation +standard deviations -> \code{high} (5 points -> \code{medium}); rare-cross +excess of at least 3 cells and four standard deviations -> \code{high} +(1 cell and three -> \code{medium}). In collaborate mode these verdicts +raise \code{masque_high_leakage} and block package-managed writers; in +local mode they are reported in the notes and graded \code{low}. \item Ignore column retained in local -> \code{low} (informational). } diff --git a/man/clean_table.Rd b/man/clean_table.Rd index cb99b2e..7503fe8 100644 --- a/man/clean_table.Rd +++ b/man/clean_table.Rd @@ -33,7 +33,10 @@ column whose name changed (empty if none); values were trimmed, each a named character \code{original -> clean}; \item \code{near_duplicates} - a data.table of report-only label pairs (\code{col}, \code{a}, \code{b}, \code{kind}) that look like typos but were left -untouched; +untouched. Labels that differ only inside a run of digits +(\code{"G01"} and \code{"G02"}, \code{"Plot12"} and \code{"Plot13"}) are different +codes and are not listed. The printed report shows the first 20 +pairs and the count, and a recipe keeps the same; \item \code{mode} - the \code{clean} mode applied. } } @@ -50,6 +53,11 @@ labels that look alike is a judgement call; \code{\link[=conform_table]{conform_ The corrections are returned alongside the cleaned data, so \code{\link[=mask]{mask()}} records them in the recipe and \code{\link[=apply_recipe]{apply_recipe()}} re-applies the same cleaning to the original. + +Two input columns with the same name are refused with a +\code{masque_duplicate_names_refusal} error: the recipe records renamed +columns by their original name, so the two could not be told apart on +the round trip. Rename them first. } \examples{ df <- data.frame( diff --git a/man/conform_table.Rd b/man/conform_table.Rd index 6877092..50fb231 100644 --- a/man/conform_table.Rd +++ b/man/conform_table.Rd @@ -45,11 +45,16 @@ An object of class \code{masque_conformance}: a list with \item \code{cleaning} - the \code{masque_cleaning} record from \code{\link[=clean_table]{clean_table()}}; \item \code{merges} - one row per near-duplicate pair considered (\code{col}, \code{from}, \code{to}, \code{n_from}, \code{n_to}, \code{applied}, \code{reason}); +\item \code{merge_map} - the merges as applied to the data (\code{col}, \code{from}, +\code{to}), one row per merged-away label, each \code{to} a label left in +the column. This is what \code{\link[=mask]{mask()}} records and \code{\link[=apply_recipe]{apply_recipe()}} +replays; \item \code{types} - one row per column whose storage was considered (\code{col}, \code{from}, \code{to}, \code{applied}, \code{reason}); \item \code{assumptions} - every decision as a plain sentence (\code{col}, \code{assumption}, \code{applied}), which is the thing to read; -\item \code{modes} - the modes applied. +\item \code{modes} - the modes applied; +\item \code{max_levels} - the cardinality threshold used. } } \description{ @@ -76,6 +81,21 @@ Two labels that differ by an edit are merged only when one spelling is clearly more common; a column is converted only when every value parses; numeric columns keep their storage. Anything not changed for these reasons is reported as a \verb{not applied} row with the reason. + +Merges follow chains: if \code{"Nsw"} merges onto \code{"nsw"} and \code{"nsw"} onto +\code{"NSW"}, both end on \code{"NSW"}. A label whose merges end on two different +labels (\code{"colur"} beside \code{"color"} and \code{"colour"} that occur equally +often) is not merged; it is reported, and a \code{masque_conform_ambiguous_merge} +warning is raised under \code{merge_labels = "auto"}. Codes that differ only +inside a run of digits (\code{"G01"} and \code{"G02"}) are never treated as +near-duplicates. Which columns hold categories is decided the same way +whatever \code{types} is, so \code{types = "off"} still merges labels and leaves +storage as it is. + +Pass the result to \code{\link[=mask]{mask()}} (or as a table of \code{\link[=mask_set]{mask_set()}}): the recipe +records the merges and type changes, and \code{\link[=apply_recipe]{apply_recipe()}} replays them on +the original so a pipeline built on the conformed clone sees the same +labels and storage. } \examples{ df <- data.frame( diff --git a/man/detect_design.Rd b/man/detect_design.Rd index a403a41..9dcf4ef 100644 --- a/man/detect_design.Rd +++ b/man/detect_design.Rd @@ -60,7 +60,7 @@ evidence gives an explicit uncertain result, never a guessed single trial. Supply \code{env} to define the environment basis, or use \code{env = FALSE} to run the pre-0.9 whole-table path exactly. -After the scope step, the pooled legacy detector runs six independent +Before the scope step, the pooled legacy detector runs six independent design rules. Each returns a score in \eqn{[0, 1]}. The highest-scoring class above \code{threshold} is one of \code{"CRD"}, \code{"RCBD"}, \code{"IBD/alpha-lattice"}, \code{"row-column"}, \code{"split-plot"}, \code{"factorial"}, @@ -70,6 +70,15 @@ experiment groups are diagnostics only. Dense connectivity calculations that exceed the package safety bound are reported as \code{not_computed}. None of these diagnostics proves the original randomisation protocol. +When scope resolves, \code{block_cols} is the block basis the environments +agree on, or empty when they disagree; an environment column is never a +block. \code{evidence$block_revision} records the change. + +For a column without a treatment-pattern name, a treatment hint from the +rules is auto-applied only when the column is crossed with a design +column. A column with one value per level of a design column, or with an +outcome-pattern name, is a covariate and never a candidate. + The detector never edits \code{df}. Its job is to recommend a role assignment, surface the evidence, and (optionally) draw a sanity check via \code{\link[=plot]{plot()}}. diff --git a/man/jitter_coordinates.Rd b/man/jitter_coordinates.Rd index 932f255..762b4f5 100644 --- a/man/jitter_coordinates.Rd +++ b/man/jitter_coordinates.Rd @@ -50,7 +50,9 @@ A site that cannot be placed on land is set to \code{NA} on both axes, with a warning: masque never emits the true coordinate as though it were masked.} \item{seed}{Optional integer seed for reproducibility. The caller's RNG state -is preserved.} +is preserved. Several published jitters of the same point average toward +it: keep the seed (a \code{\link[=mask]{mask()}} recipe records it) and do not publish more +than one jittered release of the same sites.} \item{.group}{Internal. A pre-computed grouping vector of length \code{nrow(df)}, used by \code{\link[=mask]{mask()}} to group by the \emph{original} table's site diff --git a/man/mask.Rd b/man/mask.Rd index 18c7d7c..d3fd29a 100644 --- a/man/mask.Rd +++ b/man/mask.Rd @@ -15,15 +15,19 @@ mask( ladder = c("hierarchy", "levels"), coords = NULL, events = NULL, + grain = TRUE, allow_unmasked_coords = FALSE, quiet = FALSE, .shared_maps = list(), + .cleaning = NULL, ... ) } \arguments{ \item{df}{A data frame, or the \code{masque_conformance} object returned by -\code{\link[=conform_table]{conform_table()}}, whose \code{data} element is then masked.} +\code{\link[=conform_table]{conform_table()}}, whose \code{data} element is then masked. Its cleaning +record and its merges and type changes go on the recipe, so +\code{\link[=apply_recipe]{apply_recipe()}} replays them; \code{clean} is not applied a second time.} \item{roles}{A roles table from \code{\link[=propose_roles]{propose_roles()}} (possibly edited). Tables from masque <= 0.5.0 are upgraded with a deprecation @@ -32,7 +36,13 @@ warning; see \code{\link[=roles_validate]{roles_validate()}}.} \item{mode}{Either \code{"local"} or \code{"collaborate"}. When omitted, inherit \code{attr(roles, "mode")}, falling back to \code{"local"} for a roles table with no mode provenance. An explicit collaborate-to-local downgrade raises a -classed \code{masque_mode_downgrade} warning.} +classed \code{masque_mode_downgrade} warning. A table of fewer than five rows +in which any column would be re-simulated (a scrambled covariate, +outcome, date or text column, or an \code{events} spec) is refused in +collaborate mode with \code{masque_too_few_rows_refusal}, because such a +clone is the original values resampled; local mode warns with +\code{masque_too_few_rows}. Kept, aliased and dropped columns are not +re-simulated, so a short lookup table of those masks in either mode.} \item{seed}{Optional integer for reproducibility.} @@ -107,14 +117,21 @@ trace in the clone.} latitude/longitude pairs and each is coarsened in place by an on-land jitter (see \code{\link[=jitter_coordinates]{jitter_coordinates()}}) instead of being copula-scrambled into implausible locations. A pair is a named vector \code{c(lat = "lat_col", lon = "lon_col")} or a named list that also carries jitter parameters -(\code{by}, \code{method}, \code{min_km}, \code{max_km}, \code{sd_km}, \code{on_land}); pass several -pairs as a list. Defaults to a donut of 5-20 km on land. One displacement +(\code{by}, \code{method}, \code{min_km}, \code{max_km}, \code{sd_km}, \code{on_land}); a single pair +may be passed bare, and several pairs as a list of pairs. Defaults to a +donut of 5-20 km on land. A declared column must have the action \code{keep} +in \code{roles}, or still carry the action \code{\link[=propose_roles]{propose_roles()}} proposed, which +the declaration replaces with \code{keep} (noted on the recipe); an +explicitly chosen \code{drop} or \code{scramble} is refused with a +\code{masque_coords_action_refusal} error. One displacement is drawn per \strong{site} and broadcast to that site's rows: \code{by} names the columns of \code{df} that identify a site, and is omitted when rows sharing an identical input coordinate already identify one. A declared pair always survives masking, coarsened; the recipe records the parameters and the site grouping it was coarsened under, and \code{\link[=apply_recipe]{apply_recipe()}} retargets to the real -coordinates.} +coordinates. Several published jitters of the same point average toward +it: keep the seed, which the recipe records, and do not publish more than +one jittered release of the same sites.} \item{events}{Optional linked time-to-event outcomes. One spec per group of linked events, \code{list(time = c(...), status = c(...))}, with the time and @@ -127,12 +144,44 @@ gaps between successive events, the rows' status patterns are permuted re-simulated within each status pattern by the empirical copula, then summed back into times. Times therefore stay ordered, a gap that every row of a pattern holds at zero stays zero, and the count of each status -pattern is exact. A pattern with fewer than five rows draws each of its +pattern is exact. A summed final time is capped at the longest final +time observed in the rows its gaps were drawn from (the pattern, within +treatment when conditional), so no clone row runs past the observed +follow-up. Only the final time and the times tied to it move; a row +whose earlier time already passes the cap is scaled down as a whole, +which keeps its order and its zero gaps. The recipe records how many +rows were capped. A pattern with fewer than five rows draws each of its free gaps from every pattern in which that gap is free, with a \code{masque_events_pooled} warning; when no other pattern has the gap free, \code{mask()} stops. No hazard model is fitted and no covariate-to-event relation is kept.} +\item{grain}{Single logical (default \code{TRUE}). Synthesise each column at +its \strong{grain}: a scrambled covariate, outcome or date column, or an +aliased categorical covariate, that holds at most one distinct +non-missing value within every unit of a retained design column, or of a +pair of them (\code{site:year}), is re-simulated once per unit and each unit's +new value is written to that unit's rows. The grain is the coarsest such +unit with +at least two units and at least two rows per unit on average; a unit with +no value stays missing. Numeric columns of one grain are drawn jointly +through the copula on the unit-level table (with collaborate-mode +jitter against the unit-level values), and categorical and date columns +are permuted across units, then aliased where their action is \code{alias}. +The clone therefore keeps one value per unit, the frequencies of a +categorical across units, and the correlation among columns of the same +grain. It does \strong{not} keep the correlation between a grained column +and row-level columns, and grained columns are not conditioned on +treatment when \code{conditional = TRUE}. In local mode the unit values are +drawn from the original unit values. In collaborate mode a grain with +fewer than five units is synthesised row by row instead, with a +\code{masque_grain_small_units} warning, because resampling that few real +values would disclose them. Each grain is recorded on the recipe +(\code{grain}) with a line in its warnings. \code{FALSE} synthesises every column +row by row, as masque 0.15 did; detected grains are still recorded. +Coordinates declared to \code{coords}, \code{events} columns and linked columns +are never grained.} + \item{allow_unmasked_coords}{Single logical. A masked table must not carry a real coordinate, so by default \code{mask()} \strong{stops} when a column whose name says coordinate (\code{gps}, \code{lat}, \code{lon}, \code{easting}, ...) would be @@ -152,6 +201,10 @@ raised.} \code{original -> alias} level maps for cross-table linked columns, set by \code{\link[=mask_set]{mask_set()}}. Not for direct use.} +\item{.cleaning}{Internal. The cleaning record of a table \code{\link[=mask_set]{mask_set()}} +has already cleaned; \code{mask()} then skips its own cleaning and stores +this record on the recipe. Not for direct use.} + \item{...}{Must be empty. An unused argument, for example a misspelled name, is an error.} } diff --git a/man/mask_set.Rd b/man/mask_set.Rd index 442d8be..e976f93 100644 --- a/man/mask_set.Rd +++ b/man/mask_set.Rd @@ -14,12 +14,17 @@ mask_set( alias_names = FALSE, conditional = FALSE, ladder = c("hierarchy", "levels"), - quiet = FALSE + quiet = FALSE, + sheets = NULL, + info = NULL, + ... ) } \arguments{ \item{input}{A folder path, an \code{.xlsx} / \code{.xls} path, or a named list -of data frames - anything \code{\link[=read_set]{read_set()}} accepts.} +of data frames - anything \code{\link[=read_set]{read_set()}} accepts. A list element may also +be a \code{\link[=conform_table]{conform_table()}} result: its cleaning record and conform +decisions are used for that table and recorded on its recipe.} \item{roles}{Optional named list of roles tables, one per input table (names must match). When \code{NULL} (default), \code{\link[=propose_roles]{propose_roles()}} runs on @@ -53,7 +58,23 @@ the treatment-to-outcome relationship survives the clone. See \item{ladder}{Drop order of the conditioning ladder, \code{"hierarchy"} (default) or \code{"levels"}; passed through to each \code{\link[=mask]{mask()}} call.} -\item{quiet}{Logical; suppress the link / hygiene report.} +\item{quiet}{Logical; suppress the link / hygiene report, including the +hint that a table looks like documentation.} + +\item{sheets}{For an Excel workbook, the sheets to read (default: all); +passed to \code{\link[=read_set]{read_set()}}.} + +\item{info}{Optional character vector of table names (sheet names, or +file names without extension) to set aside as documentation, for +example a data dictionary. They are set aside before masking: not +masked, not in the synthetic set, not written by \code{\link[=write_set]{write_set()}}, and +their content is not kept. Their names are +recorded on the recipe bundle. A table that looks like documentation +and is not named here raises a hint (a message of class +\code{masque_info_hint}); the hint decides nothing.} + +\item{...}{Must be empty: any argument here is refused with +\code{masque_unused_arg_refusal}.} } \value{ A \code{masque_set} S7 object. Use \code{\link[=synthetic]{synthetic()}} for the named list diff --git a/man/masque.Rd b/man/masque.Rd index c8676c3..fbf35f2 100644 --- a/man/masque.Rd +++ b/man/masque.Rd @@ -17,7 +17,11 @@ masque( ask = interactive(), overwrite = FALSE, quiet = FALSE, - allow_high = FALSE + allow_high = FALSE, + detect = TRUE, + sheets = NULL, + info = NULL, + ... ) } \arguments{ @@ -33,7 +37,13 @@ review is skipped.} \code{.xlsx} with \code{writexl}). For a set, a folder (one CSV per table) or an \code{.xlsx} workbook. When \code{NULL} (default) nothing is written.} -\item{mode}{Either \code{"local"} (default) or \code{"collaborate"}.} +\item{mode}{Either \code{"local"} or \code{"collaborate"}. \code{"local"} makes an +owner-only development copy (real labels may remain); \code{"collaborate"} +makes a copy meant to be shared, with labels aliased, numbers jittered, +free text dropped and the leakage audit run. When omitted and \code{roles} +are supplied, the mode the roles were proposed for is used, exactly as +in \code{\link[=mask]{mask()}} and \code{\link[=mask_set]{mask_set()}}; with no \code{roles}, the proposal is made for +\code{"local"}.} \item{seed}{Optional integer for reproducibility.} @@ -65,6 +75,21 @@ the collaborate-mode audit flagged HIGH leakage, the write is refused. Pass \code{TRUE} to write anyway after your own review; the override raises a \code{masque_high_override} warning and is recorded in the recipe's warnings, so the exception stays auditable.} + +\item{detect}{Logical (default \code{TRUE}). Passed to \code{\link[=propose_roles]{propose_roles()}} when +roles are proposed: \code{FALSE} skips design detection and proposes from +column names and types alone.} + +\item{sheets}{For an Excel workbook, the sheets to read (default: all). +See \code{\link[=mask_set]{mask_set()}}.} + +\item{info}{For a set, names of documentation tables to set aside: not +masked and not written. See \code{\link[=mask_set]{mask_set()}}.} + +\item{...}{Further arguments for the masking verb: \code{\link[=mask]{mask()}} for one +table (for example \code{coords}, \code{events}, \code{allow_unmasked_coords}), +\code{\link[=mask_set]{mask_set()}} for a set (for example \code{links}). An argument the verb +does not take is refused (\code{masque_unused_arg_refusal}).} } \value{ A \code{masque} object (single-table input) or a \code{masque_set} @@ -125,5 +150,6 @@ head(synthetic(m)) } \seealso{ \code{\link[=mask]{mask()}}, \code{\link[=mask_set]{mask_set()}}, \code{\link[=propose_roles]{propose_roles()}}, \code{\link[=set_role]{set_role()}}, -\code{\link[=write_set]{write_set()}}, \code{\link[=save_recipe]{save_recipe()}}. +\code{\link[=apply_recipe]{apply_recipe()}} and \code{\link[=unmask]{unmask()}} for the round trip, \code{\link[=save_recipe]{save_recipe()}} to +keep the recipe, \code{\link[=write_set]{write_set()}}. } diff --git a/man/propose_roles.Rd b/man/propose_roles.Rd index 604a593..bc35d92 100644 --- a/man/propose_roles.Rd +++ b/man/propose_roles.Rd @@ -93,16 +93,21 @@ action \code{scramble} (row permutation). \item ID-pattern names (\verb{\\\\bid\\\\b}, \verb{_id$}, \verb{^id_}) -> role \code{id}; kept in local mode, dropped in collaborate mode. \item Design-pattern names (\code{rep}, \code{block}, \code{row}, \verb{col(umn)?}, -\code{range}, \verb{plot(no)?}, \code{site}, \verb{env(ironment)?}, \code{trial}, \code{year}, -\code{season}, \code{colrep}, \code{tos}) -> role \code{design}, action \code{keep}. +\code{range}, \verb{plot(no)?}, \code{site}, \code{site_name}, \code{trial_site}, +\verb{env(ironment)?}, \code{trial}, \code{year}, \code{season}, \code{colrep}, \code{tos}; the +two-word names also spaced or dotted, as in \verb{Site Name} and +\code{Site.Name}) -> role \code{design}, action \code{keep}. \item Treatment-pattern names (\code{treatment}, \code{variety}, \code{cultivar}, \code{genotype}, \verb{^trt}, \verb{^dose}) -> role \code{treatment}; kept in local mode, aliased in collaborate mode. +\item Numeric columns with an outcome-pattern name (\code{yield}, \code{yield_t_ha}, +\code{Yield..t.ha.}, \code{grain_yield}, \code{response}, \code{outcome}) -> role +\code{outcome}, action \code{scramble}. \item Character columns with > 50\% unique values on non-NA -> role \code{text}; kept in local mode, dropped in collaborate mode. \item Unsupported classes -> role \code{other}, action \code{keep}, with a note. \item Everything else -> role \code{covariate}, action \code{scramble}. Re-role -response variables as \code{outcome}. +other response variables as \code{outcome}. } No outcome is required: with no column roled \code{outcome}, the copula @@ -114,6 +119,14 @@ default (\code{detect = TRUE}) and applies the detected design's the default action for any promoted column. The design summary is stashed as \code{attr(roles, "design")}. Pass \code{detect = FALSE} for the name-only heuristic. + +For a column without a treatment-pattern name, a treatment hint from the +design rules is applied only when the column is crossed with a design +column: a design-pattern name, a column roled \code{design}, or a detected +environment. A column with one value per level of a design +column is a unit-level covariate and is never promoted. Hints that fail +are listed in \code{attr(roles, "design")@recommended_roles} with +\code{auto_apply = FALSE} and the reason. } \section{Multi-environment trials}{ diff --git a/man/read_recipe.Rd b/man/read_recipe.Rd index 92535d9..b0f7e01 100644 --- a/man/read_recipe.Rd +++ b/man/read_recipe.Rd @@ -10,13 +10,13 @@ read_recipe(path) \item{path}{File path.} } \value{ -A \code{masque_recipe} object. +A \code{masque_recipe}, or a \code{masque_recipe_set} bundle. } \description{ Loads a recipe written by \code{\link[=save_recipe]{save_recipe()}}. Validates that the file -contains a \code{masque_recipe} and informs (does not error) if the recipe -was written by a different package version than the one currently -installed. +contains a \code{masque_recipe} or a \code{masque_recipe_set} bundle and informs +(does not error) if it was written by a different package version than +the one currently installed. Bundles are read from masque 0.16.0 on. } \examples{ r <- propose_roles(iris) diff --git a/man/read_set.Rd b/man/read_set.Rd index 1460271..1d38e92 100644 --- a/man/read_set.Rd +++ b/man/read_set.Rd @@ -11,7 +11,8 @@ read_set(input, sheets = NULL, pattern = NULL) of data frames.} \item{sheets}{For an Excel workbook, an optional character vector of -sheet names to read (default: all sheets).} +sheet names to read (default: all sheets). Any other input with +\code{sheets} set is refused (\code{masque_bad_sheets_refusal}).} \item{pattern}{For a folder, an optional regular expression to select files (default: all \code{.csv} / \code{.tsv} / \code{.fst}).} diff --git a/man/reveal_maps.Rd b/man/reveal_maps.Rd index 7888c91..5d6b973 100644 --- a/man/reveal_maps.Rd +++ b/man/reveal_maps.Rd @@ -4,10 +4,15 @@ \alias{reveal_maps} \title{Reveal the level maps held inside a recipe} \usage{ -reveal_maps(rec) +reveal_maps(rec, table = NULL) } \arguments{ -\item{rec}{A \code{masque_recipe} object (e.g., from \code{recipe(m)}).} +\item{rec}{A \code{masque_recipe} object (e.g., from \code{recipe(m)}), or the +\code{masque_recipe_set} bundle of a \code{\link[=mask_set]{mask_set()}} result.} + +\item{table}{For a bundle, the names of the tables whose maps to reveal. +\code{NULL} (default) reveals every table. Must be \code{NULL} for a single +recipe.} } \value{ \code{rec}, invisibly. @@ -16,7 +21,9 @@ reveal_maps(rec) Explicit, audited reveal: prints each \code{original -> synthetic} level map held by the recipe. \strong{Use sparingly.} Recipe maps are at least as sensitive as the original data; printing them defeats the redaction -built into \code{\link[=print]{print()}} and \code{\link[=summary]{summary()}}. +built into the \code{print()}, \code{format()}, \code{str()} and \code{knitr::knit_print()} +methods of recipes, recipe bundles, \code{masque} and \code{masque_set} objects. +\code{saveRDS()} and \code{dput()} write the maps by design. } \examples{ r <- propose_roles(iris) diff --git a/man/save_recipe.Rd b/man/save_recipe.Rd index 33bda4c..be23f7d 100644 --- a/man/save_recipe.Rd +++ b/man/save_recipe.Rd @@ -7,9 +7,11 @@ save_recipe(rec, path, include_simulator = FALSE) } \arguments{ -\item{rec}{A \code{masque_recipe} object, e.g. from \code{recipe(m)}.} +\item{rec}{A \code{masque_recipe}, or the \code{masque_recipe_set} bundle of a +\code{\link[=mask_set]{mask_set()}} result, e.g. from \code{recipe(m)}.} -\item{path}{File path. By convention, \code{.rds} extension.} +\item{path}{File path ending in \code{.rds} (any case); other extensions are +refused.} \item{include_simulator}{Logical. Has no effect; the recipe is always written without simulator state.} From a47e92d597cf20562bb88c6070241df9c918e378 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:05:31 +1030 Subject: [PATCH 34/66] test: fixtures clear the five-row floor and no longer align covariates with Rep Merged interaction: the engine's five-row refusal and grain detection fired on fixtures written by the other branches (4-row link tables, covariates cycling in step with Rep). --- tests/testthat/test-audit_mask.R | 4 ++-- tests/testthat/test-fix0160-conform-recipe.R | 2 +- tests/testthat/test-fix0160-front-door.R | 4 ++-- tests/testthat/test-fix0160-info-sheets.R | 3 ++- tests/testthat/test-recipe-apply.R | 2 +- tests/testthat/test-safety-gate.R | 2 +- tests/testthat/test-typed-refusals.R | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/testthat/test-audit_mask.R b/tests/testthat/test-audit_mask.R index 5199b58..1ab9e54 100644 --- a/tests/testthat/test-audit_mask.R +++ b/tests/testthat/test-audit_mask.R @@ -64,7 +64,7 @@ test_that("audit_mask flags PII-retained as HIGH", { set.seed(0) df <- data.frame( Rep = rep(1:4, 25), - contact_person = factor(rep(c("A", "B"), 50)), # PII pattern -> auto ignore + contact_person = factor(rep(c("A", "B"), each = 50)), # PII pattern -> auto ignore yield = rnorm(100), stringsAsFactors = FALSE ) @@ -107,7 +107,7 @@ test_that("Collaborate mode warns at mask() time on HIGH leakage", { set.seed(0) df <- data.frame( Rep = rep(1:4, 25), - contact_email = factor(rep(c("a", "b"), 50)), + contact_email = factor(rep(c("a", "b"), each = 50)), yield = rnorm(100), stringsAsFactors = FALSE ) diff --git a/tests/testthat/test-fix0160-conform-recipe.R b/tests/testthat/test-fix0160-conform-recipe.R index 87e5dbc..a42dc1c 100644 --- a/tests/testthat/test-fix0160-conform-recipe.R +++ b/tests/testthat/test-fix0160-conform-recipe.R @@ -109,7 +109,7 @@ test_that("mask_set() takes a conformance object per table and replays it", { raw <- state_raw() cf <- state_cf(raw) other <- data.frame( - genotype = sprintf("G%d", 1:4), height = c(80, 95, 102, 88), + genotype = sprintf("G%d", 1:5), height = c(80, 95, 102, 88, 91), stringsAsFactors = FALSE ) ro <- propose_roles(other, mode = "local", detect = FALSE) diff --git a/tests/testthat/test-fix0160-front-door.R b/tests/testthat/test-fix0160-front-door.R index c42609d..0d55d1c 100644 --- a/tests/testthat/test-fix0160-front-door.R +++ b/tests/testthat/test-fix0160-front-door.R @@ -65,8 +65,8 @@ door_roles <- function(df, mode) { door_set <- function() { df <- door_trial() lines <- data.frame( - genotype = c("Scope", "Compass", "Spartacus", "Trojan"), - height = c(80, 95, 102, 88), stringsAsFactors = FALSE + genotype = c("Scope", "Compass", "Spartacus", "Trojan", "Mace"), + height = c(80, 95, 102, 88, 91), stringsAsFactors = FALSE ) list(trial = df, lines = lines) } diff --git a/tests/testthat/test-fix0160-info-sheets.R b/tests/testthat/test-fix0160-info-sheets.R index 4e7cf65..60bcc24 100644 --- a/tests/testthat/test-fix0160-info-sheets.R +++ b/tests/testthat/test-fix0160-info-sheets.R @@ -167,7 +167,8 @@ test_that("a sheet that looks like documentation gets a hint, not a decision", { ) expect_false(any(grepl("documentation", msgs))) plain <- list(trial = tabs$trial, lines = data.frame( - genotype = c("Scope", "Compass", "Mace"), height = c(80, 95, 102) + genotype = c("Scope", "Compass", "Mace", "Trojan", "Spartacus"), + height = c(80, 95, 102, 88, 91) )) msgs <- testthat::capture_messages( mask_set(plain, mode = "local", seed = 1) diff --git a/tests/testthat/test-recipe-apply.R b/tests/testthat/test-recipe-apply.R index aaac44a..76b2923 100644 --- a/tests/testthat/test-recipe-apply.R +++ b/tests/testthat/test-recipe-apply.R @@ -43,7 +43,7 @@ test_that("logical covariate aliases round-trip back to logical values", { n <- 90 df <- data.frame( Rep = rep(1:3, length.out = n), - sprayed = rep(c(TRUE, FALSE, NA), length.out = n), + sprayed = rep(c(TRUE, FALSE, NA), each = n / 3), yield = rnorm(n), stringsAsFactors = FALSE ) diff --git a/tests/testthat/test-safety-gate.R b/tests/testthat/test-safety-gate.R index 33f5970..f10b5f8 100644 --- a/tests/testthat/test-safety-gate.R +++ b/tests/testthat/test-safety-gate.R @@ -6,7 +6,7 @@ high_fixture <- function() { set.seed(0) df <- data.frame( Rep = rep(1:4, 25), - contact_email = factor(rep(c("a", "b"), 50)), + contact_email = factor(rep(c("a", "b"), each = 50)), yield = rnorm(100), stringsAsFactors = FALSE ) diff --git a/tests/testthat/test-typed-refusals.R b/tests/testthat/test-typed-refusals.R index 7a9992f..8f38a9c 100644 --- a/tests/testthat/test-typed-refusals.R +++ b/tests/testthat/test-typed-refusals.R @@ -189,7 +189,7 @@ test_that("unmask() refuses ambiguous atomic input with multiple level maps and df <- data.frame( Rep = rep(1:4, 25), Genotype = factor(rep(LETTERS[1:5], each = 20)), - cov_c = factor(rep(c("alpha", "beta"), 50)), + cov_c = factor(rep(c("alpha", "beta"), each = 50)), yield = rnorm(100, mean = 10, sd = 2), stringsAsFactors = FALSE ) From c6d3308ff2aad3d1c787bd5beb191c10ddd6395f Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:13:48 +1030 Subject: [PATCH 35/66] feat: make the namespace data.table-aware package-wide Adds .datatable.aware and data.table (>= 1.15.0) to Imports. Column subsets that relied on data.frame semantics go through .cols(), which keeps the input class. Removes the local opt-ins in conform_table.R. Grain constancy now uses dt[, j, by =]. --- DESCRIPTION | 2 +- R/conform_table.R | 3 -- R/detect_environment.R | 16 ++++--- R/detect_plot.R | 4 +- R/detect_rules.R | 2 +- R/events.R | 12 +++-- R/grain.R | 18 ++++--- R/jitter_coordinates.R | 4 +- R/leakage.R | 2 +- R/mask.R | 6 +-- R/masque-package.R | 3 ++ R/recipe_apply.R | 2 +- R/roles_actions.R | 2 +- R/synthesise_geospatial.R | 2 +- R/synthesise_numeric_conditional.R | 2 +- R/utils_data_table.R | 9 ++++ tests/testthat/test-coord-site-invariant.R | 6 +-- tests/testthat/test-fix0160-datatable.R | 55 ++++++++++++++++++++++ 18 files changed, 112 insertions(+), 38 deletions(-) create mode 100644 tests/testthat/test-fix0160-datatable.R diff --git a/DESCRIPTION b/DESCRIPTION index 7b0e4eb..6f84f92 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -28,7 +28,7 @@ Depends: R (>= 4.2.0) Imports: cli, - data.table, + data.table (>= 1.15.0), digest, Matrix, S7, diff --git a/R/conform_table.R b/R/conform_table.R index ca519f5..20b1362 100644 --- a/R/conform_table.R +++ b/R/conform_table.R @@ -198,8 +198,6 @@ utils::globalVariables(c( MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing .conform_merges <- function(df, near_dups, mode, cat_cols) { - # data.table syntax in this frame only (data.table >= 1.15.0). - .datatable.aware <- TRUE if (identical(mode, "off") || is.null(near_dups) || !nrow(near_dups)) { return(.empty_merges()) } @@ -250,7 +248,6 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing # The effective merge map: each source follows its merges to a label nothing # is merged away from. A source that reaches two such labels is not merged. .resolve_merges <- function(merges) { - .datatable.aware <- TRUE none <- list( merges = merges[, setdiff(names(merges), "candidate"), drop = FALSE], map = .empty_merge_map(), diff --git a/R/detect_environment.R b/R/detect_environment.R index 6c39585..1091598 100644 --- a/R/detect_environment.R +++ b/R/detect_environment.R @@ -99,7 +99,9 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 ) } - all_missing <- env[vapply(df[env], function(x) all(is.na(x)), logical(1L))] + all_missing <- env[vapply( + .cols(df, env), function(x) all(is.na(x)), logical(1L) + )] if (length(all_missing) > 0L) { .abort_invalid_environment( "Environment column{?s} {?is/are} all-missing: {.field {all_missing}}." @@ -114,7 +116,7 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 return(factor(df[[cols]], exclude = NA)) } args <- c( - unname(df[cols]), + unname(.cols(df, cols)), list(drop = TRUE, lex.order = TRUE, sep = ":") ) do.call(interaction, args) @@ -160,7 +162,7 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 .evaluate_environment_candidate <- function(df, candidate, treatment_key, allow_no_treatment = FALSE) { - env_complete <- stats::complete.cases(df[candidate$cols]) + env_complete <- stats::complete.cases(.cols(df, candidate$cols)) coverage <- mean(env_complete) env_key <- .interaction_key(df, candidate$cols) n_env <- nlevels(env_key) @@ -412,7 +414,7 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 rec[[field]] <- defaults[[field]] } } - rec[names(.empty_role_recommendations())] + .cols(rec, names(.empty_role_recommendations())) } .environment_role_recommendations <- function(df, cols, confidence, @@ -421,7 +423,7 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 if (length(cols) == 0L) { return(.empty_role_recommendations()) } - kinds <- vapply(df[cols], col_kind, character(1L)) + kinds <- vapply(.cols(df, cols), col_kind, character(1L)) categorical <- kinds %in% .categorical_kinds() # `preserve_only`: a suspected environment is kept unchanged in both modes; # only a high-confidence environment is aliased in collaborate mode. @@ -660,11 +662,11 @@ GROUP_MAX_SHARED_FRACTION <- 0.05 treatment_cols, threshold, tie_delta) { slice <- droplevels(df[rows, , drop = FALSE]) candidate_cols <- setdiff(names(slice), env_cols) - is_constant <- vapply(slice[candidate_cols], function(x) { + is_constant <- vapply(.cols(slice, candidate_cols), function(x) { length(unique(stats::na.omit(x))) <= 1L }, logical(1L)) candidate_cols <- candidate_cols[!is_constant] - slice <- slice[, candidate_cols, drop = FALSE] + slice <- .cols(slice, candidate_cols) if (nrow(slice) < 2L || ncol(slice) == 0L) { return(data.frame( diff --git a/R/detect_plot.R b/R/detect_plot.R index 87fbe75..7ae2cf0 100644 --- a/R/detect_plot.R +++ b/R/detect_plot.R @@ -222,10 +222,10 @@ local(S7::method(plot, design_summary) <- function( rows <- !is.na(env_key) & as.character(env_key) == env_label slice <- droplevels(df[rows, , drop = FALSE]) candidate_cols <- setdiff(names(slice), x@env_cols) - constants <- vapply(slice[candidate_cols], function(column) { + constants <- vapply(.cols(slice, candidate_cols), function(column) { length(unique(stats::na.omit(column))) <= 1L }, logical(1L)) - slice <- slice[, candidate_cols[!constants], drop = FALSE] + slice <- .cols(slice, candidate_cols[!constants]) detected <- .detect_design_legacy(slice, scope = .scope_disabled()) if (length(x@treatment_col) > 0L) { detected@treatment_col <- intersect(x@treatment_col, names(slice)) diff --git a/R/detect_rules.R b/R/detect_rules.R index 1aaa13e..8f51bd4 100644 --- a/R/detect_rules.R +++ b/R/detect_rules.R @@ -323,7 +323,7 @@ )) } - cells_filled <- nrow(unique(df[, c(sp$row, sp$col), drop = FALSE])) + cells_filled <- nrow(unique(.cols(df, c(sp$row, sp$col)))) cells_total <- sp$n_row * sp$n_col fill_pct <- cells_filled / cells_total diff --git a/R/events.R b/R/events.R index d9cc8f7..77d21cf 100644 --- a/R/events.R +++ b/R/events.R @@ -66,13 +66,13 @@ } # roles_validate() has already refused a non-numeric outcome. - not_whole <- status[!vapply(df[status], .is_whole_valued, logical(1))] + not_whole <- status[!vapply(.cols(df, status), .is_whole_valued, logical(1))] if (length(not_whole)) { .events_refuse( "Event status{?es} {.field {not_whole}} must be whole-numbered codes." ) } - has_na <- cols[vapply(df[cols], anyNA, logical(1))] + has_na <- cols[vapply(.cols(df, cols), anyNA, logical(1))] if (length(has_na)) { .events_refuse(c( "Event column{?s} {.field {has_na}} contain{?s/} missing values.", @@ -84,7 +84,7 @@ } if (length(time) > 1L) { - tm <- as.matrix(df[time]) + tm <- as.matrix(.cols(df, time)) back <- which(apply(tm, 1L, function(r) any(diff(r) < 0))) if (length(back)) { .events_refuse(c( @@ -119,12 +119,14 @@ k <- length(spec$time) # Gaps and status patterns --------------------------------------------------- - tm <- as.matrix(df[spec$time]) + tm <- as.matrix(.cols(df, spec$time)) gaps <- tm if (k > 1L) { gaps[, -1L] <- tm[, -1L, drop = FALSE] - tm[, -k, drop = FALSE] } - pattern <- do.call(paste, c(lapply(df[spec$status], as.character), sep = "/")) + pattern <- do.call( + paste, c(lapply(.cols(df, spec$status), as.character), sep = "/") + ) # Permute whole status rows, within strata when the clone is conditional, so # pattern counts (per stratum) are exact and no real row keeps its own. diff --git a/R/grain.R b/R/grain.R index 4a35b75..f9192f5 100644 --- a/R/grain.R +++ b/R/grain.R @@ -65,10 +65,7 @@ if (!ok) { return(FALSE) } - x <- df[[cl]] - obs <- !is.na(x) - pairs <- data.table::data.table(g = gid[obs], x = x[obs]) - data.table::uniqueN(pairs) == data.table::uniqueN(pairs$g) + .constant_within(gid, df[[cl]]) }, logical(1L)) list(n_groups = n_groups, constant = constant) }) @@ -147,7 +144,7 @@ # Internal: dense group id (1, 2, ...) of each row over the unit columns; a # missing design value is a group of its own. .unit_id <- function(df, unit) { - data.table::frankv(df[, unit, drop = FALSE], ties.method = "dense") + data.table::frankv(.cols(df, unit), ties.method = "dense") } # Internal: synthesise one grain group on its unit table and broadcast each @@ -161,7 +158,7 @@ o <- which(!is.na(x)) x[o[match(seq_len(n_units), row_unit[o])]] } - ut <- lapply(df[cols], unit_value) + ut <- lapply(.cols(df, cols), unit_value) kind <- roles$kind[match(cols, roles$col)] action <- roles$action[match(cols, roles$col)] @@ -201,3 +198,12 @@ } list(columns = out, level_maps = maps) } + + +# TRUE when `x` holds at most one distinct non-missing value per group id. +.constant_within <- function(gid, x) { + g <- V1 <- NULL + obs <- !is.na(x) + pairs <- data.table::data.table(g = gid[obs], x = x[obs]) + pairs[, data.table::uniqueN(x), by = g][, all(V1 <= 1L)] +} diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index 2e31ad0..ef4664e 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -400,7 +400,7 @@ jitter_coordinates <- function(df, lat_col, lon_col, "`by` column{?s} not found in `df`: {.field {missing_cols}}." ) } - list(mode = "column", key = unname(as.list(df[, by, drop = FALSE]))) + list(mode = "column", key = unname(as.list(.cols(df, by)))) } # Collapse one or more grouping vectors into a single site key. NA is a level @@ -614,7 +614,7 @@ jitter_coordinates <- function(df, lat_col, lon_col, "Internal: coordinate grouping needs the original table." ) } - grp <- .site_key(unname(as.list(original[, s$by, drop = FALSE]))) + grp <- .site_key(unname(as.list(.cols(original, s$by)))) } res <- .jitter_coords_impl( synth, lat_col = s$lat, lon_col = s$lon, diff --git a/R/leakage.R b/R/leakage.R index 3c2f791..456de32 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -171,7 +171,7 @@ if (!length(used)) { return(NULL) } - .row_group_id(df[used]) + .row_group_id(.cols(df, used)) } # Dense integer id of each row's combination of the columns in `cols` (a list diff --git a/R/mask.R b/R/mask.R index 582d280..1c86113 100644 --- a/R/mask.R +++ b/R/mask.R @@ -751,7 +751,7 @@ mask <- function(df, !is_shared & !is_event & !is_grain) conditional_report <- NULL if (length(num_idx) >= 1L) { - x_num <- df[, num_idx, drop = FALSE] + x_num <- .cols(df, num_idx) if (isTRUE(conditional)) { conditional_report <- .conditioning_ladder( df, @@ -807,7 +807,7 @@ mask <- function(df, if (length(events)) { trt_cols <- roles$col[role == "treatment" & action != "drop"] strata <- if (isTRUE(conditional) && length(trt_cols)) { - do.call(paste, c(lapply(df[trt_cols], as.character), sep = "/")) + do.call(paste, c(lapply(.cols(df, trt_cols), as.character), sep = "/")) } for (spec in events) { ev <- .synthesise_events( @@ -897,7 +897,7 @@ mask <- function(df, drop_idx <- which(action == "drop" & !is_shared) if (length(drop_idx)) { dropped <- roles$col[drop_idx] - synth <- synth[, setdiff(names(synth), dropped), drop = FALSE] + synth <- .cols(synth, setdiff(names(synth), dropped)) warnings <- c( warnings, sprintf( diff --git a/R/masque-package.R b/R/masque-package.R index ff572e7..f0e0664 100644 --- a/R/masque-package.R +++ b/R/masque-package.R @@ -36,3 +36,6 @@ #' #' @keywords internal "_PACKAGE" + +# Lets data.table's `[` see masque code as data.table-aware. +.datatable.aware <- TRUE diff --git a/R/recipe_apply.R b/R/recipe_apply.R index c660397..8511e9b 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -94,7 +94,7 @@ apply_recipe <- function(original, rec, check_integrity = TRUE) { .check_recipe_integrity(original, rec) } - out <- original[, retained, drop = FALSE] + out <- .cols(original, retained) for (col in names(rec@level_maps)) { if (!(col %in% names(out))) next diff --git a/R/roles_actions.R b/R/roles_actions.R index 38046e1..4d9dec0 100644 --- a/R/roles_actions.R +++ b/R/roles_actions.R @@ -328,7 +328,7 @@ set_role <- function(roles, cols, role = NULL, action = NULL) { roles$role <- role2 roles$action <- action2 - roles <- roles[, .reorder_roles_cols(names(roles)), drop = FALSE] + roles <- .cols(roles, .reorder_roles_cols(names(roles))) attr(roles, "mode") <- mode attr(roles, "proposed_actions") <- stats::setNames( action2[!pinned], roles$col[!pinned] diff --git a/R/synthesise_geospatial.R b/R/synthesise_geospatial.R index ca66203..de78c1b 100644 --- a/R/synthesise_geospatial.R +++ b/R/synthesise_geospatial.R @@ -126,7 +126,7 @@ synthesise_geospatial <- function(synth, original, # 1. Per (anchor, site) replication counts in the original. has_xy <- !is.na(original[[lat_col]]) & !is.na(original[[lon_col]]) - o <- original[has_xy, c(anchor_col, lat_col, lon_col), drop = FALSE] + o <- .cols(original, c(anchor_col, lat_col, lon_col))[has_xy, ] if (nrow(o) > 0L) { o$.key <- paste(as.character(o[[anchor_col]]), o[[lat_col]], o[[lon_col]], diff --git a/R/synthesise_numeric_conditional.R b/R/synthesise_numeric_conditional.R index 8fa0a1d..070375d 100644 --- a/R/synthesise_numeric_conditional.R +++ b/R/synthesise_numeric_conditional.R @@ -242,7 +242,7 @@ synthesise_numeric_conditional <- function(x_num, groups, min_stratum = 5L) { if (length(parts) == 1L) { return(df[[parts]]) } - interaction(lapply(df[parts], as.character), drop = TRUE, sep = ":") + interaction(lapply(.cols(df, parts), as.character), drop = TRUE, sep = ":") } # Levels with a single row share one pooled level, so the shift a row diff --git a/R/utils_data_table.R b/R/utils_data_table.R index 6a891b3..c6e9ede 100644 --- a/R/utils_data_table.R +++ b/R/utils_data_table.R @@ -7,3 +7,12 @@ data.table::as.data.table(x) } } + +# Column subset of a data.frame or data.table that keeps its class. +.cols <- function(x, j) { + if (data.table::is.data.table(x)) { + x[, j, with = FALSE] + } else { + x[, j, drop = FALSE] + } +} diff --git a/tests/testthat/test-coord-site-invariant.R b/tests/testthat/test-coord-site-invariant.R index aa61c2a..5ff4235 100644 --- a/tests/testthat/test-coord-site-invariant.R +++ b/tests/testthat/test-coord-site-invariant.R @@ -193,8 +193,8 @@ test_that("mask(coords=) inherits the site invariant by default", { syn <- .masked_sites( df, list(list(lat = "lat", lon = "lon", on_land = FALSE)) ) - expect_equal(nrow(unique(as.data.frame(syn[c("lat", "lon")]))), 5L) - by_site <- split(as.data.frame(syn[c("lat", "lon")]), df$trial) + expect_equal(nrow(unique(as.data.frame(syn[, c("lat", "lon")]))), 5L) + by_site <- split(as.data.frame(syn[, c("lat", "lon")]), df$trial) one_pair <- function(s) nrow(unique(s)) == 1L expect_true(all(vapply(by_site, one_pair, logical(1)))) }) @@ -213,7 +213,7 @@ test_that("mask(coords=) accepts `by` and groups on the original table", { coords = list(list(lat = "lat", lon = "lon", by = "trial", on_land = FALSE)))) syn <- synthetic(m) - expect_equal(nrow(unique(as.data.frame(syn[c("lat", "lon")]))), 3L) + expect_equal(nrow(unique(as.data.frame(syn[, c("lat", "lon")]))), 3L) expect_equal(length(unique(syn$lat[1:4])), 1L) rec <- recipe(m) diff --git a/tests/testthat/test-fix0160-datatable.R b/tests/testthat/test-fix0160-datatable.R new file mode 100644 index 0000000..f9339aa --- /dev/null +++ b/tests/testthat/test-fix0160-datatable.R @@ -0,0 +1,55 @@ +test_that("package code can use data.table's [ with by =", { + expect_true(isTRUE(get(".datatable.aware", envir = asNamespace("masque")))) + expect_true(.constant_within(c(1L, 1L, 2L, 2L), c(5, 5, NA, 7))) + expect_false(.constant_within(c(1L, 1L, 2L, 2L), c(5, 6, 7, 7))) + expect_true(.constant_within(integer(), numeric())) +}) + +dt_fixture <- function() { + set.seed(11) + data.frame( + block = factor(rep(1:4, each = 6)), + trt = factor(rep(letters[1:6], 4)), + y = rnorm(24), + x = rnorm(24), + grp = sample(c("u", "v"), 24, TRUE), + stringsAsFactors = FALSE + ) +} + +test_that("data.frame and data.table inputs give the same clone and round trip", { + df <- dt_fixture() + dt <- data.table::as.data.table(df) + for (mode in c("local", "collaborate")) { + roles <- propose_roles(df, mode = mode) + m_df <- suppressWarnings(mask(df, roles, seed = 3L, mode = mode)) + m_dt <- suppressWarnings(mask(dt, roles, seed = 3L, mode = mode)) + expect_s3_class(synthetic(m_df), "data.table") + expect_identical(synthetic(m_df), synthetic(m_dt)) + + for (inp in list(df, dt)) { + back <- apply_recipe(inp, recipe(m_df)) + expect_s3_class(back, "data.table") + expect_identical(back, apply_recipe(df, recipe(m_df))) + } + s <- synthetic(m_df) + u_dt <- unmask(s, recipe(m_df)) + u_df <- unmask(as.data.frame(s), recipe(m_df)) + expect_s3_class(u_dt, "data.table") + expect_identical(as.character(u_dt$trt), as.character(u_df$trt)) + expect_setequal(unique(as.character(u_dt$trt)), levels(df$trt)) + } +}) + +test_that("a version-1 roles table upgrades as a data.table or a data.frame", { + v1 <- data.frame( + col = c("a", "b"), role = c("treatment", "keep"), + kind = c("factor", "numeric"), mask_levels = c("permute", NA), + stringsAsFactors = FALSE + ) + up_df <- .roles_upgrade(v1, "local") + up_dt <- .roles_upgrade(data.table::as.data.table(v1), "local") + expect_identical(up_dt$action, up_df$action) + expect_identical(names(up_dt), names(up_df)) + expect_identical(up_dt$action, c("scramble", "keep")) +}) From fe72d406e171b42039177b053b7c719e1f994623 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:16:42 +1030 Subject: [PATCH 36/66] feat(roles): a numeric treatment can be aliased or permuted A dose or N-rate treatment accepts alias and scramble. Its values are keyed by an exact character form, so unmask() and apply_recipe() return the original numeric type and values. Collaborate mode aliases a numeric treatment by default; local mode keeps it. --- R/alias_levels.R | 25 +++- R/permute_levels.R | 23 ++- R/recipe_apply.R | 36 ++++- R/roles_actions.R | 12 +- tests/testthat/test-alias_levels.R | 5 +- .../testthat/test-fix0160-numeric-treatment.R | 137 ++++++++++++++++++ tests/testthat/test-permute_levels.R | 4 +- tests/testthat/test-role_options.R | 4 +- 8 files changed, 230 insertions(+), 16 deletions(-) create mode 100644 tests/testthat/test-fix0160-numeric-treatment.R diff --git a/R/alias_levels.R b/R/alias_levels.R index e7bd4c3..974f550 100644 --- a/R/alias_levels.R +++ b/R/alias_levels.R @@ -35,7 +35,10 @@ #' exposure can be checked. This is a re-identification #' cost, not a differential-privacy guarantee. #' -#' @param x A factor, character, or logical vector. +#' A numeric vector (a dose, an N rate) is aliased through its exact +#' character form ([.level_key()]), so the map restores every value. +#' +#' @param x A factor, character, logical, or numeric vector. #' @param prefix Character scalar used as the alias prefix (e.g., `"trt"` #' for treatment, `"_L"` for covariate level). #' @return A list with elements: @@ -62,6 +65,9 @@ alias_levels <- function(x, prefix) { out <- factor(unname(map[as.character(x)]), levels = alias_pool) return(list(x = out, map = map)) } + if (is.numeric(x)) { + x <- .level_key(x) + } if (is.character(x) || is.logical(x)) { x_chr <- as.character(x) uvals <- sort(unique(stats::na.omit(x_chr))) @@ -76,7 +82,7 @@ alias_levels <- function(x, prefix) { cli::cli_abort( paste0( - "alias_levels() supports factor, character, or logical; ", + "alias_levels() supports factor, character, logical, or numeric; ", "got {.cls {class(x)[1]}}." ) ) @@ -105,3 +111,18 @@ alias_levels <- function(x, prefix) { is_aliasable_level_vector <- function(x) { is.factor(x) || is.character(x) || is.logical(x) } + +# Exact character form of numeric values, the key a level map holds them by: +# the first form that reads back to the same double, hexadecimal as the last. +.level_key <- function(x) { + if (!is.numeric(x)) { + return(as.character(x)) + } + key <- as.character(x) + for (fmt in c("%.16g", "%.17g", "%a")) { + inexact <- !is.na(x) & suppressWarnings(as.numeric(key)) != x + if (!any(inexact)) break + key[inexact] <- sprintf(fmt, x[inexact]) + } + key +} diff --git a/R/permute_levels.R b/R/permute_levels.R index c4f12d9..22259a6 100644 --- a/R/permute_levels.R +++ b/R/permute_levels.R @@ -8,7 +8,7 @@ #' This is the local-mode opt-in path. The collaborate-mode default #' is `alias_levels()` (opaque rename), not permutation. #' -#' @param x A factor (or character vector). +#' @param x A factor, character or numeric vector. #' @return A list with elements: #' \itemize{ #' \item `x`: the permuted vector (same type and length as input). @@ -39,7 +39,22 @@ permute_levels <- function(x) { out <- ifelse(is.na(x), NA_character_, map[as.character(x)]) return(list(x = out, map = map)) } - cli::cli_abort( - "permute_levels() supports factor or character; got {.cls {class(x)[1]}}." - ) + if (is.numeric(x)) { + # A numeric treatment keeps its storage class; the map holds exact keys. + key <- .level_key(x) + uvals <- unique(stats::na.omit(key)) + if (length(uvals) <= 1L) { + return(list(x = x, map = stats::setNames(uvals, uvals))) + } + perm <- sample(uvals, length(uvals), replace = FALSE) + map <- stats::setNames(perm, uvals) + out <- x + obs <- !is.na(x) + out[obs] <- x[obs][match(map[key[obs]], key[obs])] + return(list(x = out, map = map)) + } + cli::cli_abort(paste0( + "permute_levels() supports factor, character or numeric; got ", + "{.cls {class(x)[1]}}." + )) } diff --git a/R/recipe_apply.R b/R/recipe_apply.R index c660397..9800cd9 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -207,8 +207,11 @@ unmask <- function(x, rec, column = NULL) { } if (is.atomic(x) && is.null(dim(x))) { - # Numeric, logical and date input passes through unchanged. - if (!is.character(x) && !is.factor(x)) { + # Numeric, logical and date input passes through unchanged, except a + # numeric treatment the recipe permuted. + numeric_map <- is.numeric(x) && !is.null(column) && + column %in% names(rec@level_maps) + if (!is.character(x) && !is.factor(x) && !numeric_map) { if (!is.null(column) && length(rec@roles$col) > 0L && !(column %in% rec@roles$col)) { cli::cli_abort(c( @@ -332,11 +335,33 @@ unmask <- function(x, rec, column = NULL) { } else { new_chr } + } else if (is.numeric(val)) { + # A numeric treatment: aliases are character, a permutation stays numeric. + key <- .level_key(val) + unmapped <- !is.na(key) & !(key %in% names(map)) + if (any(unmapped)) { + .fail_unmapped(unique(key[unmapped]), col = col, direction = "forward") + } + new_chr <- unname(map[key]) + if (!setequal(unname(map), names(map))) { + return(new_chr) + } + .numeric_from_key(new_chr, val) } else { val } } +# Internal: numeric values from their exact keys, in the storage class of +# `like`. +.numeric_from_key <- function(key, like) { + num <- suppressWarnings(as.numeric(key)) + if (is.integer(like) || identical(like, "integer")) { + return(as.integer(num)) + } + num +} + # Internal: level order of a relabelled factor in the clone, which must not # reveal the map. A permutation keeps the original order; aliases sort. .clone_levels <- function(map) { @@ -379,6 +404,13 @@ unmask <- function(x, rec, column = NULL) { } else { new_chr } + } else if (is.numeric(val)) { + key <- .level_key(val) + unmapped <- !is.na(key) & !(key %in% names(inv)) + if (any(unmapped)) { + .fail_unmapped(unique(key[unmapped]), col = col, direction = "inverse") + } + .numeric_from_key(unname(inv[key]), val) } else { val } diff --git a/R/roles_actions.R b/R/roles_actions.R index 38046e1..0c1a740 100644 --- a/R/roles_actions.R +++ b/R/roles_actions.R @@ -145,7 +145,11 @@ set_role <- function(roles, cols, role = NULL, action = NULL) { switch(role, design = "keep", treatment = { - if (collab && kind %in% .categorical_kinds()) "alias" else "keep" + if (collab && kind %in% c(.categorical_kinds(), .numeric_kinds())) { + "alias" + } else { + "keep" + } }, outcome = { if (kind %in% .numeric_kinds()) "scramble" else "keep" @@ -201,11 +205,11 @@ set_role <- function(roles, cols, role = NULL, action = NULL) { } } - if (role == "treatment" && !(kind %in% .categorical_kinds())) { + if (role == "treatment" && + !(kind %in% c(.categorical_kinds(), .numeric_kinds()))) { return(paste0( "treatment scramble / alias requires a factor / character / ", - "logical column; a numeric treatment (e.g. a dose) can only be ", - "kept or dropped" + "logical or numeric column" )) } diff --git a/tests/testthat/test-alias_levels.R b/tests/testthat/test-alias_levels.R index 78ae67e..22ad53d 100644 --- a/tests/testthat/test-alias_levels.R +++ b/tests/testthat/test-alias_levels.R @@ -54,5 +54,8 @@ test_that("alias_levels errors on bad prefix", { }) test_that("alias_levels errors on non-categorical input", { - expect_error(masque:::alias_levels(1:5, prefix = "x"), "supports") + expect_error( + masque:::alias_levels(as.Date("2024-01-01") + 0:4, prefix = "x"), + "supports" + ) }) diff --git a/tests/testthat/test-fix0160-numeric-treatment.R b/tests/testthat/test-fix0160-numeric-treatment.R new file mode 100644 index 0000000..4351414 --- /dev/null +++ b/tests/testthat/test-fix0160-numeric-treatment.R @@ -0,0 +1,137 @@ +# A numeric treatment (a dose, an N rate) can be aliased or permuted, and in +# collaborate mode it is aliased by default, so a dose trial's default clone is +# not blocked by the audit. + +# Split plot modelled on agridat::yates.oats: 6 blocks, 3 varieties on whole +# plots, nitrogen 0 / 0.2 / 0.4 / 0.6 on sub-plots, two missing doses. +.fx_dose <- function() { + withr::with_seed(11L, { + g <- expand.grid( + nitro = c(0, 0.2, 0.4, 0.6), gen = c("Golden", "Marvel", "Victory"), + block = sprintf("B%d", 1:6), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(80 + 70 * g$nitro + stats::rnorm(nrow(g), 0, 12), 1) + g$nitro[c(5L, 40L)] <- NA + g[, c("block", "gen", "nitro", "yield")] + }) +} + +.fx_dose_roles <- function(df, mode, nitro_action = NULL) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, "block", role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment") + r <- set_role(r, "nitro", role = "treatment", action = nitro_action) + set_role(r, "yield", role = "outcome", action = "scramble") +} + +.fx_same_partition <- function(a, b) { + identical( + masque:::.row_group_id(list(a)), masque:::.row_group_id(list(b)) + ) +} + +test_that("a numeric treatment accepts alias and scramble", { + df <- .fx_dose() + for (a in c("alias", "scramble")) { + r <- .fx_dose_roles(df, "collaborate", nitro_action = a) + expect_no_error(roles_validate(r, df, mode = "collaborate")) + } + opts <- role_options(kind = "numeric") + expect_true(all( + c("keep", "scramble", "alias", "drop") %in% + opts$action[opts$role == "treatment"] + )) +}) + +test_that("collaborate aliases a numeric treatment by default, local keeps", { + expect_identical( + masque:::.default_action("treatment", "numeric", "collaborate"), "alias" + ) + expect_identical( + masque:::.default_action("treatment", "integer", "collaborate"), "alias" + ) + expect_identical( + masque:::.default_action("treatment", "numeric", "local"), "keep" + ) + df <- .fx_dose() + expect_identical( + .fx_dose_roles(df, "collaborate")$action[3L], "alias" + ) + expect_identical(.fx_dose_roles(df, "local")$action[3L], "keep") +}) + +test_that("the default collaborate clone of a dose trial is not blocked", { + df <- .fx_dose() + r <- .fx_dose_roles(df, "collaborate") + expect_no_warning( + m <- mask(df, r, mode = "collaborate", seed = 3), + class = "masque_high_leakage" + ) + a <- m@audit + expect_identical(a$leakage_class[a$col == "nitro"], "low") + expect_identical(a$alias_status[a$col == "nitro"], "aliased") + + s <- synthetic(m) + expect_type(s$nitro, "character") + expect_true(all(grepl("^nitro_trt_[0-9]{3}$", s$nitro[!is.na(s$nitro)]))) + expect_false(any(stats::na.omit(s$nitro) %in% as.character(df$nitro))) + expect_identical(is.na(s$nitro), is.na(df$nitro)) + expect_true(.fx_same_partition(s$nitro, df$nitro)) + expect_true(.fx_same_partition( + paste(s$block, s$nitro), paste(df$block, df$nitro) + )) + expect_identical(s$block, df$block) +}) + +test_that("an aliased numeric treatment round-trips to its type and values", { + df <- .fx_dose() + m <- mask(df, .fx_dose_roles(df, "collaborate"), mode = "collaborate", + seed = 3) + back <- unmask(synthetic(m), recipe(m)) + expect_type(back$nitro, "double") + expect_identical(back$nitro, df$nitro) + fwd <- apply_recipe(df, recipe(m)) + expect_identical(fwd$nitro, synthetic(m)$nitro) + expect_identical( + unmask(synthetic(m)$nitro, recipe(m), column = "nitro"), df$nitro + ) +}) + +test_that("integer and binary-inexact doses round-trip exactly", { + df <- .fx_dose() + df$nrate <- as.integer(round(df$nitro * 100)) + df$frac <- df$nitro + 0.1 + 0.2 + r <- .fx_dose_roles(df, "collaborate") + r <- set_role(r, c("nrate", "frac"), role = "treatment") + m <- mask(df, r, mode = "collaborate", seed = 5) + back <- unmask(synthetic(m), recipe(m)) + expect_type(back$nrate, "integer") + expect_identical(back$nrate, df$nrate) + expect_identical(back$frac, df$frac) + expect_identical(apply_recipe(df, recipe(m))$frac, synthetic(m)$frac) +}) + +test_that("a permuted numeric treatment stays numeric and round-trips", { + df <- .fx_dose() + r <- .fx_dose_roles(df, "local", nitro_action = "scramble") + m <- mask(df, r, mode = "local", seed = 7) + s <- synthetic(m) + expect_type(s$nitro, "double") + expect_setequal(unique(stats::na.omit(s$nitro)), c(0, 0.2, 0.4, 0.6)) + expect_false(identical(s$nitro, df$nitro)) + expect_identical(is.na(s$nitro), is.na(df$nitro)) + expect_true(.fx_same_partition(s$nitro, df$nitro)) + expect_identical(unmask(s, recipe(m))$nitro, df$nitro) + expect_identical(apply_recipe(df, recipe(m))$nitro, s$nitro) +}) + +test_that("a kept numeric treatment in collaborate still audits HIGH", { + df <- .fx_dose() + r <- .fx_dose_roles(df, "collaborate", nitro_action = "keep") + expect_warning( + m <- mask(df, r, mode = "collaborate", seed = 3), + class = "masque_high_leakage" + ) + expect_identical(m@audit$leakage_class[m@audit$col == "nitro"], "high") +}) diff --git a/tests/testthat/test-permute_levels.R b/tests/testthat/test-permute_levels.R index dd7d338..8274013 100644 --- a/tests/testthat/test-permute_levels.R +++ b/tests/testthat/test-permute_levels.R @@ -39,5 +39,7 @@ test_that("permute_levels singleton-vocab is identity", { }) test_that("permute_levels errors on non-categorical input", { - expect_error(masque:::permute_levels(1:5), "supports") + expect_error( + masque:::permute_levels(as.Date("2024-01-01") + 0:4), "supports" + ) }) diff --git a/tests/testthat/test-role_options.R b/tests/testthat/test-role_options.R index 3a9272f..bc452dc 100644 --- a/tests/testthat/test-role_options.R +++ b/tests/testthat/test-role_options.R @@ -36,7 +36,7 @@ test_that("known constraints render as expected", { expect_identical( row_of("treatment", "alias")$kinds, - "factor, character, logical" + "numeric, integer, factor, character, logical" ) expect_identical(row_of("outcome", "scramble")$kinds, "numeric, integer") expect_identical(row_of("outcome", "alias")$kinds, "none") @@ -77,7 +77,7 @@ test_that("kind filters to the combinations valid for that kind", { num <- role_options(kind = "numeric") expect_true(any(num$role == "outcome" & num$action == "scramble")) - expect_false(any(num$role == "treatment" & num$action == "alias")) + expect_true(any(num$role == "treatment" & num$action == "alias")) oth <- role_options(kind = "other") expect_setequal(unique(oth$action), c("keep", "drop")) From 0ac6ec8c418fae9617f28a37c39648b2049331c3 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:16:48 +1030 Subject: [PATCH 37/66] feat: roles() accessor and data_card() for the C8 v0.1 card (F11, F9, N2) roles() returns the public role table of a clone under the clone's own column names. data_card() builds the card from the clone and that table alone, with the design block copied from the roles, event provenance split between structure and clone_estimate, a refused table name outside the schema pattern, one card per table for sets, and a deterministic YAML writer. yaml joins Suggests for the parse-back tests. --- DESCRIPTION | 3 +- R/data_card.R | 529 ++++++++++++++++++ R/roles_accessor.R | 68 +++ tests/testthat/fixtures/data_card.schema.json | 110 ++++ tests/testthat/helper-data-card.R | 160 ++++++ tests/testthat/test-fix0160-data-card.R | 287 ++++++++++ 6 files changed, 1156 insertions(+), 1 deletion(-) create mode 100644 R/data_card.R create mode 100644 R/roles_accessor.R create mode 100644 tests/testthat/fixtures/data_card.schema.json create mode 100644 tests/testthat/helper-data-card.R create mode 100644 tests/testthat/test-fix0160-data-card.R diff --git a/DESCRIPTION b/DESCRIPTION index 7b0e4eb..e67eb16 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -46,7 +46,8 @@ Suggests: rmarkdown, spelling, testthat (>= 3.0.0), - writexl + writexl, + yaml VignetteBuilder: knitr Config/testthat/edition: 3 URL: https://max578.github.io/masque/, https://github.com/max578/masque diff --git a/R/data_card.R b/R/data_card.R new file mode 100644 index 0000000..c6fbc9f --- /dev/null +++ b/R/data_card.R @@ -0,0 +1,529 @@ +# data_card.R -- data_card(): a C8 v0.1 data card computed from a clone. + +#' Write a data card for a masked clone +#' +#' Describes a clone in a data card (card version 0.1): its size, its +#' design columns and, for each column, a role, a kind and counts. An +#' analysis planner can read the card in place of the data. No value of +#' any column is written: a factor's levels are counted, never listed. +#' +#' The card is computed from the clone alone: [synthetic()], the role table +#' from [roles()], the mode, the package version and the column names of any +#' `events` spec given to [mask()]. No other part of the private recipe is +#' read, so the card can travel with the clone. On a clone whose column +#' names were aliased, the card uses the aliases throughout. +#' +#' @section Design block: +#' The design block is copied from the role table, never re-detected. The +#' treatment columns are those with the role `treatment`, and the block +#' columns those with the role `design`, less any named in `env`, +#' `spatial` or `unit`. Environment, spatial and unit columns are named by +#' the caller, since the role table does not tell them apart from blocks. +#' No design class is written. For a linked-event spec, the event pair +#' chosen by `primary_event` gives the event time and status columns, the +#' other status columns of that spec are listed as competing, and the card +#' records the number of rows with the primary status equal to 1 and the +#' share of rows with neither the primary nor a competing event (censored). +#' +#' @section Provenance: +#' Every statistic carries a provenance class. +#' * `structure`: exact in the clone, so equal to the same statistic on the +#' original table. Row and column counts, missing counts and shares, the +#' level counts and zero shares of kept and aliased columns and of event +#' status columns, the treatment, block and event columns, the +#' environment count and the event count. +#' * `clone_estimate`: re-simulated, so approximate. The level counts and +#' zero shares of scrambled columns, including event times, and the +#' censored share. +#' * `analyst`: named in this call: the environment, spatial and unit +#' columns. +#' +#' @section The YAML file: +#' The card is written by a small deterministic writer: keys in the order of +#' the contract, every string double-quoted and escaped, counts as integers, +#' shares with six decimals, UTF-8 with `\n` line ends. The same clone gives +#' the same bytes. +#' +#' @param m A `masque` object from [mask()], or a `masque_set` from +#' [mask_set()]. +#' @param file Where to write the card in YAML, or `NULL` (the default) to +#' write nothing. For a set, a directory, which receives one file per +#' table named `
.data_card.yaml`. +#' @param name The table name on the card. Defaults to `"clone"` for a +#' single table, and to the table names for a set (where `name` may give +#' one name per table, in order). A name must start with a letter and +#' hold only letters, digits, `_` and `.`. Any other name is refused with +#' `masque_bad_card_name_refusal`, never cleaned. +#' @param env,spatial,unit Optional names of clone columns: the environment +#' columns (trials, sites, years), the columns giving a spatial or temporal +#' position, and the one column identifying the unit of repeated +#' measurement. Environment columns must have the role `design`. For a +#' set, a named list giving the columns per table. A name that is not a +#' column of the clone is refused with `masque_bad_card_column_refusal`. +#' @param primary_event Which event pair of the first `events` spec is the +#' card's event time and status, counted in the order the events occur. +#' Ignored when the clone has no `events` spec. +#' +#' @returns The card as a list, invisibly when `file` is given. For a set, a +#' named list of cards, one per table. +#' +#' @examples +#' r <- propose_roles(iris) +#' r <- set_role(r, "Species", role = "treatment", action = "alias") +#' r <- set_role(r, "Sepal.Length", role = "outcome") +#' m <- mask(iris, r, seed = 1) +#' card <- data_card(m) +#' card$table +#' card$design +#' +#' path <- tempfile(fileext = ".yaml") +#' data_card(m, file = path) +#' head(readLines(path), 12) +#' +#' @seealso [roles()], [synthetic()], [mask()], [mask_set()]. +#' +#' @export +data_card <- function( + m, + file = NULL, + name = NULL, + env = NULL, + spatial = NULL, + unit = NULL, + primary_event = 1L +) { + if (S7::S7_inherits(m, masque_set)) { + return(.data_card_set(m, file, name, env, spatial, unit, primary_event)) + } + r <- roles(m) + card <- .card_build( + m@synthetic, + r, + mode = m@mode, + events = .card_event_specs(m@recipe), + name = if (is.null(name)) "clone" else name, + env = env, + spatial = spatial, + unit = unit, + primary_event = primary_event + ) + if (is.null(file)) { + return(card) + } + .card_write(card, file) + invisible(card) +} + +#' @noRd +.data_card_set <- function(m, file, name, env, spatial, unit, primary_event) { + tables <- names(m@synthetic) + if (is.null(name)) { + name <- tables + } else if (!is.character(name) || length(name) != length(tables)) { + .card_refuse( + "masque_bad_card_name_refusal", + "For a set, {.arg name} must give one name per table ({length(tables)})." + ) + } + for (arg in c("env", "spatial", "unit")) { + val <- get(arg) + if (!is.null(val) && (!is.list(val) || !all(names(val) %in% tables))) { + .card_refuse( + "masque_bad_card_column_refusal", + "For a set, {.arg {arg}} must be a list named by table." + ) + } + } + all_roles <- roles(m) + cards <- lapply(seq_along(tables), function(i) { + nm <- tables[i] + .card_build( + m@synthetic[[nm]], + all_roles[[nm]], + mode = m@mode, + events = .card_event_specs(m@recipe@recipes[[nm]]), + name = name[i], + env = env[[nm]], + spatial = spatial[[nm]], + unit = unit[[nm]], + primary_event = primary_event + ) + }) + names(cards) <- tables + if (is.null(file)) { + return(cards) + } + dir.create(file, recursive = TRUE, showWarnings = FALSE) + for (i in seq_along(cards)) { + .card_write( + cards[[i]], + file.path(file, paste0(cards[[i]]$table$name, ".data_card.yaml")) + ) + } + invisible(cards) +} + +# Building the card ------------------------------------------------------------ + +# The card from a clone and its public role table. `events` is a list of +# `list(time =, status =)` in the clone's column names. +#' @noRd +.card_build <- function( + clone, + roles, + mode, + events = list(), + name = "clone", + env = NULL, + spatial = NULL, + unit = NULL, + primary_event = 1L +) { + .check_card_name(name) + cols <- names(clone) + role_of <- stats::setNames(roles$role[match(cols, roles$col)], cols) + action_of <- stats::setNames(roles$action[match(cols, roles$col)], cols) + if (anyNA(role_of)) { + .card_refuse( + "masque_bad_card_roles_refusal", + "The role table has no row for {.field {cols[is.na(role_of)]}}." + ) + } + .check_card_cols(env, cols, "env", role_of, need_design = TRUE) + .check_card_cols(spatial, cols, "spatial", role_of) + .check_card_cols(unit, cols, "unit", role_of, single = TRUE) + ev <- .card_events(clone, events, primary_event) + exact <- action_of %in% c("keep", "alias") + names(exact) <- cols + exact[ev$status_cols] <- TRUE + + # Design --------------------------------------------------------------------- + design <- list() + prov <- list() + treatment <- cols[role_of == "treatment"] + blocks <- setdiff(cols[role_of == "design"], c(env, spatial, unit)) + if (length(treatment)) { + design$treatment_cols <- treatment + prov$treatment_cols <- "structure" + } + if (length(blocks)) { + design$block_cols <- blocks + prov$block_cols <- "structure" + } + if (length(env)) { + design$env_cols <- env + prov$env_cols <- "analyst" + } + if (length(spatial)) { + design$spatial_cols <- spatial + prov$spatial_cols <- "analyst" + } + if (length(unit)) { + design$unit_col <- unit + prov$unit_col <- "analyst" + } + if (length(env)) { + env_dt <- stats::na.omit(data.table::as.data.table(clone), cols = env) + design$n_env <- data.table::uniqueN(env_dt, by = env) + prov$n_env <- if (all(exact[env])) "structure" else "clone_estimate" + } + if (!is.null(ev$design)) { + design <- c(design, ev$design) + prov <- c(prov, ev$provenance) + } + if (length(design)) { + design$provenance <- prov + } + + # Columns -------------------------------------------------------------------- + n <- nrow(clone) + columns <- lapply(cols, function(nm) { + x <- clone[[nm]] + kind <- .card_kind(x) + entry <- list(name = nm, role = role_of[[nm]], kind = kind) + p <- list(kind = if (sub("ordered", "factor", kind) == roles$kind[ + match(nm, roles$col) + ]) "structure" else "clone_estimate") + est <- if (exact[[nm]]) "structure" else "clone_estimate" + miss <- is.na(x) + if (kind != "other") { + entry$n_levels <- data.table::uniqueN(x[!miss]) + p$n_levels <- est + } + entry$n_missing <- sum(miss) + p$n_missing <- "structure" + entry$na_share <- if (n) round(mean(miss), 6L) else 0 + p$na_share <- "structure" + if (kind %in% c("numeric", "integer") && any(!miss)) { + entry$zero_share <- round(mean(x[!miss] == 0), 6L) + p$zero_share <- est + } + entry$provenance <- p + entry + }) + + card <- list( + card_version = "0.1", + produced_by = list( + package = "masque", + version = as.character(utils::packageVersion("masque")), + mode = mode + ), + table = list( + name = name, + source = "clone", + n_rows = n, + n_cols = length(cols), + provenance = list(n_rows = "structure", n_cols = "structure") + ) + ) + if (length(design)) { + card$design <- design + } + card$columns <- columns + card +} + +# The event fields of the design block, from the first spec. +#' @noRd +.card_events <- function(clone, events, primary_event) { + if (!length(events)) { + return(list(status_cols = character())) + } + spec <- events[[1L]] + k <- length(spec$time) + if (!is.numeric(primary_event) || length(primary_event) != 1L || + is.na(primary_event) || primary_event != round(primary_event) || + primary_event < 1 || primary_event > k) { + .card_refuse( + "masque_bad_card_event_refusal", + "{.arg primary_event} must be a whole number from 1 to {k}." + ) + } + i <- as.integer(primary_event) + competing <- spec$status[-i] + status <- clone[[spec$status[i]]] + other <- if (length(competing)) { + Reduce(`|`, lapply(competing, function(s) clone[[s]] == 1)) + } else { + rep(FALSE, nrow(clone)) + } + design <- list( + event_time_col = spec$time[i], + event_status_col = spec$status[i] + ) + prov <- list(event_time_col = "structure", event_status_col = "structure") + if (length(competing)) { + design$competing_status_cols <- competing + prov$competing_status_cols <- "structure" + } + design$n_events <- sum(status == 1) + prov$n_events <- "structure" + design$censored_share <- if (nrow(clone)) { + round(mean(status == 0 & !other), 6L) + } else { + 0 + } + prov$censored_share <- "clone_estimate" + list( + design = design, + provenance = prov, + status_cols = unlist(lapply(events, `[[`, "status")) + ) +} + +# The `events` specs recorded at mask() time, under the clone's names. +#' @noRd +.card_event_specs <- function(rec) { + map <- rec@column_name_map + rename <- function(x) { + hits <- x %in% names(map) + x[hits] <- unname(unlist(map[x[hits]])) + x + } + lapply(rec@events, function(e) { + list(time = rename(e$time), status = rename(e$status)) + }) +} + +#' @noRd +.card_kind <- function(x) { + if (inherits(x, c("POSIXct", "POSIXlt", "difftime"))) { + return("datetime") + } + if (inherits(x, "Date")) { + return("date") + } + if (is.ordered(x)) { + return("ordered") + } + if (is.factor(x)) { + return("factor") + } + if (is.logical(x)) { + return("logical") + } + if (is.integer(x)) { + return("integer") + } + if (is.numeric(x)) { + return("numeric") + } + if (is.character(x)) { + return("character") + } + "other" +} + +# Checks --------------------------------------------------------------------- + +#' @noRd +.check_card_name <- function(name) { + ok <- is.character(name) && length(name) == 1L && !is.na(name) && + grepl("^[A-Za-z][A-Za-z0-9_.]*$", name) + if (!ok) { + .card_refuse( + "masque_bad_card_name_refusal", + c( + paste0( + "The card's table name must start with a letter and hold only ", + "letters, digits, {.val _} and {.val .}." + ), + i = "Pass a name that does: {.arg name} is never cleaned." + ) + ) + } + invisible(name) +} + +#' @noRd +.check_card_cols <- function( + x, + cols, + arg, + role_of, + need_design = FALSE, + single = FALSE +) { + if (is.null(x)) { + return(invisible(x)) + } + if (!is.character(x) || !length(x) || anyNA(x) || + (single && length(x) != 1L)) { + .card_refuse( + "masque_bad_card_column_refusal", + paste0( + "{.arg {arg}} must name ", + if (single) "one column" else "columns", + " of the clone." + ) + ) + } + absent <- setdiff(x, cols) + if (length(absent)) { + .card_refuse( + "masque_bad_card_column_refusal", + c( + "{.arg {arg}} names {length(absent)} column{?s} not in the clone.", + i = "Use the clone's own column names, as {.fn roles} lists them." + ) + ) + } + if (need_design && !all(role_of[x] == "design")) { + .card_refuse( + "masque_bad_card_column_refusal", + "{.arg {arg}} must name columns with the role {.val design}." + ) + } + invisible(x) +} + +#' @noRd +.card_refuse <- function(class, message) { + cli::cli_abort( + message, + class = c(class, "orchestra_refusal"), + .envir = parent.frame() + ) +} + +# The YAML writer ------------------------------------------------------------ + +.card_seq_keys <- c( + "treatment_cols", "block_cols", "env_cols", "spatial_cols", + "competing_status_cols" +) + +#' @noRd +.card_write <- function(card, path) { + writeBin(charToRaw(enc2utf8(.card_yaml(card))), path) + invisible(path) +} + +#' @noRd +.card_yaml <- function(x) { + paste0(paste(.yaml_mapping(x, 0L, character()), collapse = "\n"), "\n") +} + +#' @noRd +# `seq_keys` are the keys whose character value is a sequence even when it +# holds one name; they apply in the design block only. +.yaml_mapping <- function(x, indent, seq_keys) { + pad <- strrep(" ", indent) + out <- character() + for (k in names(x)) { + v <- x[[k]] + key <- paste0(pad, k, ":") + if (is.list(v) && !is.null(names(v))) { + inner <- if (k == "design") .card_seq_keys else character() + out <- c(out, key, .yaml_mapping(v, indent + 2L, inner)) + } else if (is.list(v) || k %in% seq_keys) { + if (!length(v)) { + out <- c(out, paste0(key, " []")) + } else { + items <- lapply(v, .yaml_item, indent = indent + 2L) + out <- c(out, key, unlist(items)) + } + } else { + out <- c(out, paste(key, .yaml_scalar(v))) + } + } + out +} + +#' @noRd +.yaml_item <- function(item, indent) { + pad <- strrep(" ", indent) + if (is.list(item)) { + lines <- .yaml_mapping(item, indent + 2L, character()) + lines[1L] <- paste0(pad, "- ", substring(lines[1L], indent + 3L)) + return(lines) + } + paste0(pad, "- ", .yaml_scalar(item)) +} + +#' @noRd +.yaml_scalar <- function(v) { + if (is.character(v)) { + return(.yaml_quote(v)) + } + if (is.integer(v)) { + return(as.character(v)) + } + sprintf("%.6f", v) +} + +#' @noRd +.yaml_quote <- function(s) { + s <- enc2utf8(s) + s <- gsub("\\", "\\\\", s, fixed = TRUE) + s <- gsub("\"", "\\\"", s, fixed = TRUE) + cp <- utf8ToInt(s) + ctl <- cp < 32L | cp == 127L + if (any(ctl)) { + chars <- vapply(cp, intToUtf8, character(1L)) + chars[ctl] <- sprintf("\\u%04X", cp[ctl]) + s <- paste(chars, collapse = "") + } + paste0("\"", s, "\"") +} diff --git a/R/roles_accessor.R b/R/roles_accessor.R new file mode 100644 index 0000000..3f976e4 --- /dev/null +++ b/R/roles_accessor.R @@ -0,0 +1,68 @@ +# roles_accessor.R -- roles(): the public role table of a clone. + +#' The role table of a masked clone +#' +#' Returns the role each column of the clone was masked under, in the +#' clone's own vocabulary: one row per column of [synthetic()], in the same +#' order and under the same names. When the column names were aliased +#' (`alias_names` in [mask()]), the rows carry the aliases, so the table can +#' travel with the clone without disclosing the original names. A dropped +#' column is not in the clone and has no row. +#' +#' Only the four public columns are returned. The proposal's notes and its +#' value summaries describe the original table and stay on the private +#' recipe, as does the full role table used on the original +#' (`recipe(m)@roles`). +#' +#' @param m A `masque` object from [mask()], or a `masque_set` from +#' [mask_set()]. +#' +#' @returns For a `masque`, a data.table with columns `col`, `role`, +#' `action` and `kind`. For a `masque_set`, a named list of such tables, +#' one per table of the set. +#' +#' @examples +#' r <- propose_roles(iris) +#' r <- set_role(r, "Sepal.Length", role = "outcome") +#' m <- mask(iris, r, seed = 1) +#' roles(m) +#' +#' @seealso [synthetic()], [data_card()], [mask()]. +#' +#' @export +roles <- function(m) { + if (S7::S7_inherits(m, masque_set)) { + out <- lapply(names(m@synthetic), function(nm) { + .public_roles(m@recipe@recipes[[nm]], m@synthetic[[nm]]) + }) + return(stats::setNames(out, names(m@synthetic))) + } + if (!S7::S7_inherits(m, masque_obj)) { + cli::cli_abort( + paste0( + "`m` must be a {.cls masque} or {.cls masque_set} object; got ", + "{.cls {class(m)[1]}}." + ), + class = c("masque_bad_masque_obj_refusal", "orchestra_refusal") + ) + } + .public_roles(m@recipe, m@synthetic) +} + +#' @noRd +.public_roles <- function(rec, synth) { + r <- as.data.frame(rec@roles) + clone_col <- r$col + map <- rec@column_name_map + if (length(map)) { + hits <- clone_col %in% names(map) + clone_col[hits] <- unname(unlist(map[clone_col[hits]])) + } + idx <- match(names(synth), clone_col) + data.table::data.table( + col = names(synth), + role = r$role[idx], + action = r$action[idx], + kind = r$kind[idx] + ) +} diff --git a/tests/testthat/fixtures/data_card.schema.json b/tests/testthat/fixtures/data_card.schema.json new file mode 100644 index 0000000..0ad30ba --- /dev/null +++ b/tests/testthat/fixtures/data_card.schema.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "composeR/data_card.schema.json", + "title": "Data card, contract version 0.1", + "description": "Facts about one table and no value of any column: its size, its design columns (with, for a time-to-event outcome, the time, status and competing-status columns and the event and censoring counts), and for each column a role, a kind and counts. Every statistic carries a provenance class in the provenance map beside it: structure (exact in a masked clone), clone_estimate (re-simulated in a clone, approximate), computed (from the table itself), dictionary (from a published data dictionary) or analyst (stated by the analyst). Level labels are values and never appear; levels are counted.", + "type": "object", + "required": ["card_version", "produced_by", "table"], + "additionalProperties": false, + "properties": { + "card_version": {"type": "string", "enum": ["0.1"]}, + "produced_by": { + "type": "object", + "required": ["package", "version"], + "additionalProperties": false, + "properties": { + "package": {"type": "string", "pattern": "^[A-Za-z][A-Za-z0-9.]*$"}, + "version": {"type": "string", "pattern": "^[0-9]+([.-][0-9]+)+$"}, + "mode": {"type": "string"} + } + }, + "table": { + "type": "object", + "required": ["name", "n_rows", "n_cols", "provenance"], + "additionalProperties": false, + "properties": { + "name": {"type": "string", "pattern": "^[A-Za-z][A-Za-z0-9_.]*$"}, + "source": {"type": "string", "enum": ["clone", "data_frame", "dictionary", "datahub"]}, + "n_rows": {"type": "integer"}, + "n_cols": {"type": "integer"}, + "provenance": { + "type": "object", + "additionalProperties": false, + "properties": { + "n_rows": {"$ref": "#/$defs/class"}, + "n_cols": {"$ref": "#/$defs/class"} + } + } + } + }, + "design": { + "type": "object", + "required": ["provenance"], + "additionalProperties": false, + "properties": { + "class": {"type": "string"}, + "treatment_cols": {"$ref": "#/$defs/names"}, + "block_cols": {"$ref": "#/$defs/names"}, + "env_cols": {"$ref": "#/$defs/names"}, + "spatial_cols": {"$ref": "#/$defs/names"}, + "unit_col": {"type": "string"}, + "n_env": {"type": "integer"}, + "event_time_col": {"type": "string"}, + "event_status_col": {"type": "string"}, + "competing_status_cols": {"$ref": "#/$defs/names"}, + "n_events": {"type": "integer"}, + "censored_share": {"type": "number"}, + "provenance": { + "type": "object", + "additionalProperties": false, + "properties": { + "class": {"$ref": "#/$defs/class"}, + "treatment_cols": {"$ref": "#/$defs/class"}, + "block_cols": {"$ref": "#/$defs/class"}, + "env_cols": {"$ref": "#/$defs/class"}, + "spatial_cols": {"$ref": "#/$defs/class"}, + "unit_col": {"$ref": "#/$defs/class"}, + "n_env": {"$ref": "#/$defs/class"}, + "event_time_col": {"$ref": "#/$defs/class"}, + "event_status_col": {"$ref": "#/$defs/class"}, + "competing_status_cols": {"$ref": "#/$defs/class"}, + "n_events": {"$ref": "#/$defs/class"}, + "censored_share": {"$ref": "#/$defs/class"} + } + } + } + }, + "columns": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": {"type": "string"}, + "role": {"type": "string", "enum": ["id", "date", "subject", "context", "design", "treatment", "outcome", "covariate", "text", "other"]}, + "kind": {"type": "string", "enum": ["numeric", "integer", "logical", "factor", "ordered", "character", "date", "datetime", "other"]}, + "n_levels": {"type": "integer"}, + "n_missing": {"type": "integer"}, + "na_share": {"type": "number"}, + "zero_share": {"type": "number"}, + "provenance": { + "type": "object", + "additionalProperties": false, + "properties": { + "kind": {"$ref": "#/$defs/class"}, + "n_levels": {"$ref": "#/$defs/class"}, + "n_missing": {"$ref": "#/$defs/class"}, + "na_share": {"$ref": "#/$defs/class"}, + "zero_share": {"$ref": "#/$defs/class"} + } + } + } + } + } + }, + "$defs": { + "class": {"type": "string", "enum": ["structure", "clone_estimate", "computed", "dictionary", "analyst"]}, + "names": {"type": "array", "items": {"type": "string"}} + } +} diff --git a/tests/testthat/helper-data-card.R b/tests/testthat/helper-data-card.R new file mode 100644 index 0000000..75afc26 --- /dev/null +++ b/tests/testthat/helper-data-card.R @@ -0,0 +1,160 @@ +# A copy of composeR's data_card.schema.json (contract C8 v0.1, composeR +# commit 5f75c2e) and a small validator for the JSON Schema keywords it uses. +# The schema is parsed with yaml, which reads this JSON as YAML. + +.card_schema <- function() { + yaml::read_yaml(testthat::test_path("fixtures", "data_card.schema.json")) +} + +.schema_type_ok <- function(x, type) { + switch(type, + object = is.list(x) && (length(x) == 0L || !is.null(names(x))), + # yaml reads a sequence of strings as a character vector. + array = (is.list(x) || is.character(x)) && is.null(names(x)), + string = is.character(x) && length(x) == 1L, + integer = (is.integer(x) || (is.numeric(x) && x == round(x))) && + length(x) == 1L, + number = is.numeric(x) && length(x) == 1L, + FALSE + ) +} + +# Returns a character vector of errors, empty when `x` is valid. +.schema_errors <- function(x, node, root, path = "$") { + if (!is.null(node[["$ref"]])) { + key <- sub("^#/\\$defs/", "", node[["$ref"]]) + return(.schema_errors(x, root[["$defs"]][[key]], root, path)) + } + if (!is.null(node$type) && !.schema_type_ok(x, node$type)) { + return(sprintf("%s: not of type %s", path, node$type)) + } + errors <- character() + if (!is.null(node$enum) && !(x %in% node$enum)) { + errors <- c(errors, sprintf("%s: '%s' not in enum", path, x)) + } + if (!is.null(node$pattern) && !grepl(node$pattern, x, perl = TRUE)) { + errors <- c(errors, sprintf("%s: '%s' fails pattern", path, x)) + } + if (identical(node$type, "object")) { + missing <- setdiff(node$required, names(x)) + errors <- c(errors, sprintf("%s: missing '%s'", path, missing)) + if (isFALSE(node$additionalProperties)) { + extra <- setdiff(names(x), names(node$properties)) + errors <- c(errors, sprintf("%s: unexpected '%s'", path, extra)) + } + for (nm in intersect(names(x), names(node$properties))) { + errors <- c(errors, .schema_errors( + x[[nm]], node$properties[[nm]], root, paste0(path, ".", nm) + )) + } + } + if (identical(node$type, "array") && !is.null(node$items)) { + for (i in seq_along(x)) { + errors <- c(errors, .schema_errors( + x[[i]], node$items, root, sprintf("%s[%d]", path, i) + )) + } + } + errors +} + +# The rules composeR checks beside the schema (composeR R/data_card.R, +# .card_rule_errors): every statistic has a provenance class and every class +# names a statistic, counts fit the table, design columns are listed. +.card_rule_errors <- function(card) { + errors <- character() + classed <- function(obj, path, unclassed) { + stats <- setdiff(names(obj), c(unclassed, "provenance")) + classes <- names(obj$provenance) + c( + sprintf("%s.%s: no provenance class", path, setdiff(stats, classes)), + sprintf("%s.provenance.%s: names no statistic", path, + setdiff(classes, stats)) + ) + } + tab <- card$table + errors <- c(errors, classed(tab, "$.table", c("name", "source"))) + design <- card$design + if (!is.null(design)) { + errors <- c(errors, classed(design, "$.design", character())) + if (xor(is.null(design$event_time_col), is.null(design$event_status_col))) { + errors <- c(errors, "$.design: event time and status come together") + } + if (!is.null(design$n_events) && + (design$n_events < 0 || design$n_events > tab$n_rows)) { + errors <- c(errors, "$.design.n_events out of range") + } + if (!is.null(design$censored_share) && + (design$censored_share < 0 || design$censored_share > 1)) { + errors <- c(errors, "$.design.censored_share out of range") + } + } + cols <- card$columns + col_names <- vapply(cols, `[[`, character(1L), "name") + if (anyDuplicated(col_names) > 0L) { + errors <- c(errors, "$.columns: duplicated name") + } + if (length(cols) > tab$n_cols) { + errors <- c(errors, "$.columns: more columns than n_cols") + } + for (i in seq_along(cols)) { + col <- cols[[i]] + path <- sprintf("$.columns[%d]", i) + errors <- c(errors, classed(col, path, c("name", "role"))) + for (f in intersect(c("n_levels", "n_missing"), names(col))) { + if (col[[f]] < 0 || col[[f]] > tab$n_rows) { + errors <- c(errors, sprintf("%s.%s out of range", path, f)) + } + } + for (f in intersect(c("na_share", "zero_share"), names(col))) { + if (col[[f]] < 0 || col[[f]] > 1) { + errors <- c(errors, sprintf("%s.%s out of range", path, f)) + } + } + } + named <- unlist(design[c( + "treatment_cols", "block_cols", "env_cols", "spatial_cols", "unit_col", + "event_time_col", "event_status_col", "competing_status_cols" + )], use.names = FALSE) + absent <- setdiff(named, col_names) + c(errors, sprintf("$.design: names '%s', not a listed column", absent)) +} + +.expect_valid_card <- function(card) { + errs <- c(.schema_errors(card, .card_schema(), .card_schema()), + .card_rule_errors(card)) + expect_identical(errs, character(), info = paste(errs, collapse = "; ")) +} + +# A small MET-shaped table with missing values and a named treatment. +.card_plots <- function() { + withr::with_seed(3L, { + n <- 72L + d <- data.frame( + site = factor(rep(c("SITEALPHA", "SITEBRAVO", "SITECHARLIE"), + each = 24L)), + rep = rep(rep(1:3, each = 8L), 3L), + gen = factor(rep(sprintf("GENO%02d", 1:8), 9L)), + rating = factor(sample(c("low", "mid", "high"), n, replace = TRUE), + levels = c("low", "mid", "high"), ordered = TRUE), + rain = rep(c(410, 395, 520), each = 24L) + stats::rnorm(n), + weeds = stats::rpois(n, 1), + yield = round(5 + stats::rnorm(n), 2) + ) + d$yield[c(3L, 17L, 40L)] <- NA + d$weeds[c(5L, 9L)] <- NA + d + }) +} + +.card_roles <- function(df, mode = "collaborate") { + r <- propose_roles(df, mode = mode) + r <- set_role(r, "site", role = "design", action = "alias") + r <- set_role(r, "rep", role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", action = "alias") + r <- set_role(r, "rating", role = "covariate", action = "scramble") + r <- set_role(r, "rain", role = "covariate", action = "scramble") + r <- set_role(r, "weeds", role = "covariate", action = "scramble") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r +} diff --git a/tests/testthat/test-fix0160-data-card.R b/tests/testthat/test-fix0160-data-card.R new file mode 100644 index 0000000..3327b6c --- /dev/null +++ b/tests/testthat/test-fix0160-data-card.R @@ -0,0 +1,287 @@ +# F11 and the data card (contract C8 v0.1): roles(), data_card(), the YAML +# writer, provenance, the name map, sets and the leak rule. + +skip_if_not_installed("yaml") + +.card_mask <- function(alias_names = FALSE, seed = 424242L) { + df <- .card_plots() + suppressWarnings(mask( + df, .card_roles(df), mode = "collaborate", seed = seed, + alias_names = alias_names + )) +} + +# roles() --------------------------------------------------------------------- + +test_that("roles() returns the four public columns, one row per clone column", { + m <- .card_mask() + r <- roles(m) + expect_s3_class(r, "data.table") + expect_identical(names(r), c("col", "role", "action", "kind")) + expect_identical(r$col, names(synthetic(m))) + expect_identical(r$role[r$col == "gen"], "treatment") + expect_identical(r$action[r$col == "site"], "alias") +}) + +test_that("roles() speaks the clone's names on an aliased clone", { + m <- .card_mask(alias_names = TRUE) + r <- roles(m) + expect_identical(r$col, names(synthetic(m))) + expect_false(any(names(.card_plots()) %in% r$col)) + map <- recipe(m)@column_name_map + expect_identical(r$role[r$col == map$gen], "treatment") +}) + +test_that("roles() leaves out a dropped column and refuses a non-masque", { + df <- .card_plots() + r <- set_role(.card_roles(df), "weeds", action = "drop") + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = 1L)) + expect_false("weeds" %in% roles(m)$col) + expect_error(roles(df), class = "masque_bad_masque_obj_refusal") +}) + +# The card against the composeR schema ---------------------------------------- + +test_that("data_card() is valid against composeR's C8 v0.1 schema", { + m <- .card_mask() + card <- data_card(m) + .expect_valid_card(card) + expect_identical(card$card_version, "0.1") + expect_identical(card$produced_by$package, "masque") + expect_identical( + card$produced_by$version, as.character(utils::packageVersion("masque")) + ) + expect_identical(card$produced_by$mode, "collaborate") + expect_identical(card$table$name, "clone") + expect_identical(card$table$source, "clone") + expect_identical(card$table$n_rows, 72L) + expect_identical(card$table$n_cols, 7L) +}) + +test_that("the written YAML parses back to the card, and is valid", { + m <- .card_mask() + path <- withr::local_tempfile(fileext = ".yaml") + card <- data_card(m, file = path) + back <- yaml::read_yaml(path) + expect_identical(back, card) + .expect_valid_card(back) + bytes <- readBin(path, "raw", file.size(path)) + expect_false(any(bytes == as.raw(13L))) + lines <- readLines(path, encoding = "UTF-8") + expect_false(any(grepl("[ \t]$", lines))) + # A one-name column list is a sequence; a provenance class is a scalar. + expect_true(" treatment_cols:" %in% lines) + expect_true(" - \"gen\"" %in% lines) + expect_true(" treatment_cols: \"structure\"" %in% lines) +}) + +# Design block from the role table (N2) ---------------------------------------- + +test_that("the design block is the role table, not the detector", { + m <- .card_mask(alias_names = TRUE) + map <- recipe(m)@column_name_map + card <- data_card(m, env = map$site, unit = NULL) + d <- card$design + expect_null(d$class) + expect_identical(unlist(d$treatment_cols), map$gen) + expect_identical(unlist(d$block_cols), map$rep) + expect_identical(unlist(d$env_cols), map$site) + expect_identical(d$n_env, 3L) + expect_identical(d$provenance$treatment_cols, "structure") + expect_identical(d$provenance$env_cols, "analyst") + expect_identical(d$provenance$n_env, "structure") + .expect_valid_card(card) +}) + +test_that("env, spatial and unit must name design columns of the clone", { + m <- .card_mask(alias_names = TRUE) + expect_error(data_card(m, env = "site"), class = "masque_bad_card_column_refusal") + map <- recipe(m)@column_name_map + expect_error( + data_card(m, env = map$yield), class = "masque_bad_card_column_refusal" + ) + card <- data_card(m, unit = map$rep) + expect_identical(card$design$unit_col, map$rep) + expect_false(map$rep %in% unlist(card$design$block_cols)) +}) + +# Provenance ------------------------------------------------------------------ + +test_that("every statistic tagged structure is exact against the original", { + df <- .card_plots() + m <- .card_mask(alias_names = TRUE) + map <- recipe(m)@column_name_map + real <- stats::setNames(names(map), unlist(map)) + card <- data_card(m) + checked <- 0L + for (col in card$columns) { + x <- df[[real[[col$name]]]] + obs <- x[!is.na(x)] + for (stat in names(col$provenance)) { + if (col$provenance[[stat]] != "structure" || stat == "kind") next + truth <- switch(stat, + n_levels = length(unique(obs)), + n_missing = sum(is.na(x)), + na_share = round(mean(is.na(x)), 6), + zero_share = round(mean(obs == 0), 6) + ) + expect_equal(col[[stat]], truth, info = paste(col$name, stat)) + checked <- checked + 1L + } + } + expect_gt(checked, 10L) + by_name <- stats::setNames(card$columns, vapply(card$columns, `[[`, "", "name")) + expect_identical(by_name[[map$yield]]$provenance$n_levels, "clone_estimate") + expect_identical(by_name[[map$site]]$provenance$n_levels, "structure") + expect_identical(by_name[[map$rating]]$kind, "ordered") +}) + +test_that("events: pattern counts are structure, times and censoring estimates", { + df <- .events_fixture(n = 300L) + m <- suppressWarnings(mask( + df, .events_roles(df), mode = "collaborate", seed = 7L, + events = .events_spec + )) + card <- data_card(m) + d <- card$design + expect_identical(d$event_time_col, "time_rec") + expect_identical(d$event_status_col, "status_rec") + expect_identical(unlist(d$competing_status_cols), "status_death") + expect_identical(d$n_events, sum(df$status_rec == 1L)) + expect_identical(d$provenance$n_events, "structure") + expect_identical(d$provenance$censored_share, "clone_estimate") + by_name <- stats::setNames(card$columns, vapply(card$columns, `[[`, "", "name")) + expect_identical(by_name$time_rec$provenance$n_levels, "clone_estimate") + expect_identical(by_name$status_rec$provenance$n_levels, "structure") + expect_identical(by_name$status_death$provenance$zero_share, "structure") + .expect_valid_card(card) + + card2 <- data_card(m, primary_event = 2L) + expect_identical(card2$design$event_time_col, "time_death") + expect_identical(unlist(card2$design$competing_status_cols), "status_rec") + expect_identical(card2$design$n_events, sum(df$status_death == 1L)) + expect_error( + data_card(m, primary_event = 3L), class = "masque_bad_card_event_refusal" + ) +}) + +test_that("event columns are carded under their aliases", { + df <- .events_fixture(n = 300L) + m <- suppressWarnings(mask( + df, .events_roles(df), mode = "collaborate", seed = 7L, + events = .events_spec, alias_names = TRUE + )) + map <- recipe(m)@column_name_map + d <- data_card(m)$design + expect_identical(d$event_time_col, map$time_rec) + expect_identical(d$event_status_col, map$status_rec) +}) + +# Table name ------------------------------------------------------------------ + +test_that("a table name outside the schema pattern is refused, not cleaned", { + m <- .card_mask() + for (bad in c("my table", "1trial", "", "trial-2", NA_character_)) { + expect_error( + data_card(m, name = bad), + class = "masque_bad_card_name_refusal", info = bad + ) + } + err <- tryCatch(data_card(m, name = "a b"), error = identity) + expect_s3_class(err, "orchestra_refusal") + expect_identical(data_card(m, name = "plots.v2")$table$name, "plots.v2") +}) + +# The card is a subset of the clone (S3, O1) ----------------------------------- + +test_that("rebuilt from synthetic(m) and roles(m) alone, the card is byte-equal", { + for (alias in c(FALSE, TRUE)) { + m <- .card_mask(alias_names = alias) + a <- withr::local_tempfile(fileext = ".yaml") + b <- withr::local_tempfile(fileext = ".yaml") + data_card(m, file = a) + rebuilt <- .card_build( + synthetic(m), roles(m), mode = m@mode, events = list() + ) + .card_write(rebuilt, b) + expect_identical( + readBin(a, "raw", file.size(a)), readBin(b, "raw", file.size(b)), + info = paste("alias_names =", alias) + ) + } + # Same clone, card written twice: same bytes. + m <- .card_mask() + a <- withr::local_tempfile(fileext = ".yaml") + b <- withr::local_tempfile(fileext = ".yaml") + data_card(m, file = a) + data_card(m, file = b) + expect_identical(tools::md5sum(a)[[1L]], tools::md5sum(b)[[1L]]) +}) + +test_that("an aliased clone's card carries no real label, name or seed", { + m <- .card_mask(alias_names = TRUE, seed = 424242L) + path <- withr::local_tempfile(fileext = ".yaml") + data_card(m, file = path, env = recipe(m)@column_name_map$site) + text <- paste(readLines(path, encoding = "UTF-8"), collapse = "\n") + df <- .card_plots() + tokens <- c( + "424242", "SITEALPHA", "SITEBRAVO", "SITECHARLIE", "GENO01", "GENO08", + paste0("\"", names(df), "\"") + ) + for (tok in tokens) { + expect_false(grepl(tok, text, fixed = TRUE), info = tok) + } + for (nm in names(df)) { + expect_false(grepl(paste0("\\b", nm, "\\b"), text, perl = TRUE), info = nm) + } +}) + +# Sets ------------------------------------------------------------------------ + +test_that("a set gives one card per table, and one file per table", { + plots <- .card_plots() + sites <- data.frame( + site = factor(c("SITEALPHA", "SITEBRAVO", "SITECHARLIE")), + rain = c(410, 395, 520) + ) + rs <- list( + plots = .card_roles(plots), + sites = set_role( + set_role(propose_roles(sites, mode = "collaborate"), "site", + role = "design", action = "alias"), + "rain", role = "covariate", action = "keep" + ) + ) + s <- suppressWarnings(mask_set( + list(plots = plots, sites = sites), roles = rs, mode = "collaborate", + seed = 5L, quiet = TRUE + )) + cards <- data_card(s) + expect_identical(names(cards), c("plots", "sites")) + for (nm in names(cards)) { + .expect_valid_card(cards[[nm]]) + expect_identical(cards[[nm]]$table$name, nm) + expect_identical(cards[[nm]]$table$n_rows, nrow(synthetic(s)[[nm]])) + } + rset <- roles(s) + expect_identical(names(rset), c("plots", "sites")) + expect_identical(rset$sites$col, names(synthetic(s)$sites)) + + dir <- withr::local_tempdir() + data_card(s, file = dir) + expect_setequal( + list.files(dir), c("plots.data_card.yaml", "sites.data_card.yaml") + ) + expect_identical( + yaml::read_yaml(file.path(dir, "sites.data_card.yaml")), cards$sites + ) +}) + +test_that("the writer quotes and escapes every string", { + txt <- .card_yaml(list(a = "say \"hi\"\\ok", b = list(), c = 2L, d = 0.5)) + expect_identical( + txt, + "a: \"say \\\"hi\\\"\\\\ok\"\nb: []\nc: 2\nd: 0.500000\n" + ) + expect_identical(yaml::yaml.load(txt)$a, "say \"hi\"\\ok") +}) From 27d7347933a28020432d5e6f19ba1a5e9febbea9 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:16:48 +1030 Subject: [PATCH 38/66] docs: data card section in the getting-started vignette --- vignettes/getting_started.Rmd | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/vignettes/getting_started.Rmd b/vignettes/getting_started.Rmd index bf361e2..4973d20 100644 --- a/vignettes/getting_started.Rmd +++ b/vignettes/getting_started.Rmd @@ -245,6 +245,48 @@ The genotype column `gen` appears in both tables and is masked to the same codes in each, so the field and laboratory tables still join. See *Confidentiality and the threat model* for the set-level controls. +### Describing the clone: the data card + +A collaborator planning an analysis needs to know the shape of the data +before any code is written: how many rows, which columns are the design, +which is the outcome, how much is missing. `data_card()` writes those +facts down without a single value of any column. It reads only the clone +and its role table, which `roles()` returns in the clone's own column +names: + +```{r data-card-roles} +knitr::kable( + roles(m), + caption = "The role table of the clone, as the data card reads it." +) +``` + +The role table says which columns are design columns, but not which of +them give a plot's position or identify the plot. The custodian names +those when writing the card: + +```{r data-card} +card_file <- tempfile(fileext = ".yaml") +card <- data_card( + m, + file = card_file, + name = "alpha_trial", + spatial = c("row", "col"), + unit = "plot" +) +writeLines(head(readLines(card_file), 30)) +``` + +Every count carries a provenance class. `structure` marks a count that is +exact in the clone and so equal on the real table, such as the number of +missing values or the number of genotypes. `clone_estimate` marks one +that was re-simulated, such as the number of distinct yields. `analyst` +marks what was named in the call. Level labels never appear: the card +says there are 24 genotypes and does not list them. When column names +are aliased, the card uses the aliases. For a set, `data_card(ms)` +returns one card per table, and `data_card(ms, file = dir)` writes one +file per table. + ### Multi-environment structure For a multi-environment trial, `detect_design()` answers three questions From d6ff74763f98a3280d6ecd2030fdba22668b9da6 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:16:52 +1030 Subject: [PATCH 39/66] fix: read 0.16.0 recipe properties safely and print them redacted A recipe or bundle saved by masque 0.15.0 lacks conform, events, grain and info; print and apply_recipe now read them through .prop_or(). The recipe print gains a conformance count line and grain units with their column counts; bundle and masque_set prints name set-aside tables. --- R/grain.R | 13 +++ R/recipe_apply.R | 2 +- R/recipe_print.R | 34 +++++- R/recipe_set.R | 11 ++ tests/testthat/test-fix0160-old-recipe.R | 109 ++++++++++++++++++ .../testthat/test-fix0160-redaction-surface.R | 52 +++++++++ 6 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-fix0160-old-recipe.R diff --git a/R/grain.R b/R/grain.R index f9192f5..64df312 100644 --- a/R/grain.R +++ b/R/grain.R @@ -207,3 +207,16 @@ pairs <- data.table::data.table(g = gid[obs], x = x[obs]) pairs[, data.table::uniqueN(x), by = g][, all(V1 <= 1L)] } + +# Unit and column count only; aliased unit names show their alias. +.grain_print_line <- function(g, name_map = NULL) { + unit <- g$unit + hits <- unit %in% names(name_map) + unit[hits] <- unlist(name_map[unit[hits]], use.names = FALSE) + n <- length(g$cols) + sprintf( + "%s: %d column%s, synthesised per %s%s", + paste(unit, collapse = " : "), n, if (n == 1L) "" else "s", + g$synthesised, if (nzchar(g$reason)) sprintf(" (%s)", g$reason) else "" + ) +} diff --git a/R/recipe_apply.R b/R/recipe_apply.R index 8511e9b..be94a11 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -73,7 +73,7 @@ apply_recipe <- function(original, rec, check_integrity = TRUE) { # conform_table() merges and type changes the clone was built on. original <- .apply_cleaning_forward(original, rec@cleaning) # A recipe saved before 0.16.0 has no `conform` property at all. - conform <- if (S7::prop_exists(rec, "conform")) rec@conform else NULL + conform <- .prop_or(rec, "conform", NULL) original <- .apply_conform_forward(original, conform) dropped <- .recipe_dropped_cols(rec) diff --git a/R/recipe_print.R b/R/recipe_print.R index aca56f2..ccd5529 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -39,7 +39,22 @@ local(S7::method(print, masque_recipe) <- function(x, ...) { } } - events <- S7::prop(x, "events") + conform <- .prop_or(x, "conform", NULL) + if (!is.null(conform)) { + line <- .conform_print_line(conform) + cli::cli_bullets(c("*" = "{line}")) + } + + grain <- .prop_or(x, "grain", list()) + if (length(grain)) { + cli::cli_h2("Grain ({length(grain)} unit{?s})") + for (g in grain) { + line <- .grain_print_line(g, x@column_name_map) + cli::cli_bullets(c("*" = "{line}")) + } + } + + events <- .prop_or(x, "events", list()) if (length(events)) { cli::cli_h2("Linked events ({length(events)} group{?s})") for (er in events) { @@ -92,6 +107,23 @@ local(S7::method(print, masque_recipe) <- function(x, ...) { invisible(x)}) +# A property of a recipe written by an older masque may be absent. +.prop_or <- function(x, name, default) { + if (S7::prop_exists(x, name)) S7::prop(x, name) else default +} + +# Counts only: merge sources and targets are original labels. +.conform_print_line <- function(conform) { + n_merge <- NROW(conform$merges) + n_type <- NROW(conform$types) + sprintf( + "Conformance: %d label merge%s, %d type change%s", + n_merge, if (n_merge == 1L) "" else "s", + n_type, if (n_type == 1L) "" else "s" + ) +} + + # local(): see the plot method in detect_plot.R. local(S7::method(format, masque_recipe) <- function(x, ...) { sprintf( diff --git a/R/recipe_set.R b/R/recipe_set.R index ed43ca1..8348b88 100644 --- a/R/recipe_set.R +++ b/R/recipe_set.R @@ -48,6 +48,7 @@ local(S7::method(print, masque_set) <- function(x, ...) { "*" = sprintf("Mode: %s", x@mode), "*" = sprintf("Tables: %d", length(x@synthetic)) )) + .print_info_line(x@recipe) for (nm in names(x@synthetic)) { tab <- x@synthetic[[nm]] cli::cli_li( @@ -79,6 +80,15 @@ local(S7::method(print, masque_set) <- function(x, ...) { invisible(x)}) +# Table names only; a set-aside table's content is never read for display. +.print_info_line <- function(bundle) { + info <- .prop_or(bundle, "info", character()) + if (length(info)) { + line <- sprintf("Set aside, not masked: %s", paste(info, collapse = ", ")) + cli::cli_bullets(c("*" = "{line}")) + } +} + # local(): see the plot method in detect_plot.R. local(S7::method(print, masque_recipe_set) <- function(x, ...) { cli::cli_h1("masque_recipe_set") @@ -88,6 +98,7 @@ local(S7::method(print, masque_recipe_set) <- function(x, ...) { "*" = sprintf("Tables: %d", length(x@recipes)), "*" = sprintf("masque version: %s", x@masque_version) )) + .print_info_line(x) # Link names and tables only; the shared maps stay redacted. if (length(x@links)) { diff --git a/tests/testthat/test-fix0160-old-recipe.R b/tests/testthat/test-fix0160-old-recipe.R new file mode 100644 index 0000000..90a4489 --- /dev/null +++ b/tests/testthat/test-fix0160-old-recipe.R @@ -0,0 +1,109 @@ +# Objects as masque 0.15.0 wrote them: the class definition each one carries +# lacks the properties added in 0.16.0, and their attributes are absent. +.age_s7 <- function(x, drop) { + cls <- attr(x, "S7_class") + old_cls <- S7::new_class( + cls@name, package = "masque", + properties = cls@properties[setdiff(names(cls@properties), drop)] + ) + for (p in drop) attr(x, p) <- NULL + attr(x, "S7_class") <- old_cls + x +} +.age_recipe <- function(rec) .age_s7(rec, c("conform", "events", "grain")) + +.old_fixture <- function() { + withr::with_seed(2L, { + g <- expand.grid( + gen = sprintf("G%02d", 1:8), rep = 1:3, trial = sprintf("T%d", 1:4), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 4, 0.5), 2) + g$rain <- c(312.4, 287.9, 401.2, 355.6)[match(g$trial, sprintf("T%d", 1:4))] + g$soil <- c("clay", "loam", "sand", "clay")[match(g$trial, sprintf("T%d", 1:4))] + g + }) +} + +.old_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", + action = if (mode == "collaborate") "alias" else "keep") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, "rain", role = "covariate", action = "scramble") + set_role(r, "soil", role = "covariate", action = "alias") +} + +test_that("a 0.15.0 recipe prints, applies, unmasks, audits and reads back", { + df <- .old_fixture() + for (mode in c("local", "collaborate")) { + m <- suppressWarnings(mask(df, .old_roles(df, mode), mode = mode, seed = 4L)) + rec <- recipe(m) + expect_true(length(rec@grain) > 0L) + old <- .age_recipe(rec) + expect_false(S7::prop_exists(old, "grain")) + + expect_no_error(utils::capture.output(print(old), type = "message")) + expect_no_error(format(old)) + expect_no_error(utils::capture.output(utils::str(old))) + expect_identical(apply_recipe(df, old), apply_recipe(df, rec)) + s <- synthetic(m) + expect_identical(unmask(s, old), unmask(s, rec)) + + m_old <- m + attr(m_old, "recipe") <- old + expect_no_error( + a_old <- audit_mask(m_old, original = df, print = FALSE) + ) + expect_identical( + a_old$leakage_class, + audit_mask(m, original = df, print = FALSE)$leakage_class + ) + + path <- withr::local_tempfile(fileext = ".rds") + saveRDS(old, path) + back <- read_recipe(path) + expect_identical(apply_recipe(df, back), apply_recipe(df, rec)) + } +}) + +test_that("a 0.15.0 recipe bundle prints, applies, unmasks, audits and reads back", { + df <- .old_fixture() + sites <- data.frame( + trial = sprintf("T%d", 1:6), lat_band = rep(c("n", "s"), 3L), + elev = c(120, 340, 95, 410, 220, 180), stringsAsFactors = FALSE + ) + tables <- list(plots = df, sites = sites) + roles <- list( + plots = .old_roles(df, "collaborate"), + sites = set_role( + set_role(propose_roles(sites, mode = "collaborate", detect = FALSE), + "trial", role = "design", action = "keep"), + "elev", role = "covariate", action = "scramble" + ) + ) + ms <- suppressWarnings( + mask_set(tables, roles = roles, mode = "collaborate", seed = 5L, quiet = TRUE) + ) + rs <- recipe(ms) + old <- .age_s7(rs, "info") + old@recipes <- lapply(rs@recipes, .age_recipe) + expect_false(S7::prop_exists(old, "info")) + + expect_no_error(utils::capture.output(print(old), type = "message")) + expect_no_error(format(old)) + expect_identical(apply_recipe(tables, old), apply_recipe(tables, rs)) + syn <- synthetic(ms) + expect_identical(unmask(syn, old), unmask(syn, rs)) + + ms_old <- ms + attr(ms_old, "recipe") <- old + expect_no_error(utils::capture.output(print(ms_old), type = "message")) + expect_no_error(audit_mask(ms_old, print = FALSE)) + + path <- withr::local_tempfile(fileext = ".rds") + saveRDS(old, path) + expect_identical(apply_recipe(tables, read_recipe(path)), + apply_recipe(tables, rs)) +}) diff --git a/tests/testthat/test-fix0160-redaction-surface.R b/tests/testthat/test-fix0160-redaction-surface.R index 3e94fd2..9e3ca6b 100644 --- a/tests/testthat/test-fix0160-redaction-surface.R +++ b/tests/testthat/test-fix0160-redaction-surface.R @@ -172,3 +172,55 @@ test_that("a knitted document shows no label or seed", { } expect_match(text, "masque_recipe_set") }) + +.leak_conform_grain <- function() { + df <- .leak_plots() + df$soil <- ifelse( + df$site == "LEAKTOKEN_ALPHA", "LEAKTOKEN_ALPHA_clay", "LEAKTOKEN_BETA_loam" + ) + df$note <- rep(c("LEAKTOKEN_BETA_X", "LEAKTOKEN_BETA_x"), c(50L, 10L)) + cf <- conform_table(df, merge_labels = "auto", quiet = TRUE) + r <- .leak_roles(cf$data[, c("site", "rep", "gen", "yield")]) + extra <- propose_roles(cf$data, mode = "collaborate") + extra <- extra[extra$col %in% c("soil", "note")] + r <- set_role(rbind(r, extra), c("soil", "note"), role = "covariate", + action = "alias") + suppressWarnings(mask(cf, r, mode = "collaborate", seed = 424242L)) +} + +test_that("the recipe print counts conform decisions and names grain units only", { + skip_if_not_installed("knitr") + m <- .leak_conform_grain() + rec <- recipe(m) + expect_identical(nrow(rec@conform$merges), 1L) + expect_true(length(rec@grain) > 0L) + + out <- capture_full(print(rec)) + expect_match(out, "Conformance: 1 label merge", fixed = TRUE) + expect_match(out, "type change", fixed = TRUE) + expect_match(out, "Grain", fixed = TRUE) + expect_match(out, "site: 1 column", fixed = TRUE) + .expect_no_tokens(.display_routes(rec), "masque_recipe with conform and grain") + .expect_no_tokens(.display_routes(m), "masque with conform and grain") +}) + +test_that("the bundle print names set-aside tables and shows none of their content", { + skip_if_not_installed("knitr") + sites <- data.frame( + site = factor(rep(c("LEAKTOKEN_ALPHA", "LEAKTOKEN_BETA"), 5L)), + year = rep(2016:2020, each = 2L), + rain = c(410, 395, 520, 480, 300, 350, 610, 590, 455, 470) + ) + readme <- data.frame(text = c("LEAKTOKEN_ALPHA is the north site", "x")) + tables <- list(plots = .leak_plots(), sites = sites, readme = readme) + roles <- lapply(tables[c("plots", "sites")], .leak_roles) + ms <- mask_set( + tables, roles = roles, mode = "collaborate", seed = 424242L, + quiet = TRUE, info = "readme" + ) + out <- capture_full(print(recipe(ms))) + expect_match(out, "Set aside, not masked: readme", fixed = TRUE) + expect_match(capture_full(print(ms)), "readme", fixed = TRUE) + .expect_no_tokens(.display_routes(recipe(ms)), "bundle with info") + .expect_no_tokens(.display_routes(ms), "masque_set with info") +}) From 6b914a5591100b8492e0d8135a1a50261f02c5f6 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:16:59 +1030 Subject: [PATCH 40/66] style: one-line comments in the numeric treatment change --- R/alias_levels.R | 3 +-- R/recipe_apply.R | 6 ++---- tests/testthat/test-fix0160-numeric-treatment.R | 7 ++----- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/R/alias_levels.R b/R/alias_levels.R index 974f550..f2743b7 100644 --- a/R/alias_levels.R +++ b/R/alias_levels.R @@ -112,8 +112,7 @@ is_aliasable_level_vector <- function(x) { is.factor(x) || is.character(x) || is.logical(x) } -# Exact character form of numeric values, the key a level map holds them by: -# the first form that reads back to the same double, hexadecimal as the last. +# Exact character key of numeric values: the first form that reads back. .level_key <- function(x) { if (!is.numeric(x)) { return(as.character(x)) diff --git a/R/recipe_apply.R b/R/recipe_apply.R index 9800cd9..0c8d024 100644 --- a/R/recipe_apply.R +++ b/R/recipe_apply.R @@ -207,8 +207,7 @@ unmask <- function(x, rec, column = NULL) { } if (is.atomic(x) && is.null(dim(x))) { - # Numeric, logical and date input passes through unchanged, except a - # numeric treatment the recipe permuted. + # Numeric, logical and date input passes through, bar a mapped column. numeric_map <- is.numeric(x) && !is.null(column) && column %in% names(rec@level_maps) if (!is.character(x) && !is.factor(x) && !numeric_map) { @@ -352,8 +351,7 @@ unmask <- function(x, rec, column = NULL) { } } -# Internal: numeric values from their exact keys, in the storage class of -# `like`. +# Internal: numeric values from their exact keys, in the class of `like`. .numeric_from_key <- function(key, like) { num <- suppressWarnings(as.numeric(key)) if (is.integer(like) || identical(like, "integer")) { diff --git a/tests/testthat/test-fix0160-numeric-treatment.R b/tests/testthat/test-fix0160-numeric-treatment.R index 4351414..9eba9c3 100644 --- a/tests/testthat/test-fix0160-numeric-treatment.R +++ b/tests/testthat/test-fix0160-numeric-treatment.R @@ -1,9 +1,6 @@ -# A numeric treatment (a dose, an N rate) can be aliased or permuted, and in -# collaborate mode it is aliased by default, so a dose trial's default clone is -# not blocked by the audit. +# A numeric treatment can be aliased or permuted; collaborate aliases it. -# Split plot modelled on agridat::yates.oats: 6 blocks, 3 varieties on whole -# plots, nitrogen 0 / 0.2 / 0.4 / 0.6 on sub-plots, two missing doses. +# Split plot after agridat::yates.oats, nitrogen 0-0.6, two missing doses. .fx_dose <- function() { withr::with_seed(11L, { g <- expand.grid( From cd172faa0f2ad6bcb8a20300d27a1ff6abd5851a Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:17:07 +1030 Subject: [PATCH 41/66] fix(jitter): collaborate jitter no longer clips onto the real extremes The clip to the observed range copied the real minimum and maximum into the clone. Values may now sit up to half a resolution step outside the range. Event gaps keep a lower bound of zero, since a negative gap would reorder the times. Collaborate digests in two tests re-taken. --- R/events.R | 4 +- R/synthesise_numeric_jitter.R | 33 +++++----- tests/testthat/test-fix0160-grain.R | 11 ++-- tests/testthat/test-fix0160-jitter-noclip.R | 62 +++++++++++++++++++ tests/testthat/test-fix0160-reproducibility.R | 6 +- .../testthat/test-synthesise_numeric_jitter.R | 14 +++-- 6 files changed, 98 insertions(+), 32 deletions(-) create mode 100644 tests/testthat/test-fix0160-jitter-noclip.R diff --git a/R/events.R b/R/events.R index d9cc8f7..91d625e 100644 --- a/R/events.R +++ b/R/events.R @@ -264,7 +264,9 @@ x_new <- synthesise_numeric_local(x_obs) if (isTRUE(jitter)) { for (j in seq_along(x_new)) { - x_new[[j]] <- synthesise_numeric_collaborate(x_obs[[j]], x_new[[j]]) + x_new[[j]] <- synthesise_numeric_collaborate( + x_obs[[j]], x_new[[j]], lower = 0 + ) } } as.matrix(x_new)[seq_len(n), , drop = FALSE] diff --git a/R/synthesise_numeric_jitter.R b/R/synthesise_numeric_jitter.R index 9c06d77..dc6a6cc 100644 --- a/R/synthesise_numeric_jitter.R +++ b/R/synthesise_numeric_jitter.R @@ -3,11 +3,13 @@ #' Layered on top of `synthesise_numeric_local()` in collaborate mode. #' Takes a synthetic column (drawn from the observed multiset by the #' local copula path) and perturbs it with uniform jitter inside the -#' observed measurement resolution, then re-clips to the observed range. +#' observed measurement resolution. The result is not clipped to the +#' observed range, since a clip copies the real minimum and maximum into the +#' clone; a value can fall up to half a resolution step outside it. #' Whole-numbered columns (integer class, or a double whose observed values -#' are all whole) get bounded stochastic rounding, so the output stays -#' whole-numbered in its original storage class without escaping the -#' observed range. A column with at most two distinct observed values is +#' are all whole) get stochastic rounding, so the output stays +#' whole-numbered in its original storage class. A column with at most two +#' distinct observed values is #' left as drawn: the local copula draw already returns observed values, #' and jitter followed by rounding would flip a share of them. #' @@ -16,14 +18,17 @@ #' the distribution shape and integer storage class intact. #' #' @param x_obs Original observed numeric vector (used to determine -#' resolution and clipping bounds). +#' resolution). #' @param x_new Synthetic vector to perturb (same length as `x_obs`). +#' @param lower Hard lower bound applied after jitter and rounding, for a +#' quantity that cannot be negative by definition (event gaps); `-Inf` +#' (the default) applies none. #' #' @return A numeric vector of the same length and storage class as #' `x_obs`. #' @keywords internal #' @noRd -synthesise_numeric_collaborate <- function(x_obs, x_new) { +synthesise_numeric_collaborate <- function(x_obs, x_new, lower = -Inf) { if (length(x_obs) != length(x_new)) { cli::cli_abort("`x_obs` and `x_new` must have the same length.") } @@ -40,19 +45,11 @@ synthesise_numeric_collaborate <- function(x_obs, x_new) { x_new <- x_new + stats::runif(length(x_new), -res / 2, res / 2) } - lo <- suppressWarnings(min(x_obs, na.rm = TRUE)) - hi <- suppressWarnings(max(x_obs, na.rm = TRUE)) - if (is.finite(lo) && is.finite(hi)) { - x_new <- pmin(pmax(x_new, lo), hi) - } - + x_new <- pmax(x_new, lower) if (is_whole) { - if (is.finite(lo) && is.finite(hi)) { - x_new <- .bounded_stochastic_round(x_new, lo, hi, as_int = is_int) - } else { - x_new <- round(x_new) - if (is_int) x_new <- as.integer(x_new) - } + x_new <- .bounded_stochastic_round( + x_new, ceiling(lower), Inf, as_int = is_int + ) } x_new diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R index adcefb9..3af956a 100644 --- a/tests/testthat/test-fix0160-grain.R +++ b/tests/testthat/test-fix0160-grain.R @@ -230,7 +230,10 @@ test_that("collaborate unit values are not confined to the original values", { )) vals <- .fx_unit_values(synthetic(m)$aet, df$trial) expect_lt(mean(vals %in% orig), 1) - expect_true(all(vals >= min(orig) & vals <= max(orig))) + # No clip: a value may sit up to half a resolution step past the range. + half <- masque:::.detect_resolution(orig) / 2 + expect_true(all(vals >= min(orig) - half & vals <= max(orig) + half)) + expect_false(any(vals %in% range(orig))) expect_identical(recipe(m)@grain[[1]]$n_units, 6L) # Soil is aliased after the unit-level permutation. expect_false(any(synthetic(m)$soil %in% df$soil)) @@ -292,16 +295,16 @@ test_that("fewer than five units in collaborate mode fall back to rows", { }) test_that("grain = FALSE is the row-level clone of masque 0.15", { - # Digests of the same calls on the unmodified 0.16.0 base (12ead24). + # Base 12ead24 digests; collaborate re-taken after the jitter clip went. base <- c( local_FALSE = "65154f4d80860124e54c67e3fcccc934d3757823dac158f5d2dda8ac98764601", local_TRUE = "789a764980d220b4924e1367dd6e47fe8db31c903d9b02196d5635390a689808", collaborate_FALSE = - "bb8c3fa8180eb32a18887d517ca9e709b5049f8e5306bd6c5b6294b6fbbd2f39", + "684cc07c5fa22d2c70bd897250250a0f79fde0b2091b149b539623f688ac5de1", collaborate_TRUE = - "cf60a1627fc0b9aff9c81a6ffa1fe5c733e7605134d9b9ba0bd3e0e804fc106f" + "154490c0135a592e49167c306127438a6723266fb96142762682674673db10e6" ) df <- .fx_met() for (cb in .fx_combos) { diff --git a/tests/testthat/test-fix0160-jitter-noclip.R b/tests/testthat/test-fix0160-jitter-noclip.R new file mode 100644 index 0000000..8221477 --- /dev/null +++ b/tests/testthat/test-fix0160-jitter-noclip.R @@ -0,0 +1,62 @@ +# Collaborate jitter does not clip onto the real minimum and maximum. + +test_that("jittered draws at the extremes do not land on the real extremes", { + x_obs <- withr::with_seed(1L, round(stats::rgamma(200, 2, 1), 3)) + lo <- min(x_obs) + hi <- max(x_obs) + x_new <- c(rep(lo, 100), rep(hi, 100)) + out <- withr::with_seed(2L, { + masque:::synthesise_numeric_collaborate(x_obs, x_new) + }) + expect_false(any(out == lo)) + expect_false(any(out == hi)) + expect_true(any(out < lo)) + expect_true(any(out > hi)) +}) + +test_that("whole-valued jitter rounds without a clip to the observed range", { + x_obs <- withr::with_seed(1L, sample(1L:20L, 400, replace = TRUE)) + out <- withr::with_seed(2L, { + masque:::synthesise_numeric_collaborate(x_obs, rep(20L, 400)) + }) + expect_type(out, "integer") + expect_true(any(out > 20L)) +}) + +test_that("a lower bound, when given, is still respected", { + x_obs <- withr::with_seed(1L, c(0, round(stats::rexp(199), 2))) + out <- withr::with_seed(2L, { + masque:::synthesise_numeric_collaborate(x_obs, rep(0, 200), lower = 0) + }) + expect_true(all(out >= 0)) + xi <- withr::with_seed(1L, sample(0L:9L, 200, replace = TRUE)) + oi <- withr::with_seed(2L, { + masque:::synthesise_numeric_collaborate(xi, rep(0L, 200), lower = 0) + }) + expect_true(all(oi >= 0L)) +}) + +test_that("no grained unit value of the E3 fixture is a real extreme", { + df <- withr::with_seed(1L, { + g <- expand.grid( + gen = sprintf("G%02d", 1:10), rep = 1:3, trial = sprintf("T%d", 1:6), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.4), 2) + g$aet <- c(312.456, 287.901, 401.233, 355.678, 298.112, 376.543)[ + match(g$trial, sprintf("T%d", 1:6)) + ] + g + }) + r <- suppressWarnings(propose_roles(df, mode = "collaborate", detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", action = "alias") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, "aet", role = "covariate", action = "scramble") + for (s in 1:5) { + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = s)) + aet <- synthetic(m)$aet + expect_false(any(aet %in% range(df$aet))) + expect_false(any(synthetic(m)$yield %in% range(df$yield))) + } +}) diff --git a/tests/testthat/test-fix0160-reproducibility.R b/tests/testthat/test-fix0160-reproducibility.R index d8cfc89..1a2adfa 100644 --- a/tests/testthat/test-fix0160-reproducibility.R +++ b/tests/testthat/test-fix0160-reproducibility.R @@ -60,16 +60,16 @@ } test_that("a table with no grained column clones as on the 0.16.0 base", { - # Digests of the same calls on the unmodified base (12ead24). + # Base 12ead24 digests; collaborate re-taken after the jitter clip went. base <- c( local_FALSE = "ba60bae634f27d436a003f1ab22bc4146a77e64aea643d9716349d0325499097", local_TRUE = "dbecf2f681904bbef71d17f1169cddca7eb78ff67e85040611a038d0046c75c3", collaborate_FALSE = - "f7e80803ea82d18e6647d2ef6d085eb1d3614ea74fe090aa7393007912b74ecc", + "3c4422bc3cbe1ceed6e709e396459ed71e9dbb1a0b32cc7716f2d5b8d2a7fe73", collaborate_TRUE = - "3290ab8bf490881ab9ec6fe683e1745b57ac9a76d3c082c51480a76cd07c3727" + "51faf1085e3f60405f8a21003c09b273a759cb83238197acfa53406185b5863a" ) df <- .repro_fixture() for (mode in c("local", "collaborate")) { diff --git a/tests/testthat/test-synthesise_numeric_jitter.R b/tests/testthat/test-synthesise_numeric_jitter.R index 61af4fb..cc7494e 100644 --- a/tests/testthat/test-synthesise_numeric_jitter.R +++ b/tests/testthat/test-synthesise_numeric_jitter.R @@ -30,12 +30,13 @@ test_that(".bounded_stochastic_round preserves average (expectation)", { expect_equal(mean(out), 3.5, tolerance = 0.05) }) -test_that("synthesise_numeric_collaborate stays within observed range", { +test_that("synthesise_numeric_collaborate stays within half a step of range", { set.seed(1) x_obs <- rgamma(200, 2, 1) x_new <- masque:::synthesise_numeric_local(data.frame(z = x_obs))$z out <- masque:::synthesise_numeric_collaborate(x_obs, x_new) - expect_true(all(out >= min(x_obs)) && all(out <= max(x_obs))) + half <- masque:::.detect_resolution(x_obs) / 2 + expect_true(all(out >= min(x_obs) - half) && all(out <= max(x_obs) + half)) }) test_that("synthesise_numeric_collaborate preserves integer class", { @@ -45,7 +46,8 @@ test_that("synthesise_numeric_collaborate preserves integer class", { x_new <- masque:::synthesise_numeric_local(data.frame(z = x_obs))$z out <- masque:::synthesise_numeric_collaborate(x_obs, x_new) expect_type(out, "integer") - expect_true(all(out >= min(x_obs)) && all(out <= max(x_obs))) + # Unclipped rounding can step one unit past the observed range. + expect_true(all(out >= min(x_obs) - 1L) && all(out <= max(x_obs) + 1L)) }) test_that("synthesise_numeric_collaborate drops exact-match below local", { @@ -139,8 +141,8 @@ test_that("Collaborate mode preserves integer storage on int outcome columns", { m <- mask(df, r, mode = "collaborate", seed = 1) expect_type(synthetic(m)$grain_count, "integer") rng <- range(synthetic(m)$grain_count, na.rm = TRUE) - expect_gte(rng[1], min(df$grain_count)) - expect_lte(rng[2], max(df$grain_count)) + expect_gte(rng[1], min(df$grain_count) - 1L) + expect_lte(rng[2], max(df$grain_count) + 1L) }) test_that("collaborate keeps a 0/1 double column on its two values", { @@ -171,7 +173,7 @@ test_that("collaborate keeps a whole-valued double column whole", { expect_type(out, "double") kept <- out[!is.na(out)] expect_true(all(kept == round(kept))) - expect_true(all(kept >= 1 & kept <= 2000)) + expect_true(all(kept >= 0 & kept <= 2001)) expect_identical(is.na(out), is.na(x_new)) # Still perturbed, not a copy of the local draw expect_lt(mean(out == x_new, na.rm = TRUE), 0.9) From 7300489f4aebd469065d8246670f09538ca99599 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:17:07 +1030 Subject: [PATCH 42/66] feat(audit): kept outcome is HIGH in collaborate; grain columns graded on excess A collaborate outcome with action keep audits HIGH unless mask() gets allow_kept_outcome = TRUE, which is recorded on the recipe and threaded through mask_set(). Grained columns and whole-valued numeric covariates are graded on their same-row excess over chance, not raw exact matches. A grain synthesised row by row gets an audit note. --- R/audit.R | 15 +++- R/leakage.R | 68 ++++++++++++--- R/mask.R | 17 ++++ R/mask_set.R | 4 + R/recipe.R | 4 + tests/testthat/test-fix0160-audit-grain.R | 84 ++++++++++++++++++ tests/testthat/test-fix0160-audit-joint.R | 8 +- tests/testthat/test-fix0160-kept-outcome.R | 99 ++++++++++++++++++++++ 8 files changed, 283 insertions(+), 16 deletions(-) create mode 100644 tests/testthat/test-fix0160-audit-grain.R create mode 100644 tests/testthat/test-fix0160-kept-outcome.R diff --git a/R/audit.R b/R/audit.R index 4b00ba6..d2eb710 100644 --- a/R/audit.R +++ b/R/audit.R @@ -59,14 +59,21 @@ #' #' \itemize{ #' \item Retained PII-pattern column -> `high`. -#' \item Treatment unaliased in collaborate -> `high`. +#' \item Outcome with action `keep` in collaborate -> `high`, or `medium` +#' when [mask()] was called with `allow_kept_outcome = TRUE`, which the +#' recipe records. +#' \item Treatment unaliased in collaborate -> `high`. A numeric treatment +#' (a dose) is aliased like a categorical one. #' \item Categorical covariate with a frequency-1 level in collaborate -> #' `high`. #' \item Outcome with exact-match-pct > 1\% in collaborate -> `medium`. #' \item Numeric covariate with exact-match-pct > 5\% in collaborate -> #' `medium`. #' \item Two-valued numeric columns skip the two exact-match rules: a 0/1 -#' column matches its own cell about half the time by chance. +#' column matches its own cell about half the time by chance. So do +#' columns synthesised once per design unit (see `grain` in [mask()]) +#' and whole-valued numeric covariates, which repeat real values by +#' construction; all three are graded on `excess_pct` instead. #' \item Content column whose `excess_pct` is at least 10 points and four #' binomial standard deviations in collaborate -> `medium`. #' \item Joint match excess of at least 20 points and four permutation @@ -78,6 +85,10 @@ #' \item Ignore column retained in local -> `low` (informational). #' } #' +#' A column constant within a design unit that was synthesised row by row +#' (`grain = FALSE`, or a collaborate grain with fewer than five units) carries +#' a note saying the clone does not keep that grain. +#' #' As a reference point, a collaborate-mode clone of `agridat::besag.met` #' (yield as the outcome, design columns kept) scores `low` on every #' column, and none of its yields equals the original value. diff --git a/R/leakage.R b/R/leakage.R index 3c2f791..5866297 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -19,6 +19,13 @@ design_cols <- intersect( rec@roles$col[rec@roles$role == "design"], names(df) ) + grain <- if (S7::prop_exists(rec, "grain")) rec@grain else list() + grained <- unlist(lapply(grain, function(g) { + if (identical(g$synthesised, "unit")) g$cols + }), use.names = FALSE) + row_grain <- Filter(function(g) identical(g$synthesised, "row"), grain) + allow_kept_outcome <- S7::prop_exists(rec, "allow_kept_outcome") && + isTRUE(rec@allow_kept_outcome) rows <- lapply(cols, function(col) { i <- which(rec@roles$col == col) @@ -45,6 +52,11 @@ x <- df[[col]] two_valued <- kind %in% .numeric_kinds() && length(unique(x[!is.na(x)])) == 2L + # Grained and whole-valued numerics repeat real values by construction. + is_grained <- col %in% grained + by_excess <- in_synth && kind %in% .numeric_kinds() && + .audit_numeric_resimulated(action) && + (is_grained || (role == "covariate" && .is_whole_valued(x))) exact_match_pct <- NA_real_ comparable_n <- NA_integer_ @@ -61,9 +73,10 @@ # Same-row match against a permutation baseline: categorical, date and # two-valued columns, compared in the original label space. same <- NULL - if (in_synth && (two_valued || + if (in_synth && (two_valued || by_excess || kind %in% c(.categorical_kinds(), .date_kinds()))) { - groups <- if (two_valued && .audit_numeric_resimulated(action)) { + groups <- if ((two_valued || by_excess) && !is_grained && + .audit_numeric_resimulated(action)) { strata } same <- .same_row_stats( @@ -93,13 +106,17 @@ coarsened = is_coarsened, same_row_excess = if (is.null(same)) NA_real_ else same$excess, same_row_sd = if (is.null(same)) NA_real_ else same$sd, - two_valued = two_valued, linked = col %in% linked + two_valued = two_valued, linked = col %in% linked, + by_excess = by_excess, allow_kept_outcome = allow_kept_outcome ) notes <- .audit_notes( role, kind, action, pii, alias_status, mode, leakage_class, exact_match_pct, freq_min, coarsened = is_coarsened, - same = if (col %in% linked) NULL else same, two_valued = two_valued + same = if (col %in% linked) NULL else same, + two_valued = two_valued || by_excess, + allow_kept_outcome = allow_kept_outcome, + row_grain = Filter(function(g) col %in% g$cols, row_grain) ) data.table::data.table( @@ -224,7 +241,7 @@ # Same-row share, permutation chance share (within `groups`), excess and a # binomial SD counted in `units`, for one clone column. .same_row_stats <- function(x, y, map, groups = NULL, units = NULL) { - x_cmp <- if (is.null(map)) .audit_values(x) else as.character(x) + x_cmp <- if (is.null(map)) .audit_values(x) else .level_key(x) y_cmp <- .audit_values(.audit_original_space(y, map)) comparable <- !is.na(x_cmp) & !is.na(y) m <- sum(comparable) @@ -267,7 +284,7 @@ return(y) } inv <- stats::setNames(names(map), unname(map)) - unname(inv[as.character(y)]) + unname(inv[.level_key(y)]) } # Table-level checks against permutation baselines. Design, treatment, id, @@ -406,7 +423,7 @@ for (j in seq_along(cols)) { map <- rec@level_maps[[cols[j]]] x <- if (is.null(map)) .audit_values(df[[cols[j]]]) else - as.character(df[[cols[j]]]) + .level_key(df[[cols[j]]]) y <- .audit_values(.audit_original_space(synth[[cols[j]]], map)) if (is.character(x) != is.character(y)) { x <- as.character(x) @@ -537,10 +554,10 @@ } .classify_leakage <- function( - role, kind, action, pii, mode, in_synth, alias_status, + role, kind, action = NA_character_, pii, mode, in_synth, alias_status, exact_match_pct, freq_min, n_unique_levels, n_rows, coarsened = FALSE, same_row_excess = NA_real_, same_row_sd = NA_real_, two_valued = FALSE, - linked = FALSE + linked = FALSE, by_excess = FALSE, allow_kept_outcome = FALSE ) { collab <- identical(mode, "collaborate") @@ -555,6 +572,13 @@ return(if (collab) "high" else "medium") } + # 1b. Outcome kept as real values beside re-simulated columns in + # collaborate: HIGH, MEDIUM under `allow_kept_outcome = TRUE`. + if (role == "outcome" && collab && in_synth && !is.na(action) && + action == "keep") { + return(if (isTRUE(allow_kept_outcome)) "medium" else "high") + } + # 2. Treatment vocabulary visible in collaborate: HIGH. Both a plain # pass-through and a label permutation expose every real label. if (role == "treatment" && collab && @@ -571,13 +595,13 @@ # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM. Two-valued # columns match by chance about half the time; rule 5b grades them. if (role == "outcome" && collab && !isTRUE(two_valued) && - !is.na(exact_match_pct) && exact_match_pct > 1) { + !isTRUE(by_excess) && !is.na(exact_match_pct) && exact_match_pct > 1) { return("medium") } # 5. Numeric covariate with exact-match-pct > 5% in collaborate: MEDIUM if (role == "covariate" && kind %in% c("numeric", "integer") && - collab && !isTRUE(two_valued) && + collab && !isTRUE(two_valued) && !isTRUE(by_excess) && !is.na(exact_match_pct) && exact_match_pct > 5) { return("medium") } @@ -604,7 +628,8 @@ .audit_notes <- function(role, kind, action, pii, alias_status, mode, leakage_class, exact_match_pct, freq_min, - coarsened = FALSE, same = NULL, two_valued = FALSE) { + coarsened = FALSE, same = NULL, two_valued = FALSE, + allow_kept_outcome = FALSE, row_grain = list()) { collab <- identical(mode, "collaborate") bits <- character() if (pii && !isTRUE(coarsened)) bits <- c(bits, "PII-pattern column name") @@ -619,6 +644,25 @@ if (alias_status == "passthrough" && !is.na(action) && action == "keep" && role != "design" && collab && !isTRUE(coarsened)) { bits <- c(bits, "kept as-is - visible to collaborators") + if (role == "outcome") { + bits <- c(bits, if (isTRUE(allow_kept_outcome)) { + "real outcome values allowed by allow_kept_outcome = TRUE" + } else { + paste0( + "real outcome values; scramble or drop it, or pass ", + "allow_kept_outcome = TRUE" + ) + }) + } + } + for (g in row_grain) { + bits <- c(bits, sprintf( + paste0( + "constant within %s but synthesised row by row (%s); the clone ", + "does not keep that grain" + ), + paste(g$unit, collapse = ":"), g$reason + )) } if (alias_status == "passthrough" && (role == "date" || kind %in% c("date", "datetime")) && diff --git a/R/mask.R b/R/mask.R index 582d280..a7ad899 100644 --- a/R/mask.R +++ b/R/mask.R @@ -201,6 +201,12 @@ #' `coords` (coarsened), giving the column a masking action (`drop` or #' `scramble`), or -- having decided the coordinate is not sensitive -- #' setting this to `TRUE`, which is recorded on the recipe. +#' @param allow_kept_outcome Single logical. In collaborate mode an outcome +#' with the action `keep` carries its real values beside the re-simulated +#' columns, so [audit_mask()] grades it HIGH and the collaborate write gate +#' stops it. Having decided the real outcome may be shared, set this to +#' `TRUE`: the column is then graded medium and the choice is recorded on +#' the recipe. Local mode is unaffected. #' @param quiet Single logical. `TRUE` silences the `masque_mode_downgrade` #' warning raised when a roles table prepared for `"collaborate"` is used #' in `"local"` mode, for a plan that is deliberately shared across both @@ -240,6 +246,7 @@ mask <- function(df, events = NULL, grain = TRUE, allow_unmasked_coords = FALSE, + allow_kept_outcome = FALSE, quiet = FALSE, .shared_maps = list(), .cleaning = NULL, @@ -398,6 +405,13 @@ mask <- function(df, ) } .guard_unmasked_coords(df, roles, coord_cols, allow_unmasked_coords) + if (!is.logical(allow_kept_outcome) || length(allow_kept_outcome) != 1L || + is.na(allow_kept_outcome)) { + cli::cli_abort( + "`allow_kept_outcome` must be a single `TRUE` or `FALSE`.", + class = c("masque_bad_allow_kept_outcome_refusal", "orchestra_refusal") + ) + } warnings_acc <- c( warnings_acc, .check_row_floor(df, roles, mode, event_specs) ) @@ -545,6 +559,8 @@ mask <- function(df, storage_classes = list(), factor_meta = list(), coords = coord_reports, + grain = grain_res$groups, + allow_kept_outcome = allow_kept_outcome, warnings = character(), integrity_fp = "" ) @@ -623,6 +639,7 @@ mask <- function(df, events = event_reports, grain = grain_res$groups, allow_unmasked_coords = isTRUE(allow_unmasked_coords), + allow_kept_outcome = allow_kept_outcome, warnings = warnings_acc, integrity_fp = digest::digest(is.na(df), algo = "sha256") ) diff --git a/R/mask_set.R b/R/mask_set.R index ddce820..ad4a682 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -47,6 +47,8 @@ #' [mask()] for the full account. #' @param ladder Drop order of the conditioning ladder, `"hierarchy"` (default) #' or `"levels"`; passed through to each [mask()] call. +#' @param allow_kept_outcome Single logical, passed to each [mask()] call: +#' `TRUE` allows a kept outcome in a collaborate clone and records it. #' @param quiet Logical; suppress the link / hygiene report, including the #' hint that a table looks like documentation. #' @param sheets For an Excel workbook, the sheets to read (default: all); @@ -92,6 +94,7 @@ mask_set <- function(input, alias_names = FALSE, conditional = FALSE, ladder = c("hierarchy", "levels"), + allow_kept_outcome = FALSE, quiet = FALSE, sheets = NULL, info = NULL, @@ -212,6 +215,7 @@ mask_set <- function(input, alias_names = aln, conditional = conditional, ladder = ladder, + allow_kept_outcome = allow_kept_outcome, .shared_maps = shared_by_table[[nm]], .cleaning = cleaning[[nm]] ) diff --git a/R/recipe.R b/R/recipe.R index 2a10b8b..3a98c4a 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -86,6 +86,10 @@ masque_recipe <- S7::new_class( allow_unmasked_coords = S7::new_property( class = S7::class_logical, default = FALSE ), + # TRUE when a kept outcome was allowed; absent before 0.16.0. + allow_kept_outcome = S7::new_property( + class = S7::class_logical, default = FALSE + ), warnings = S7::new_property( class = S7::class_character, default = character() ), diff --git a/tests/testthat/test-fix0160-audit-grain.R b/tests/testthat/test-fix0160-audit-grain.R new file mode 100644 index 0000000..b0ff739 --- /dev/null +++ b/tests/testthat/test-fix0160-audit-grain.R @@ -0,0 +1,84 @@ +# Grained and whole-valued numeric columns are graded on excess over chance. + +# 8 trials; sowing day and rain per trial, a narrow whole-valued age per row. +.fx_ag <- function() { + withr::with_seed(3L, { + trials <- sprintf("T%d", 1:8) + g <- expand.grid( + gen = sprintf("G%02d", 1:10), rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.5), 2) + g$sow_day <- c(120, 124, 120, 131, 124, 118, 131, 120)[ + match(g$trial, trials) + ] + g$rain <- c(312.4, 287.9, 401.2, 355.6, 298.1, 376.5, 330.2, 344.4)[ + match(g$trial, trials) + ] + g$age <- sample(18:26, nrow(g), replace = TRUE) + g + }) +} + +.fx_ag_roles <- function(df) { + r <- suppressWarnings(propose_roles(df, mode = "collaborate", detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", action = "alias") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + set_role(r, c("sow_day", "rain", "age"), role = "covariate", + action = "scramble") +} + +test_that("ordinary clones of grained and whole-valued columns audit low", { + df <- .fx_ag() + r <- .fx_ag_roles(df) + for (s in 1:5) { + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = s)) + a <- m@audit + for (cl in c("sow_day", "rain", "age")) { + expect_identical(a$leakage_class[a$col == cl], "low", label = cl) + expect_false(is.na(a$excess_pct[a$col == cl])) + } + } +}) + +test_that("a leaky grained or whole-valued column is still flagged", { + df <- .fx_ag() + m <- suppressWarnings(mask(df, .fx_ag_roles(df), mode = "collaborate", + seed = 1)) + for (cl in c("sow_day", "age")) { + leak <- synthetic(m) + leak[[cl]] <- df[[cl]] + a <- masque:::.compute_audit( + data.table::as.data.table(df), leak, recipe(m), "collaborate" + ) + expect_identical(a$leakage_class[a$col == cl], "medium", label = cl) + } + + # As the only content column, the leak is a table-level HIGH. + df2 <- df[, c("trial", "rep", "gen", "sow_day")] + r2 <- .fx_ag_roles(df)[c(1:3, 5), ] + m2 <- suppressWarnings(mask(df2, r2, mode = "collaborate", seed = 1)) + leak <- synthetic(m2) + leak$sow_day <- df2$sow_day + a2 <- masque:::.compute_audit( + data.table::as.data.table(df2), leak, recipe(m2), "collaborate" + ) + expect_identical(a2$leakage_class[a2$col == "sow_day"], "medium") + expect_true("high" %in% attr(a2, "joint")$leakage_class) +}) + +test_that("a grain that fell back to row synthesis is noted in the audit", { + df <- .fx_ag() + df <- df[df$trial %in% c("T1", "T2", "T3", "T4"), ] + r <- .fx_ag_roles(df) + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = 1)) + expect_identical(recipe(m)@grain[[1]]$synthesised, "row") + a <- m@audit + for (cl in recipe(m)@grain[[1]]$cols) { + expect_match( + a$notes[a$col == cl], "synthesised row by row", label = cl + ) + } + expect_false(grepl("row by row", a$notes[a$col == "yield"])) +}) diff --git a/tests/testthat/test-fix0160-audit-joint.R b/tests/testthat/test-fix0160-audit-joint.R index 6a61aab..f1a527c 100644 --- a/tests/testthat/test-fix0160-audit-joint.R +++ b/tests/testthat/test-fix0160-audit-joint.R @@ -191,8 +191,11 @@ test_that("an outcome kept as-is trips the joint checks and blocks the write", { skip_if_not_installed("agridat") spec <- .alpha_spec spec["yield"] <- "outcome/keep" + # allow_kept_outcome lifts the column HIGH, leaving the joint checks alone. w <- expect_warning( - m <- .joint_mask(agridat::john.alpha, spec, seed = 1L), + m <- .joint_mask( + agridat::john.alpha, spec, seed = 1L, allow_kept_outcome = TRUE + ), class = "masque_high_leakage" ) expect_match(conditionMessage(w), "[same-row joint match]", fixed = TRUE) @@ -213,7 +216,8 @@ test_that("an outcome kept as-is trips the joint checks and blocks the write", { suppressWarnings(masque( agridat::john.alpha, roles = .joint_roles(agridat::john.alpha, spec), mode = "collaborate", - seed = 1L, ask = FALSE, quiet = TRUE, out = out + seed = 1L, ask = FALSE, quiet = TRUE, out = out, + allow_kept_outcome = TRUE )), class = "masque_blocked_write" ) diff --git a/tests/testthat/test-fix0160-kept-outcome.R b/tests/testthat/test-fix0160-kept-outcome.R new file mode 100644 index 0000000..63586f6 --- /dev/null +++ b/tests/testthat/test-fix0160-kept-outcome.R @@ -0,0 +1,99 @@ +# A kept outcome audits HIGH in collaborate unless allow_kept_outcome = TRUE. + +.fx_kept <- function() { + withr::with_seed(4L, { + g <- expand.grid( + gen = sprintf("G%d", 1:6), rep = 1:4, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.5), 2) + g$height <- round(stats::rnorm(nrow(g), 90, 6), 1) + g + }) +} + +.fx_kept_roles <- function(df, mode) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = FALSE)) + r <- set_role(r, "rep", role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment") + r <- set_role(r, "yield", role = "outcome", action = "keep") + set_role(r, "height", role = "covariate", action = "scramble") +} + +test_that("a kept outcome audits HIGH in collaborate and blocks the write", { + df <- .fx_kept() + expect_warning( + m <- mask(df, .fx_kept_roles(df, "collaborate"), mode = "collaborate", + seed = 1), + "yield", class = "masque_high_leakage" + ) + expect_identical(m@audit$leakage_class[m@audit$col == "yield"], "high") + expect_match(m@audit$notes[m@audit$col == "yield"], "allow_kept_outcome") + expect_false(recipe(m)@allow_kept_outcome) + expect_error(masque:::.gate_release(m), class = "masque_blocked_write") +}) + +test_that("allow_kept_outcome = TRUE lifts the HIGH and is recorded", { + df <- .fx_kept() + expect_no_warning( + m <- mask(df, .fx_kept_roles(df, "collaborate"), mode = "collaborate", + seed = 1, allow_kept_outcome = TRUE), + class = "masque_high_leakage" + ) + expect_true(recipe(m)@allow_kept_outcome) + row <- m@audit[m@audit$col == "yield", ] + expect_identical(row$leakage_class, "medium") + expect_match(row$notes, "allow_kept_outcome = TRUE") + expect_no_error(masque:::.gate_release(m)) + a <- audit_mask(m, original = df, print = FALSE) + expect_identical(a$leakage_class[a$col == "yield"], "medium") +}) + +test_that("local mode does not grade a kept outcome HIGH", { + df <- .fx_kept() + m <- suppressWarnings( + mask(df, .fx_kept_roles(df, "local"), mode = "local", seed = 1) + ) + a <- audit_mask(m, original = df, print = FALSE) + expect_false(a$leakage_class[a$col == "yield"] == "high") +}) + +test_that("allow_kept_outcome must be a single logical", { + df <- .fx_kept() + expect_error( + mask(df, .fx_kept_roles(df, "collaborate"), mode = "collaborate", + allow_kept_outcome = "yes"), + class = "masque_bad_allow_kept_outcome_refusal" + ) +}) + +test_that("a recipe without the property audits as not allowed", { + df <- .fx_kept() + m <- suppressWarnings(mask( + df, .fx_kept_roles(df, "collaborate"), mode = "collaborate", seed = 1, + allow_kept_outcome = TRUE + )) + rec <- recipe(m) + # As a recipe written before the property existed deserialises. + attr(rec, "allow_kept_outcome") <- NULL + expect_null(attr(rec, "allow_kept_outcome")) + a <- masque:::.compute_audit( + data.table::as.data.table(df), synthetic(m), rec, "collaborate" + ) + expect_identical(a$leakage_class[a$col == "yield"], "high") +}) + +test_that("mask_set() and masque() pass allow_kept_outcome through", { + df <- .fx_kept() + roles <- list(plots = .fx_kept_roles(df, "collaborate")) + ms <- suppressWarnings(mask_set( + list(plots = df), roles = roles, mode = "collaborate", seed = 1, + quiet = TRUE, allow_kept_outcome = TRUE + )) + expect_true(recipe(ms)@recipes$plots@allow_kept_outcome) + mq <- suppressWarnings(masque( + df, roles = .fx_kept_roles(df, "collaborate"), mode = "collaborate", + seed = 1, ask = FALSE, quiet = TRUE, allow_kept_outcome = TRUE + )) + expect_true(recipe(mq)@allow_kept_outcome) +}) From f16be3954a3d71f45e63bd9ac6da226686c80876 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:17:40 +1030 Subject: [PATCH 43/66] feat: mask_set() and masque() pass grain to every table --- R/mask_set.R | 4 ++ R/masque.R | 4 +- tests/testthat/test-fix0160-set-grain.R | 66 +++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 tests/testthat/test-fix0160-set-grain.R diff --git a/R/mask_set.R b/R/mask_set.R index ddce820..ecb5227 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -47,6 +47,8 @@ #' [mask()] for the full account. #' @param ladder Drop order of the conditioning ladder, `"hierarchy"` (default) #' or `"levels"`; passed through to each [mask()] call. +#' @param grain Single logical (default `TRUE`); passed through to each +#' [mask()] call, which synthesises a unit-constant column once per unit. #' @param quiet Logical; suppress the link / hygiene report, including the #' hint that a table looks like documentation. #' @param sheets For an Excel workbook, the sheets to read (default: all); @@ -92,6 +94,7 @@ mask_set <- function(input, alias_names = FALSE, conditional = FALSE, ladder = c("hierarchy", "levels"), + grain = TRUE, quiet = FALSE, sheets = NULL, info = NULL, @@ -212,6 +215,7 @@ mask_set <- function(input, alias_names = aln, conditional = conditional, ladder = ladder, + grain = grain, .shared_maps = shared_by_table[[nm]], .cleaning = cleaning[[nm]] ) diff --git a/R/masque.R b/R/masque.R index eae4d12..39d373c 100644 --- a/R/masque.R +++ b/R/masque.R @@ -86,8 +86,8 @@ #' @param info For a set, names of documentation tables to set aside: not #' masked and not written. See [mask_set()]. #' @param ... Further arguments for the masking verb: [mask()] for one -#' table (for example `coords`, `events`, `allow_unmasked_coords`), -#' [mask_set()] for a set (for example `links`). An argument the verb +#' table (for example `coords`, `events`, `allow_unmasked_coords`, +#' `grain`), [mask_set()] for a set (for example `links`, `grain`). An argument the verb #' does not take is refused (`masque_unused_arg_refusal`). #' #' @return A `masque` object (single-table input) or a `masque_set` diff --git a/tests/testthat/test-fix0160-set-grain.R b/tests/testthat/test-fix0160-set-grain.R new file mode 100644 index 0000000..60d61fd --- /dev/null +++ b/tests/testthat/test-fix0160-set-grain.R @@ -0,0 +1,66 @@ +.grain_set <- function() { + withr::with_seed(21L, { + plots <- expand.grid( + gen = sprintf("G%02d", 1:6), rep = 1:2, site = sprintf("S%d", 1:6), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + plots$yield <- round(stats::rnorm(nrow(plots), 4, 0.5), 2) + plots$aet <- c(312.4, 287.9, 401.2, 355.6, 298.1, 376.5)[ + match(plots$site, sprintf("S%d", 1:6)) + ] + sites <- data.frame( + site = sprintf("S%d", 1:6), elev = c(120, 340, 95, 410, 220, 180), + stringsAsFactors = FALSE + ) + list(plots = plots, sites = sites) + }) +} + +.grain_set_roles <- function(tables) { + rp <- suppressWarnings(propose_roles(tables$plots, mode = "local", + detect = FALSE)) + rp <- set_role(rp, c("site", "rep"), role = "design", action = "keep") + rp <- set_role(rp, "gen", role = "treatment", action = "keep") + rp <- set_role(rp, "yield", role = "outcome", action = "scramble") + rp <- set_role(rp, "aet", role = "covariate", action = "scramble") + rs <- propose_roles(tables$sites, mode = "local", detect = FALSE) + rs <- set_role(rs, "site", role = "design", action = "keep") + rs <- set_role(rs, "elev", role = "covariate", action = "scramble") + list(plots = rp, sites = rs) +} + +.per_site <- function(syn) { + vapply(split(syn$aet, syn$site), function(v) length(unique(v)), integer(1L)) +} + +test_that("mask_set() passes grain to every table", { + tables <- .grain_set() + roles <- .grain_set_roles(tables) + on <- suppressWarnings(mask_set(tables, roles = roles, mode = "local", + seed = 2L, quiet = TRUE)) + off <- suppressWarnings(mask_set(tables, roles = roles, mode = "local", + seed = 2L, quiet = TRUE, grain = FALSE)) + expect_true(all(.per_site(synthetic(on)$plots) == 1L)) + expect_true(any(.per_site(synthetic(off)$plots) > 1L)) + g_on <- recipe(on)@recipes$plots@grain + g_off <- recipe(off)@recipes$plots@grain + expect_identical(g_on[[1L]]$synthesised, "unit") + expect_identical(g_off[[1L]]$synthesised, "row") + expect_error( + mask_set(tables, roles = roles, mode = "local", seed = 2L, quiet = TRUE, + grain = NA), + class = "masque_bad_grain_refusal" + ) +}) + +test_that("masque() passes grain to a set", { + tables <- .grain_set() + roles <- .grain_set_roles(tables) + off <- suppressWarnings(masque(tables, roles = roles, mode = "local", + seed = 2L, ask = FALSE, quiet = TRUE, + grain = FALSE)) + expect_identical(recipe(off)@recipes$plots@grain[[1L]]$synthesised, "row") + on <- suppressWarnings(masque(tables, roles = roles, mode = "local", + seed = 2L, ask = FALSE, quiet = TRUE)) + expect_true(all(.per_site(synthetic(on)$plots) == 1L)) +}) From 69c82a7e622cdb4d0dac2d29bbf05806cf106e72 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:18:38 +1030 Subject: [PATCH 44/66] feat: propose_roles() notes columns mask() will synthesise per unit --- R/propose_roles.R | 29 ++++++++++++-------- tests/testthat/test-fix0160-propose-grain.R | 30 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 tests/testthat/test-fix0160-propose-grain.R diff --git a/R/propose_roles.R b/R/propose_roles.R index 5a87733..e0791f0 100644 --- a/R/propose_roles.R +++ b/R/propose_roles.R @@ -165,22 +165,29 @@ propose_roles <- function(df, mode = c("local", "collaborate"), attr(roles, "mode") <- mode attr(roles, "proposed_actions") <- stats::setNames(roles$action, roles$col) - if (!isTRUE(detect)) { - return(roles) - } - # Structural overlay: detect_design() consumes the name-based proposal. # Need at least 2 rows for detection to be meaningful. - if (nrow(df) < 2L) { - return(roles) + if (isTRUE(detect) && nrow(df) >= 2L) { + ds <- detect_design(df, roles = roles) + if (nrow(ds@recommended_roles) > 0L) { + roles <- .overlay_recommended_roles(roles, ds@recommended_roles, mode) + } + .warn_environment_disclosure(roles, ds, mode) + attr(roles, "design") <- ds } + .note_grain(roles, df, mode) +} - ds <- detect_design(df, roles = roles) - if (nrow(ds@recommended_roles) > 0L) { - roles <- .overlay_recommended_roles(roles, ds@recommended_roles, mode) +# Notes each column mask() would synthesise once per design unit. +.note_grain <- function(roles, df, mode) { + for (g in .detect_grain(df, roles)) { + if (identical(mode, "collaborate") && g$n_units < .MIN_STRATUM) next + unit <- paste(g$unit, collapse = ":") + idx <- match(g$cols, roles$col) + hint <- sprintf("Constant within %s; synthesised per %s.", unit, unit) + data.table::set(roles, i = idx, j = "notes", + value = paste(roles$notes[idx], hint)) } - .warn_environment_disclosure(roles, ds, mode) - attr(roles, "design") <- ds roles } diff --git a/tests/testthat/test-fix0160-propose-grain.R b/tests/testthat/test-fix0160-propose-grain.R new file mode 100644 index 0000000..8100432 --- /dev/null +++ b/tests/testthat/test-fix0160-propose-grain.R @@ -0,0 +1,30 @@ +.pg_met <- function() { + withr::with_seed(3L, { + g <- expand.grid( + genotype = sprintf("G%02d", 1:10), rep = 1:2, trial = sprintf("T%d", 1:6), + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.4), 2) + g$aet <- c(312.4, 287.9, 401.2, 355.6, 298.1, 376.5)[ + match(g$trial, sprintf("T%d", 1:6)) + ] + g$plot_cov <- round(stats::rnorm(nrow(g), 50, 8), 2) + g + }) +} + +test_that("propose_roles() notes a column that mask() will synthesise per unit", { + df <- .pg_met() + for (mode in c("local", "collaborate")) { + for (detect in c(TRUE, FALSE)) { + r <- suppressWarnings(propose_roles(df, mode = mode, detect = detect)) + expect_identical(r$role[r$col == "trial"], "design") + expect_match( + r$notes[r$col == "aet"], "Constant within trial; synthesised per trial.", + fixed = TRUE + ) + expect_false(grepl("synthesised per", r$notes[r$col == "plot_cov"])) + expect_identical(attr(r, "mode"), mode) + } + } +}) From f32a0bb7efb0993531f5b165388af7a6a63dae90 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:20:53 +1030 Subject: [PATCH 45/66] fix: seeded link maps, seed checks, literal cli labels, classed all-keep warning - mask_set() draws link maps under seed, not the caller's RNG state - jitter_coordinates() and synthesise_geospatial() refuse a bad seed with masque_bad_seed_refusal - .report_conformance() shows labels with braces as text instead of evaluating them - the all-keep warning carries class masque_all_keep - .normalise_coords() reads spec entries by exact name --- R/conform_table.R | 7 +- R/jitter_coordinates.R | 22 ++--- R/mask_set.R | 6 +- R/roles_validate.R | 2 +- R/synthesise_geospatial.R | 2 +- tests/testthat/test-fix0160-side-defects.R | 104 +++++++++++++++++++++ 6 files changed, 127 insertions(+), 16 deletions(-) create mode 100644 tests/testthat/test-fix0160-side-defects.R diff --git a/R/conform_table.R b/R/conform_table.R index 20b1362..d62cead 100644 --- a/R/conform_table.R +++ b/R/conform_table.R @@ -523,11 +523,11 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing held <- a[!a$applied, , drop = FALSE] if (nrow(done)) { cli::cli_alert_info("Applied {nrow(done)} decision{?s}:") - cli::cli_ul(sprintf("%s: %s", done$col, done$assumption)) + cli::cli_ul(.cli_literal(sprintf("%s: %s", done$col, done$assumption))) } if (nrow(held)) { cli::cli_alert_warning("Reported, NOT applied ({nrow(held)}):") - cli::cli_ul(sprintf("%s: %s", held$col, held$assumption)) + cli::cli_ul(.cli_literal(sprintf("%s: %s", held$col, held$assumption))) if (identical(unname(x$modes["merge_labels"]), "report") || identical(unname(x$modes["types"]), "report")) { cli::cli_text( @@ -540,3 +540,6 @@ MIN_EDIT_NCHAR <- 4L # below this, a one-character difference means nothing } invisible(NULL) } + +# Data text shown through cli: braces are literal, never glue expressions. +.cli_literal <- function(x) gsub("([{}])", "\\1\\1", x) diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index ef4664e..0251dcb 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -198,7 +198,7 @@ jitter_coordinates <- function(df, lat_col, lon_col, land_fn <- .resolve_land_fn(on_land) withr::local_preserve_seed() - if (!is.null(seed)) set.seed(seed) + if (!is.null(seed)) set.seed(as.integer(.check_seed(seed))) lat <- df[[lat_col]] lon <- df[[lon_col]] @@ -470,10 +470,10 @@ jitter_coordinates <- function(df, lat_col, lon_col, } lapply(coords, function(spec) { if (is.character(spec)) spec <- as.list(spec) - if (is.null(spec$lat) || is.null(spec$lon)) { + if (is.null(spec[["lat"]]) || is.null(spec[["lon"]])) { cli::cli_abort("Each `coords` entry needs `lat` and `lon` column names.") } - for (cn in c(spec$lat, spec$lon)) { + for (cn in c(spec[["lat"]], spec[["lon"]])) { if (!cn %in% names(df)) { cli::cli_abort("Coordinate column {.field {cn}} not found in `df`.") } @@ -486,20 +486,20 @@ jitter_coordinates <- function(df, lat_col, lon_col, # A `c(lat = "a", lon = "b", min_km = 5)` vector coerces the numbers to # strings; coerce them back so both the vector and the list forms work. num <- function(x, default) if (is.null(x)) default else as.numeric(x) - on_land <- spec$on_land + on_land <- spec[["on_land"]] if (is.null(on_land)) { on_land <- TRUE } else if (is.character(on_land)) { on_land <- as.logical(on_land) } list( - lat = as.character(spec$lat), - lon = as.character(spec$lon), - by = .normalise_coord_by(spec$by, df), - method = spec$method %||% "donut", - min_km = num(spec$min_km, 5), - max_km = num(spec$max_km, 20), - sd_km = num(spec$sd_km, 10), + lat = as.character(spec[["lat"]]), + lon = as.character(spec[["lon"]]), + by = .normalise_coord_by(spec[["by"]], df), + method = spec[["method"]] %||% "donut", + min_km = num(spec[["min_km"]], 5), + max_km = num(spec[["max_km"]], 20), + sd_km = num(spec[["sd_km"]], 10), on_land = on_land ) }) diff --git a/R/mask_set.R b/R/mask_set.R index ecb5227..07da58a 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -190,7 +190,11 @@ mask_set <- function(input, # Link detection -> shared alias maps. `.build_shared_maps()` fills each # group's `$map` and indexes the maps by table for the mask() calls. link_groups <- .resolve_links(tables, roles, links, mode) - built <- .build_shared_maps(tables, roles, link_groups, mode) + built <- if (is.null(seed)) { + .build_shared_maps(tables, roles, link_groups, mode) + } else { + with_rng_state(seed, .build_shared_maps(tables, roles, link_groups, mode)) + } link_groups <- built$groups shared_by_table <- built$shared_by_table diff --git a/R/roles_validate.R b/R/roles_validate.R index c5259ca..31917b8 100644 --- a/R/roles_validate.R +++ b/R/roles_validate.R @@ -207,7 +207,7 @@ roles_validate <- function(roles, df = NULL, mode = NULL) { cli::cli_warn(c( "Every action is {.val keep}: nothing will be masked.", i = "The synthetic output will equal the original byte-for-byte." - )) + ), class = "masque_all_keep") } invisible(roles) diff --git a/R/synthesise_geospatial.R b/R/synthesise_geospatial.R index de78c1b..ccec923 100644 --- a/R/synthesise_geospatial.R +++ b/R/synthesise_geospatial.R @@ -122,7 +122,7 @@ synthesise_geospatial <- function(synth, original, } withr::local_preserve_seed() - if (!is.null(seed)) set.seed(seed) + if (!is.null(seed)) set.seed(as.integer(.check_seed(seed))) # 1. Per (anchor, site) replication counts in the original. has_xy <- !is.na(original[[lat_col]]) & !is.na(original[[lon_col]]) diff --git a/tests/testthat/test-fix0160-side-defects.R b/tests/testthat/test-fix0160-side-defects.R new file mode 100644 index 0000000..6c87055 --- /dev/null +++ b/tests/testthat/test-fix0160-side-defects.R @@ -0,0 +1,104 @@ +.link_tables <- function() { + sites <- sprintf("Site%02d", 1:12) + plots <- data.frame( + site = rep(sites, each = 4L), rep = rep(1:4, 12L), + yield = round(seq(3, 6, length.out = 48L), 2), stringsAsFactors = FALSE + ) + meta <- data.frame( + site = sites, elev = seq(100, 650, by = 50), stringsAsFactors = FALSE + ) + list(plots = plots, meta = meta) +} + +.link_roles <- function(tables) { + rp <- propose_roles(tables$plots, mode = "collaborate", detect = FALSE) + rp <- set_role(rp, "site", role = "design", action = "alias") + rp <- set_role(rp, "rep", role = "design", action = "keep") + rp <- set_role(rp, "yield", role = "outcome", action = "scramble") + rm <- propose_roles(tables$meta, mode = "collaborate", detect = FALSE) + rm <- set_role(rm, "site", role = "design", action = "alias") + rm <- set_role(rm, "elev", role = "covariate", action = "scramble") + list(plots = rp, meta = rm) +} + +.link_map <- function(seed, caller_seed) { + tables <- .link_tables() + set.seed(caller_seed) + ms <- suppressWarnings(mask_set( + tables, roles = .link_roles(tables), mode = "collaborate", + seed = seed, quiet = TRUE + )) + recipe(ms)@links[[1L]]$map +} + +test_that("mask_set() link maps follow seed, not the caller's RNG state", { + a <- .link_map(seed = 7L, caller_seed = 1L) + b <- .link_map(seed = 7L, caller_seed = 999L) + expect_identical(a, b) + expect_false(identical(a, .link_map(seed = 8L, caller_seed = 1L))) +}) + +test_that("jitter_coordinates() and synthesise_geospatial() refuse a bad seed", { + df <- data.frame(lat = c(-34.9, -35.1), lon = c(138.6, 138.7)) + for (bad in list(1.5, NA_real_, c(1, 2), Inf, "1")) { + expect_error( + jitter_coordinates(df, "lat", "lon", seed = bad, on_land = FALSE), + class = "masque_bad_seed_refusal" + ) + } + orig <- data.frame( + site = c("a", "a", "b"), lat = c(-34.9, -34.9, -35.1), + lon = c(138.6, 138.6, 138.7) + ) + cent <- list(a = c(lat = -34.9, lon = 138.6), b = c(lat = -35.1, lon = 138.7)) + for (bad in list(1.5, NA_real_, c(1, 2))) { + expect_error( + synthesise_geospatial(orig, orig, "site", "lat", "lon", cent, + seed = bad), + class = "masque_bad_seed_refusal" + ) + } +}) + +test_that("conform_table() reports a label holding braces as text", { + for (lab in c("{x}", "{nrow(done)}")) { + df <- data.frame( + tag = c(rep(paste(lab, "alpha"), 6L), paste(lab, "Alpha"), + rep("plain", 5L)), + stringsAsFactors = FALSE + ) + out <- capture_full( + cf <- conform_table(df, merge_labels = "auto", quiet = FALSE) + ) + expect_identical(nrow(cf$merges), 1L) + expect_match( + gsub("\\s+", " ", out), + sprintf('"%s Alpha" and "%s alpha" are the same category', lab, lab), + fixed = TRUE + ) + expect_false(grepl("list(", out, fixed = TRUE)) + } +}) + +test_that("the all-keep warning carries a masque_ class", { + r <- propose_roles(iris, detect = FALSE) + r <- set_role(r, names(iris), action = "keep") + expect_warning(roles_validate(r, iris), class = "masque_all_keep") +}) + +test_that("a coords entry named latitude is not read as lat", { + df <- data.frame(latitude = c(-34.9, -35.1), lon = c(138.6, 138.7)) + expect_error( + .normalise_coords(list(list(latitude = "latitude", lon = "lon")), df), + "needs `lat` and `lon`" + ) + expect_error( + .normalise_coords(list(list(lat = "latitude", lon = "lon", + method_extra = "x", min = 1)), df), + NA + ) + spec <- .normalise_coords( + list(list(lat = "latitude", lon = "lon", min = 1)), df + ) + expect_identical(spec[[1L]]$min_km, 5) +}) From 787ce44c121e51d30d8731e40b7136e3929add7e Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:21:16 +1030 Subject: [PATCH 46/66] style: one-line comments in events.R, grain.R and the recipe grain property --- R/events.R | 26 ++++---------------------- R/grain.R | 8 +------- R/recipe.R | 4 +--- 3 files changed, 6 insertions(+), 32 deletions(-) diff --git a/R/events.R b/R/events.R index 77d21cf..ec53365 100644 --- a/R/events.R +++ b/R/events.R @@ -1,11 +1,4 @@ -# events.R -- Linked time-to-event outcomes for mask(events =). -# -# A spec names the time and status columns of one group of linked events, in -# the order the events occur. Times are turned into non-negative gaps, rows are -# grouped by their status pattern, and the gaps are re-simulated within each -# pattern, then summed back into times. Status patterns are permuted across -# rows (within treatment when the clone is conditional), so the count of each -# pattern is exact and a gap the pattern holds at zero stays zero. +# Linked events for mask(events =): gaps re-simulated per status pattern. # Normalise and validate mask()'s `events` against the original table and the # validated roles. Returns a list of `list(time =, status =)`. @@ -110,10 +103,7 @@ ) } -# Synthesise one event spec. `strata` is NULL (arm-blind) or a character -# vector of row strata, normally the treatment, within which status patterns -# are permuted and gaps re-simulated. Returns the synthetic event columns and a -# report for the recipe. +# `strata`: NULL (arm-blind) or the row strata patterns are permuted in. .synthesise_events <- function(df, spec, strata, jitter, min_stratum) { n <- nrow(df) k <- length(spec$time) @@ -228,13 +218,7 @@ ) } -# Cap each row's final time at `cap`. The final time moves together with the -# times tied to it by zero gaps, so those gaps stay zero; the earlier times -# are left alone. When the time before that block is already at or past the -# cap, capping would turn a positive gap into a zero gap (a tie the pattern -# need not have) or a negative one, so the whole row is scaled by -# cap / final time instead: order kept, zero gaps zero, positive gaps -# positive. +# A row whose earlier times reach `cap` is scaled, so no gap changes sign. .cap_final_time <- function(times, gaps, cap) { k <- ncol(times) over <- which(times[, k] > cap) @@ -258,9 +242,7 @@ which(pattern %in% names(free_pat)[free_pat]) } -# Draw `n` rows of gaps from the observed gap rows through the empirical -# copula, with collaborate-mode jitter when asked. The pool has at least `n` -# rows; a pool-sized draw is jittered against the pool, then cut to `n`. +# The pool has at least `n` rows; a pool-sized draw is jittered, then cut. .draw_gaps <- function(obs, n, jitter) { x_obs <- as.data.frame(obs) x_new <- synthesise_numeric_local(x_obs) diff --git a/R/grain.R b/R/grain.R index 64df312..5f0a145 100644 --- a/R/grain.R +++ b/R/grain.R @@ -1,10 +1,4 @@ -# grain.R -- The grain of a column, and synthesis at that grain. -# -# A column whose value is fixed for a whole design unit (a trial's rainfall, a -# site's soil class) has the unit as its grain. mask() synthesises such columns -# once per unit and broadcasts each unit's new value to the unit's rows, as the -# geomask does for coordinates, so the clone carries one value per unit and -# averaging a unit's rows recovers nothing. +# A column constant within a design unit is synthesised once per unit. #' Detect the grain of the columns mask() re-simulates #' diff --git a/R/recipe.R b/R/recipe.R index 2a10b8b..6ff1fcb 100644 --- a/R/recipe.R +++ b/R/recipe.R @@ -77,9 +77,7 @@ masque_recipe <- S7::new_class( # One entry per linked-event spec: columns, stratification, pattern count # and the small patterns that were pooled. Empty before 0.16.0. events = S7::new_property(class = S7::class_list, default = list()), - # One entry per design unit holding grained columns: the unit, its - # columns, the unit count, and whether they were synthesised per "unit" - # or per "row" with the reason. Empty before 0.16.0. + # One entry per design unit holding grained columns; see .detect_grain(). grain = S7::new_property(class = S7::class_list, default = list()), # TRUE when the caller deliberately wrote a real coordinate through # unmasked. FALSE, and absent on a recipe written before 0.11.0. From 54289fe2baa77d7fa5e4260aee42a083890613a8 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:24:41 +1030 Subject: [PATCH 47/66] docs: document data_card and roles, five-row mask_set example with aliased site The mask_set() example had 4-row tables, which the five-row floor now refuses in collaborate mode, and its comment claimed shared site aliases while site, a design column, was kept as is. --- NAMESPACE | 2 + R/alias_levels.R | 2 +- R/mask_set.R | 14 +++--- _pkgdown.yml | 2 + man/audit_mask.Rd | 15 +++++- man/data_card.Rd | 114 ++++++++++++++++++++++++++++++++++++++++++++++ man/mask.Rd | 8 ++++ man/mask_set.Rd | 22 ++++++--- man/masque.Rd | 4 +- man/roles.Rd | 41 +++++++++++++++++ 10 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 man/data_card.Rd create mode 100644 man/roles.Rd diff --git a/NAMESPACE b/NAMESPACE index a3db22c..62cfbc0 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -6,6 +6,7 @@ export(apply_recipe) export(audit_mask) export(clean_table) export(conform_table) +export(data_card) export(detect_design) export(jitter_coordinates) export(mask) @@ -18,6 +19,7 @@ export(read_set) export(recipe) export(reveal_maps) export(role_options) +export(roles) export(roles_validate) export(save_recipe) export(set_role) diff --git a/R/alias_levels.R b/R/alias_levels.R index f2743b7..36ef8da 100644 --- a/R/alias_levels.R +++ b/R/alias_levels.R @@ -36,7 +36,7 @@ #' cost, not a differential-privacy guarantee. #' #' A numeric vector (a dose, an N rate) is aliased through its exact -#' character form ([.level_key()]), so the map restores every value. +#' character form, so the map restores every value. #' #' @param x A factor, character, logical, or numeric vector. #' @param prefix Character scalar used as the alias prefix (e.g., `"trt"` diff --git a/R/mask_set.R b/R/mask_set.R index f89e8a5..f0b895e 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -72,16 +72,18 @@ #' @examples #' tables <- list( #' plots = data.frame( -#' site = c("A", "A", "B", "B"), -#' gen = c("x", "y", "x", "y"), -#' yield = c(3.1, 2.9, 4.0, 3.7) +#' site = rep(c("A", "B", "C", "D", "E"), each = 2), +#' gen = rep(c("x", "y"), 5), +#' yield = c(3.1, 2.9, 4.0, 3.7, 3.3, 3.5, 2.8, 3.0, 3.9, 3.6) #' ), #' sites = data.frame( -#' site = c("A", "B"), -#' rainfall = c(420, 560) +#' site = c("A", "B", "C", "D", "E"), +#' rainfall = c(420, 560, 390, 610, 480) #' ) #' ) -#' m <- mask_set(tables, mode = "collaborate", seed = 1, quiet = TRUE) +#' roles <- lapply(tables, propose_roles, mode = "collaborate") +#' roles <- lapply(roles, set_role, "site", action = "alias") +#' m <- mask_set(tables, roles = roles, seed = 1, quiet = TRUE) #' synthetic(m)$plots$site #' synthetic(m)$sites$site # same aliases -> the join still works #' diff --git a/_pkgdown.yml b/_pkgdown.yml index 0fc4405..bf60104 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -38,6 +38,8 @@ reference: - write_set - synthetic - recipe + - roles + - data_card - title: Recipe round-trip desc: Persist a recipe and re-target a pipeline between original and synthetic. diff --git a/man/audit_mask.Rd b/man/audit_mask.Rd index 1e9df43..06f1c00 100644 --- a/man/audit_mask.Rd +++ b/man/audit_mask.Rd @@ -82,14 +82,21 @@ Classification heuristics: \itemize{ \item Retained PII-pattern column -> \code{high}. -\item Treatment unaliased in collaborate -> \code{high}. +\item Outcome with action \code{keep} in collaborate -> \code{high}, or \code{medium} +when \code{\link[=mask]{mask()}} was called with \code{allow_kept_outcome = TRUE}, which the +recipe records. +\item Treatment unaliased in collaborate -> \code{high}. A numeric treatment +(a dose) is aliased like a categorical one. \item Categorical covariate with a frequency-1 level in collaborate -> \code{high}. \item Outcome with exact-match-pct > 1\\% in collaborate -> \code{medium}. \item Numeric covariate with exact-match-pct > 5\\% in collaborate -> \code{medium}. \item Two-valued numeric columns skip the two exact-match rules: a 0/1 -column matches its own cell about half the time by chance. +column matches its own cell about half the time by chance. So do +columns synthesised once per design unit (see \code{grain} in \code{\link[=mask]{mask()}}) +and whole-valued numeric covariates, which repeat real values by +construction; all three are graded on \code{excess_pct} instead. \item Content column whose \code{excess_pct} is at least 10 points and four binomial standard deviations in collaborate -> \code{medium}. \item Joint match excess of at least 20 points and four permutation @@ -101,6 +108,10 @@ local mode they are reported in the notes and graded \code{low}. \item Ignore column retained in local -> \code{low} (informational). } +A column constant within a design unit that was synthesised row by row +(\code{grain = FALSE}, or a collaborate grain with fewer than five units) carries +a note saying the clone does not keep that grain. + As a reference point, a collaborate-mode clone of \code{agridat::besag.met} (yield as the outcome, design columns kept) scores \code{low} on every column, and none of its yields equals the original value. diff --git a/man/data_card.Rd b/man/data_card.Rd new file mode 100644 index 0000000..d215489 --- /dev/null +++ b/man/data_card.Rd @@ -0,0 +1,114 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/data_card.R +\name{data_card} +\alias{data_card} +\title{Write a data card for a masked clone} +\usage{ +data_card( + m, + file = NULL, + name = NULL, + env = NULL, + spatial = NULL, + unit = NULL, + primary_event = 1L +) +} +\arguments{ +\item{m}{A \code{masque} object from \code{\link[=mask]{mask()}}, or a \code{masque_set} from +\code{\link[=mask_set]{mask_set()}}.} + +\item{file}{Where to write the card in YAML, or \code{NULL} (the default) to +write nothing. For a set, a directory, which receives one file per +table named \verb{
.data_card.yaml}.} + +\item{name}{The table name on the card. Defaults to \code{"clone"} for a +single table, and to the table names for a set (where \code{name} may give +one name per table, in order). A name must start with a letter and +hold only letters, digits, \verb{_} and \code{.}. Any other name is refused with +\code{masque_bad_card_name_refusal}, never cleaned.} + +\item{env, spatial, unit}{Optional names of clone columns: the environment +columns (trials, sites, years), the columns giving a spatial or temporal +position, and the one column identifying the unit of repeated +measurement. Environment columns must have the role \code{design}. For a +set, a named list giving the columns per table. A name that is not a +column of the clone is refused with \code{masque_bad_card_column_refusal}.} + +\item{primary_event}{Which event pair of the first \code{events} spec is the +card's event time and status, counted in the order the events occur. +Ignored when the clone has no \code{events} spec.} +} +\value{ +The card as a list, invisibly when \code{file} is given. For a set, a +named list of cards, one per table. +} +\description{ +Describes a clone in a data card (card version 0.1): its size, its +design columns and, for each column, a role, a kind and counts. An +analysis planner can read the card in place of the data. No value of +any column is written: a factor's levels are counted, never listed. +} +\details{ +The card is computed from the clone alone: \code{\link[=synthetic]{synthetic()}}, the role table +from \code{\link[=roles]{roles()}}, the mode, the package version and the column names of any +\code{events} spec given to \code{\link[=mask]{mask()}}. No other part of the private recipe is +read, so the card can travel with the clone. On a clone whose column +names were aliased, the card uses the aliases throughout. +} +\section{Design block}{ + +The design block is copied from the role table, never re-detected. The +treatment columns are those with the role \code{treatment}, and the block +columns those with the role \code{design}, less any named in \code{env}, +\code{spatial} or \code{unit}. Environment, spatial and unit columns are named by +the caller, since the role table does not tell them apart from blocks. +No design class is written. For a linked-event spec, the event pair +chosen by \code{primary_event} gives the event time and status columns, the +other status columns of that spec are listed as competing, and the card +records the number of rows with the primary status equal to 1 and the +share of rows with neither the primary nor a competing event (censored). +} + +\section{Provenance}{ + +Every statistic carries a provenance class. +\itemize{ +\item \code{structure}: exact in the clone, so equal to the same statistic on the +original table. Row and column counts, missing counts and shares, the +level counts and zero shares of kept and aliased columns and of event +status columns, the treatment, block and event columns, the +environment count and the event count. +\item \code{clone_estimate}: re-simulated, so approximate. The level counts and +zero shares of scrambled columns, including event times, and the +censored share. +\item \code{analyst}: named in this call: the environment, spatial and unit +columns. +} +} + +\section{The YAML file}{ + +The card is written by a small deterministic writer: keys in the order of +the contract, every string double-quoted and escaped, counts as integers, +shares with six decimals, UTF-8 with \verb{\\n} line ends. The same clone gives +the same bytes. +} + +\examples{ +r <- propose_roles(iris) +r <- set_role(r, "Species", role = "treatment", action = "alias") +r <- set_role(r, "Sepal.Length", role = "outcome") +m <- mask(iris, r, seed = 1) +card <- data_card(m) +card$table +card$design + +path <- tempfile(fileext = ".yaml") +data_card(m, file = path) +head(readLines(path), 12) + +} +\seealso{ +\code{\link[=roles]{roles()}}, \code{\link[=synthetic]{synthetic()}}, \code{\link[=mask]{mask()}}, \code{\link[=mask_set]{mask_set()}}. +} diff --git a/man/mask.Rd b/man/mask.Rd index d3fd29a..b5a8963 100644 --- a/man/mask.Rd +++ b/man/mask.Rd @@ -17,6 +17,7 @@ mask( events = NULL, grain = TRUE, allow_unmasked_coords = FALSE, + allow_kept_outcome = FALSE, quiet = FALSE, .shared_maps = list(), .cleaning = NULL, @@ -191,6 +192,13 @@ shape of its values would be. State otherwise by declaring the pair to \code{scramble}), or -- having decided the coordinate is not sensitive -- setting this to \code{TRUE}, which is recorded on the recipe.} +\item{allow_kept_outcome}{Single logical. In collaborate mode an outcome +with the action \code{keep} carries its real values beside the re-simulated +columns, so \code{\link[=audit_mask]{audit_mask()}} grades it HIGH and the collaborate write gate +stops it. Having decided the real outcome may be shared, set this to +\code{TRUE}: the column is then graded medium and the choice is recorded on +the recipe. Local mode is unaffected.} + \item{quiet}{Single logical. \code{TRUE} silences the \code{masque_mode_downgrade} warning raised when a roles table prepared for \code{"collaborate"} is used in \code{"local"} mode, for a plan that is deliberately shared across both diff --git a/man/mask_set.Rd b/man/mask_set.Rd index e976f93..3ca7e20 100644 --- a/man/mask_set.Rd +++ b/man/mask_set.Rd @@ -14,6 +14,8 @@ mask_set( alias_names = FALSE, conditional = FALSE, ladder = c("hierarchy", "levels"), + grain = TRUE, + allow_kept_outcome = FALSE, quiet = FALSE, sheets = NULL, info = NULL, @@ -58,6 +60,12 @@ the treatment-to-outcome relationship survives the clone. See \item{ladder}{Drop order of the conditioning ladder, \code{"hierarchy"} (default) or \code{"levels"}; passed through to each \code{\link[=mask]{mask()}} call.} +\item{grain}{Single logical (default \code{TRUE}); passed through to each +\code{\link[=mask]{mask()}} call, which synthesises a unit-constant column once per unit.} + +\item{allow_kept_outcome}{Single logical, passed to each \code{\link[=mask]{mask()}} call: +\code{TRUE} allows a kept outcome in a collaborate clone and records it.} + \item{quiet}{Logical; suppress the link / hygiene report, including the hint that a table looks like documentation.} @@ -105,16 +113,18 @@ unmasked instead. \examples{ tables <- list( plots = data.frame( - site = c("A", "A", "B", "B"), - gen = c("x", "y", "x", "y"), - yield = c(3.1, 2.9, 4.0, 3.7) + site = rep(c("A", "B", "C", "D", "E"), each = 2), + gen = rep(c("x", "y"), 5), + yield = c(3.1, 2.9, 4.0, 3.7, 3.3, 3.5, 2.8, 3.0, 3.9, 3.6) ), sites = data.frame( - site = c("A", "B"), - rainfall = c(420, 560) + site = c("A", "B", "C", "D", "E"), + rainfall = c(420, 560, 390, 610, 480) ) ) -m <- mask_set(tables, mode = "collaborate", seed = 1, quiet = TRUE) +roles <- lapply(tables, propose_roles, mode = "collaborate") +roles <- lapply(roles, set_role, "site", action = "alias") +m <- mask_set(tables, roles = roles, seed = 1, quiet = TRUE) synthetic(m)$plots$site synthetic(m)$sites$site # same aliases -> the join still works diff --git a/man/masque.Rd b/man/masque.Rd index fbf35f2..ea4cb46 100644 --- a/man/masque.Rd +++ b/man/masque.Rd @@ -87,8 +87,8 @@ See \code{\link[=mask_set]{mask_set()}}.} masked and not written. See \code{\link[=mask_set]{mask_set()}}.} \item{...}{Further arguments for the masking verb: \code{\link[=mask]{mask()}} for one -table (for example \code{coords}, \code{events}, \code{allow_unmasked_coords}), -\code{\link[=mask_set]{mask_set()}} for a set (for example \code{links}). An argument the verb +table (for example \code{coords}, \code{events}, \code{allow_unmasked_coords}, +\code{grain}), \code{\link[=mask_set]{mask_set()}} for a set (for example \code{links}, \code{grain}). An argument the verb does not take is refused (\code{masque_unused_arg_refusal}).} } \value{ diff --git a/man/roles.Rd b/man/roles.Rd new file mode 100644 index 0000000..6536aa0 --- /dev/null +++ b/man/roles.Rd @@ -0,0 +1,41 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/roles_accessor.R +\name{roles} +\alias{roles} +\title{The role table of a masked clone} +\usage{ +roles(m) +} +\arguments{ +\item{m}{A \code{masque} object from \code{\link[=mask]{mask()}}, or a \code{masque_set} from +\code{\link[=mask_set]{mask_set()}}.} +} +\value{ +For a \code{masque}, a data.table with columns \code{col}, \code{role}, +\code{action} and \code{kind}. For a \code{masque_set}, a named list of such tables, +one per table of the set. +} +\description{ +Returns the role each column of the clone was masked under, in the +clone's own vocabulary: one row per column of \code{\link[=synthetic]{synthetic()}}, in the same +order and under the same names. When the column names were aliased +(\code{alias_names} in \code{\link[=mask]{mask()}}), the rows carry the aliases, so the table can +travel with the clone without disclosing the original names. A dropped +column is not in the clone and has no row. +} +\details{ +Only the four public columns are returned. The proposal's notes and its +value summaries describe the original table and stay on the private +recipe, as does the full role table used on the original +(\code{recipe(m)@roles}). +} +\examples{ +r <- propose_roles(iris) +r <- set_role(r, "Sepal.Length", role = "outcome") +m <- mask(iris, r, seed = 1) +roles(m) + +} +\seealso{ +\code{\link[=synthetic]{synthetic()}}, \code{\link[=data_card]{data_card()}}, \code{\link[=mask]{mask()}}. +} From a5522a0f668d4a6e9c0a71960ab995fb94c79c8d Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:33:38 +1030 Subject: [PATCH 48/66] feat(audit): per-column HIGH for a leaky grained or whole-valued column Columns graded on excess over chance (grained, whole-valued covariates and, now, whole-valued outcomes) grade HIGH at 50 points and z 4 over the chance baseline, counted in grain units. A continuous grained column whose unit values are real unit values (copied or moved between units) grades HIGH. --- R/leakage.R | 79 ++++++++++++---- tests/testthat/test-fix0160-audit-grain.R | 6 +- tests/testthat/test-fix0160-grain-high.R | 110 ++++++++++++++++++++++ 3 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 tests/testthat/test-fix0160-grain-high.R diff --git a/R/leakage.R b/R/leakage.R index ea79429..6ddf582 100644 --- a/R/leakage.R +++ b/R/leakage.R @@ -20,9 +20,8 @@ rec@roles$col[rec@roles$role == "design"], names(df) ) grain <- if (S7::prop_exists(rec, "grain")) rec@grain else list() - grained <- unlist(lapply(grain, function(g) { - if (identical(g$synthesised, "unit")) g$cols - }), use.names = FALSE) + unit_grain <- Filter(function(g) identical(g$synthesised, "unit"), grain) + grained <- unlist(lapply(unit_grain, `[[`, "cols"), use.names = FALSE) row_grain <- Filter(function(g) identical(g$synthesised, "row"), grain) allow_kept_outcome <- S7::prop_exists(rec, "allow_kept_outcome") && isTRUE(rec@allow_kept_outcome) @@ -56,7 +55,13 @@ is_grained <- col %in% grained by_excess <- in_synth && kind %in% .numeric_kinds() && .audit_numeric_resimulated(action) && - (is_grained || (role == "covariate" && .is_whole_valued(x))) + (is_grained || + (role %in% c("covariate", "outcome") && .is_whole_valued(x))) + grain_units <- NULL + if (is_grained) { + g <- Filter(function(g) col %in% g$cols, unit_grain)[[1L]] + grain_units <- .unit_id(df, g$unit) + } exact_match_pct <- NA_real_ comparable_n <- NA_integer_ @@ -82,9 +87,17 @@ same <- .same_row_stats( x, synth[[col]], map, groups = groups, - units = .constant_unit(x, df, setdiff(design_cols, col)) + units = if (is_grained) { + grain_units + } else { + .constant_unit(x, df, setdiff(design_cols, col)) + } ) } + value_set <- if (is_grained && is.null(map) && !two_valued && + is.numeric(x) && !.is_whole_valued(x)) { + .unit_value_set(x, synth[[col]], grain_units) + } n_unique_levels <- NA_integer_ freq_min <- NA_integer_ @@ -107,7 +120,9 @@ same_row_excess = if (is.null(same)) NA_real_ else same$excess, same_row_sd = if (is.null(same)) NA_real_ else same$sd, two_valued = two_valued, linked = col %in% linked, - by_excess = by_excess, allow_kept_outcome = allow_kept_outcome + by_excess = by_excess, grained = is_grained, + allow_kept_outcome = allow_kept_outcome, + value_set_pct = if (is.null(value_set)) NA_real_ else value_set$pct ) notes <- .audit_notes( @@ -116,7 +131,8 @@ same = if (col %in% linked) NULL else same, two_valued = two_valued || by_excess, allow_kept_outcome = allow_kept_outcome, - row_grain = Filter(function(g) col %in% g$cols, row_grain) + row_grain = Filter(function(g) col %in% g$cols, row_grain), + value_set = value_set ) data.table::data.table( @@ -145,17 +161,11 @@ audit } -# Thresholds of the excess-over-chance rules, in percentage points of rows -# (cells for the rarity check). A finding needs both the absolute excess and -# `z` standard deviations of the permutation null, so a small table's chance -# swings are not graded. Calibrated on 20 seeds, conditional and not, of -# collaborate clones of agridat::john.alpha, a 6-trial MET and a 200-row -# clinical table: ordinary clones peaked at 9.5 points (z 3.1) per column, 0 -# points jointly and 2.8 cells (z 1.9) for rare crosses; leaky clones (kept -# outcome, kept binary outcome, kept visible crosses) reached 42.7 points -# (z 15.7) jointly and 8.8 cells (z 6.0). +# Percentage points of rows (cells for rare crosses) and z of the chance null. .AUDIT_THRESHOLDS <- list( same_row_medium = list(excess = 10, z = 4), + same_row_high = list(excess = 50, z = 4), + value_set_high = 50, joint_medium = list(excess = 5, z = 4), joint_high = list(excess = 20, z = 4), rare_medium = list(excess = 1, z = 3), @@ -270,6 +280,21 @@ ) } +# Share of a grained column's clone unit values that are real unit values. +.unit_value_set <- function(x, y, units) { + first <- function(v) { + o <- which(!is.na(v)) + v[o[!duplicated(units[o])]] + } + real <- first(x) + clone <- first(y) + if (!length(clone)) { + return(NULL) + } + hits <- sum(clone %in% real) + list(pct = 100 * hits / length(clone), hits = hits, n = length(clone)) +} + # Values comparable by equality: numbers stay numbers, the rest characters. .audit_values <- function(x) { if (is.numeric(x) && !is.factor(x)) { @@ -557,7 +582,8 @@ role, kind, action = NA_character_, pii, mode, in_synth, alias_status, exact_match_pct, freq_min, n_unique_levels, n_rows, coarsened = FALSE, same_row_excess = NA_real_, same_row_sd = NA_real_, two_valued = FALSE, - linked = FALSE, by_excess = FALSE, allow_kept_outcome = FALSE + linked = FALSE, by_excess = FALSE, grained = FALSE, + allow_kept_outcome = FALSE, value_set_pct = NA_real_ ) { collab <- identical(mode, "collaborate") @@ -592,6 +618,16 @@ return("high") } + # 3b. Excess-graded column far above chance, or real unit values: HIGH. + if (collab && in_synth && (isTRUE(by_excess) || isTRUE(grained)) && + !isTRUE(linked) && + !(role %in% c("design", "treatment", "id")) && + (.excess_clears( + same_row_excess, same_row_sd, .AUDIT_THRESHOLDS$same_row_high + ) || isTRUE(value_set_pct >= .AUDIT_THRESHOLDS$value_set_high))) { + return("high") + } + # 4. Outcome with exact-match-pct > 1% in collaborate: MEDIUM. Two-valued # columns match by chance about half the time; rule 5b grades them. if (role == "outcome" && collab && !isTRUE(two_valued) && @@ -629,7 +665,8 @@ .audit_notes <- function(role, kind, action, pii, alias_status, mode, leakage_class, exact_match_pct, freq_min, coarsened = FALSE, same = NULL, two_valued = FALSE, - allow_kept_outcome = FALSE, row_grain = list()) { + allow_kept_outcome = FALSE, row_grain = list(), + value_set = NULL) { collab <- identical(mode, "collaborate") bits <- character() if (pii && !isTRUE(coarsened)) bits <- c(bits, "PII-pattern column name") @@ -685,6 +722,12 @@ "same-row %.1f%% vs chance %.1f%%", same$observed, same$baseline )) } + if (!is.null(value_set) && collab && + value_set$pct >= .AUDIT_THRESHOLDS$value_set_high) { + bits <- c(bits, sprintf( + "%d of %d unit values are real values", value_set$hits, value_set$n + )) + } if (length(bits) == 0L) bits <- "ok" paste(bits, collapse = "; ") } diff --git a/tests/testthat/test-fix0160-audit-grain.R b/tests/testthat/test-fix0160-audit-grain.R index b0ff739..cc20e03 100644 --- a/tests/testthat/test-fix0160-audit-grain.R +++ b/tests/testthat/test-fix0160-audit-grain.R @@ -42,7 +42,7 @@ test_that("ordinary clones of grained and whole-valued columns audit low", { } }) -test_that("a leaky grained or whole-valued column is still flagged", { +test_that("a leaky grained or whole-valued column is HIGH", { df <- .fx_ag() m <- suppressWarnings(mask(df, .fx_ag_roles(df), mode = "collaborate", seed = 1)) @@ -52,7 +52,7 @@ test_that("a leaky grained or whole-valued column is still flagged", { a <- masque:::.compute_audit( data.table::as.data.table(df), leak, recipe(m), "collaborate" ) - expect_identical(a$leakage_class[a$col == cl], "medium", label = cl) + expect_identical(a$leakage_class[a$col == cl], "high", label = cl) } # As the only content column, the leak is a table-level HIGH. @@ -64,7 +64,7 @@ test_that("a leaky grained or whole-valued column is still flagged", { a2 <- masque:::.compute_audit( data.table::as.data.table(df2), leak, recipe(m2), "collaborate" ) - expect_identical(a2$leakage_class[a2$col == "sow_day"], "medium") + expect_identical(a2$leakage_class[a2$col == "sow_day"], "high") expect_true("high" %in% attr(a2, "joint")$leakage_class) }) diff --git a/tests/testthat/test-fix0160-grain-high.R b/tests/testthat/test-fix0160-grain-high.R new file mode 100644 index 0000000..e719acb --- /dev/null +++ b/tests/testthat/test-fix0160-grain-high.R @@ -0,0 +1,110 @@ +# A grained or whole-valued column that leaks grades HIGH on its own row. + +# 8 trials x 3 reps x 10 genotypes; sow_day and rain per trial, count per row. +.fx_gh <- function() { + withr::with_seed(3L, { + trials <- sprintf("T%d", 1:8) + g <- expand.grid( + gen = sprintf("G%02d", 1:10), rep = 1:3, trial = trials, + stringsAsFactors = FALSE, KEEP.OUT.ATTRS = FALSE + ) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.5), 2) + g$count <- stats::rpois(nrow(g), 4) + g$sow_day <- c(120, 124, 120, 131, 124, 118, 131, 120)[ + match(g$trial, trials) + ] + g$rain <- c(312.4, 287.9, 401.2, 355.6, 298.1, 376.5, 330.2, 344.4)[ + match(g$trial, trials) + ] + g + }) +} + +.fx_gh_mask <- function(df, seed = 1L) { + r <- suppressWarnings(propose_roles(df, mode = "collaborate", detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", action = "alias") + r <- set_role(r, c("yield", "count"), role = "outcome", action = "scramble") + r <- set_role(r, c("sow_day", "rain"), role = "covariate", + action = "scramble") + suppressWarnings(mask(df, r, mode = "collaborate", seed = seed)) +} + +.gh_audit <- function(df, clone, m) { + masque:::.compute_audit( + data.table::as.data.table(df), clone, recipe(m), "collaborate" + ) +} + +.gh_class <- function(a, col) a$leakage_class[a$col == col] + +test_that("ordinary clones never grade a grained or count column HIGH", { + df <- .fx_gh() + for (s in 1:10) { + a <- .fx_gh_mask(df, s)@audit + for (cl in c("sow_day", "rain", "count")) { + expect_identical(.gh_class(a, cl), "low", label = paste(cl, s)) + expect_false(is.na(a$excess_pct[a$col == cl])) + } + } +}) + +test_that("a grained column copied from the real units is HIGH", { + df <- .fx_gh() + m <- .fx_gh_mask(df) + for (cl in c("sow_day", "rain")) { + leak <- data.table::copy(synthetic(m)) + data.table::set(leak, j = cl, value = df[[cl]]) + a <- .gh_audit(df, leak, m) + expect_identical(.gh_class(a, cl), "high", label = cl) + } +}) + +test_that("real unit values moved to other units are HIGH", { + df <- .fx_gh() + m <- .fx_gh_mask(df) + trials <- unique(df$trial) + real <- df$rain[match(trials, df$trial)] + moved <- real[c(2:8, 1L)][match(df$trial, trials)] + leak <- data.table::copy(synthetic(m)) + data.table::set(leak, j = "rain", value = moved) + a <- .gh_audit(df, leak, m) + expect_identical(.gh_class(a, "rain"), "high") + expect_match(a$notes[a$col == "rain"], "8 of 8 unit values are real") +}) + +test_that("a whole-valued outcome is graded on excess, and a copy is HIGH", { + df <- .fx_gh() + m <- .fx_gh_mask(df) + expect_false(grepl("exact-match", m@audit$notes[m@audit$col == "count"])) + leak <- data.table::copy(synthetic(m)) + data.table::set(leak, j = "count", value = df$count) + a <- .gh_audit(df, leak, m) + expect_identical(.gh_class(a, "count"), "high") +}) + +test_that("a grained categorical column copied into alias space is HIGH", { + df <- withr::with_seed(4L, { + g <- expand.grid(gen = sprintf("G%d", 1:6), rep = 1:2, + trial = sprintf("T%02d", 1:20), stringsAsFactors = FALSE, + KEEP.OUT.ATTRS = FALSE) + g$yield <- round(stats::rnorm(nrow(g), 3, 0.5), 2) + g$soil <- sample(c("clay", "loam", "sand"), 20, TRUE)[ + match(g$trial, unique(g$trial)) + ] + g + }) + r <- suppressWarnings(propose_roles(df, mode = "collaborate", detect = FALSE)) + r <- set_role(r, c("trial", "rep"), role = "design", action = "keep") + r <- set_role(r, "gen", role = "treatment", action = "alias") + r <- set_role(r, "yield", role = "outcome", action = "scramble") + r <- set_role(r, "soil", role = "covariate", action = "alias") + for (s in 1:5) { + m <- suppressWarnings(mask(df, r, mode = "collaborate", seed = s)) + expect_false(.gh_class(m@audit, "soil") == "high") + } + map <- recipe(m)@level_maps$soil + leak <- data.table::copy(synthetic(m)) + data.table::set(leak, j = "soil", value = unname(map[df$soil])) + expect_identical(.gh_class(.gh_audit(df, leak, m), "soil"), "high") +}) From a88c9f9d364ef4f4a44cf3c11d16d35d04ecbcc9 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:34:54 +1030 Subject: [PATCH 49/66] fix: give the last six unclassed warnings a masque_ class --- R/detect_plot.R | 15 ++++- R/roles_validate.R | 4 +- R/synthesise_geospatial.R | 2 +- tests/testthat/test-fix0160-warning-classes.R | 55 +++++++++++++++++++ 4 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 tests/testthat/test-fix0160-warning-classes.R diff --git a/R/detect_plot.R b/R/detect_plot.R index 7ae2cf0..fd8db3e 100644 --- a/R/detect_plot.R +++ b/R/detect_plot.R @@ -69,7 +69,10 @@ plot_design_summary <- function(x, df, engine = c("base", "ggplot2"), ) } if (engine == "ggplot2" && !requireNamespace("ggplot2", quietly = TRUE)) { - cli::cli_warn("ggplot2 not installed; falling back to base graphics.") + cli::cli_warn( + "ggplot2 not installed; falling back to base graphics.", + class = "masque_plot_fallback" + ) engine <- "base" } @@ -253,7 +256,10 @@ local(S7::method(plot, design_summary) <- function( trt <- x@treatment_col[1L] block <- x@block_cols[1L] if (is.null(trt) || is.na(trt) || length(block) == 0L) { - cli::cli_warn("Replication tile needs treatment + block; skipping.") + cli::cli_warn( + "Replication tile needs treatment + block; skipping.", + class = "masque_plot_skipped" + ) return(invisible()) } # If block is a pairwise basis (e.g. rep + block), build interaction. @@ -331,7 +337,10 @@ local(S7::method(plot, design_summary) <- function( # row x col grid, fill = treatment, outlined by block if known. .plot_spatial <- function(x, df, engine) { if (length(x@spatial_cols) < 2L) { - cli::cli_warn("Spatial plot needs row + col; skipping.") + cli::cli_warn( + "Spatial plot needs row + col; skipping.", + class = "masque_plot_skipped" + ) return(invisible()) } row_nm <- x@spatial_cols[1L] diff --git a/R/roles_validate.R b/R/roles_validate.R index 31917b8..31ae16c 100644 --- a/R/roles_validate.R +++ b/R/roles_validate.R @@ -115,7 +115,7 @@ roles_validate <- function(roles, df = NULL, mode = NULL) { "{.val {mode}}." ), i = "Re-run {.fun propose_roles} to silence this warning." - )) + ), class = "masque_roles_upgraded") roles <- .roles_upgrade(roles, mode) } @@ -245,7 +245,7 @@ roles_validate <- function(roles, df = NULL, mode = NULL) { "Re-run {.code propose_roles(df, mode = \"{mode}\")} if you ", "want the {.val {mode}} defaults." ) - )) + ), class = "masque_roles_mode_mismatch") return(roles) } diff --git a/R/synthesise_geospatial.R b/R/synthesise_geospatial.R index ccec923..636c297 100644 --- a/R/synthesise_geospatial.R +++ b/R/synthesise_geospatial.R @@ -156,7 +156,7 @@ synthesise_geospatial <- function(synth, original, "Coordinates for {sum(synth_anchor == a, na.rm = TRUE)} ", "synthetic row{?s} will be NA." ) - )) + ), class = "masque_geo_anchor_missing") next } n_sites <- length(sites_by_anchor[[a]] %||% integer(0L)) diff --git a/tests/testthat/test-fix0160-warning-classes.R b/tests/testthat/test-fix0160-warning-classes.R new file mode 100644 index 0000000..a7cf2ed --- /dev/null +++ b/tests/testthat/test-fix0160-warning-classes.R @@ -0,0 +1,55 @@ +# Every package warning carries a masque_ class. + +test_that("a roles table without an action column warns by class", { + df <- data.frame(blk = rep(1:2, 5), y = stats::rnorm(10)) + roles <- data.frame( + col = c("blk", "y"), role = c("design", "outcome"), + kind = c("integer", "numeric"), stringsAsFactors = FALSE + ) + expect_warning( + roles_validate(roles, df, mode = "local"), + class = "masque_roles_upgraded" + ) +}) + +test_that("a hand-built roles table used in the other mode warns by class", { + df <- data.frame(blk = rep(1:2, 5), y = stats::rnorm(10)) + r <- propose_roles(df, mode = "local", detect = FALSE) + attr(r, "proposed_actions") <- NULL + expect_warning( + masque:::.reresolve_for_mode(r, "collaborate"), + class = "masque_roles_mode_mismatch" + ) +}) + +test_that("an anchor level without a centroid warns by class", { + df <- data.frame( + state = rep(c("NSW", "VIC"), each = 5), + lat = c(rep(-33, 5), rep(-37, 5)), lon = c(rep(147, 5), rep(144, 5)) + ) + expect_warning( + synthesise_geospatial(df, df, + anchor_col = "state", lat_col = "lat", lon_col = "lon", + anchor_centroids = list(NSW = c(lat = -32.5, lon = 147)), seed = 1L + ), + class = "masque_geo_anchor_missing" + ) +}) + +test_that("a design plot panel that cannot be drawn warns by class", { + df <- data.frame( + trt = rep(c("a", "b"), 6), y = stats::rnorm(12), + stringsAsFactors = FALSE + ) + ds <- suppressWarnings(detect_design(df)) + ds@spatial_cols <- character() + expect_warning( + masque:::.plot_spatial(ds, df, "base"), + class = "masque_plot_skipped" + ) + ds@treatment_col <- NA_character_ + expect_warning( + masque:::.plot_replication_tile(ds, df, "base"), + class = "masque_plot_skipped" + ) +}) From ae75cec3e2073bf39f9c52606b53bf2330e31e58 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:35:25 +1030 Subject: [PATCH 50/66] style: comment blocks cut to one line (prose guard) --- R/synthesise_numeric_jitter.R | 4 +--- tests/testthat/helper-data-card.R | 8 ++------ tests/testthat/test-fix0160-blocks.R | 8 ++------ tests/testthat/test-fix0160-events-clip.R | 4 +--- tests/testthat/test-fix0160-promotion.R | 4 +--- 5 files changed, 7 insertions(+), 21 deletions(-) diff --git a/R/synthesise_numeric_jitter.R b/R/synthesise_numeric_jitter.R index dc6a6cc..4d64820 100644 --- a/R/synthesise_numeric_jitter.R +++ b/R/synthesise_numeric_jitter.R @@ -55,9 +55,7 @@ synthesise_numeric_collaborate <- function(x_obs, x_new, lower = -Inf) { x_new } -# Internal: measurement resolution per column: 1 for whole-valued columns, -# else the smallest gap between sorted unique values, floored at a share of -# the range. +# Internal: resolution, 1 for whole-valued columns, else the smallest gap. .detect_resolution <- function(x) { x_clean <- x[!is.na(x)] if (length(x_clean) == 0L) { diff --git a/tests/testthat/helper-data-card.R b/tests/testthat/helper-data-card.R index 75afc26..a1c6a35 100644 --- a/tests/testthat/helper-data-card.R +++ b/tests/testthat/helper-data-card.R @@ -1,6 +1,4 @@ -# A copy of composeR's data_card.schema.json (contract C8 v0.1, composeR -# commit 5f75c2e) and a small validator for the JSON Schema keywords it uses. -# The schema is parsed with yaml, which reads this JSON as YAML. +# The data card JSON Schema fixture, read with yaml (JSON is valid YAML). .card_schema <- function() { yaml::read_yaml(testthat::test_path("fixtures", "data_card.schema.json")) @@ -58,9 +56,7 @@ errors } -# The rules composeR checks beside the schema (composeR R/data_card.R, -# .card_rule_errors): every statistic has a provenance class and every class -# names a statistic, counts fit the table, design columns are listed. +# Card rules checked beside the schema; returns the broken ones. .card_rule_errors <- function(card) { errors <- character() classed <- function(obj, path, unclassed) { diff --git a/tests/testthat/test-fix0160-blocks.R b/tests/testthat/test-fix0160-blocks.R index 806b9f9..6a54b55 100644 --- a/tests/testthat/test-fix0160-blocks.R +++ b/tests/testthat/test-fix0160-blocks.R @@ -1,6 +1,4 @@ -# F6: after environment scope resolves, block columns come from the -# within-environment block bases, never from an environment column. -# Every expected answer below follows from how the fixture is built. +# Block columns come from within-environment bases, never an environment. # 2 years x 4 sites x 24 genotypes x 3 reps; in each site-year-rep, six blocks # of four, labels B1-B6 restarted in every rep. True blocks: rep : block. @@ -22,9 +20,7 @@ g[, c("year", "site", "rep", "block", "genotype", "yield")] } -# 2 sites, complete blocks B1-B4 restarted in each site, plot P01-P24 the -# position within block. A cyclic allocation with one swap makes plot -# positions look almost like regular incomplete blocks. True blocks: block. +# 2 sites, blocks B1-B4 per site; plot positions mimic incomplete blocks. .f6_plot_artefact <- function() { set.seed(13) g <- expand.grid( diff --git a/tests/testthat/test-fix0160-events-clip.R b/tests/testthat/test-fix0160-events-clip.R index a026a92..bc7a25c 100644 --- a/tests/testthat/test-fix0160-events-clip.R +++ b/tests/testthat/test-fix0160-events-clip.R @@ -118,9 +118,7 @@ test_that("the recipe records how many rows were capped", { }) test_that("the cap moves the final block, or rescales a row it cannot clip", { - # Row 1: plain clip of the last time. Row 2: the last two times are tied by a - # zero gap and move together. Row 3: the second time is already past the - # cap, so the row is scaled. Row 4: under the cap, untouched. + # Rows: plain clip, tied pair moved together, scaled, untouched. gaps <- rbind(c(2, 3, 4), c(2, 5, 0), c(6, 5, 1), c(1, 1, 1)) times <- t(apply(gaps, 1L, cumsum)) cap <- c(7, 6, 8, 10) diff --git a/tests/testthat/test-fix0160-promotion.R b/tests/testthat/test-fix0160-promotion.R index ab5c899..ec82a30 100644 --- a/tests/testthat/test-fix0160-promotion.R +++ b/tests/testthat/test-fix0160-promotion.R @@ -1,6 +1,4 @@ -# F18 and N5: a column becomes a treatment by name, or by structure only when -# the table holds design evidence: a design column the candidate is crossed -# with. Expected roles follow from how each fixture is built. +# A treatment by name, or by structure only beside a crossed design column. # 80 patients, eight ICD-10 codes of ten patients each, no design column. .f18_clinical <- function() { From 8b0688478e6679dfa872bedad6fe17c7e7272a40 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:36:50 +1030 Subject: [PATCH 51/66] fix(mask_set): link maps draw from their own sub-stream of seed --- R/jitter_coordinates.R | 4 +- R/mask_set.R | 4 +- R/rng.R | 15 +++++++ tests/testthat/test-fix0160-link-seed.R | 53 +++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 tests/testthat/test-fix0160-link-seed.R diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index 0251dcb..98d5d5d 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -601,9 +601,7 @@ jitter_coordinates <- function(df, lat_col, lon_col, if (!n) { return(list(df = synth, reports = list())) } - sub_seeds <- with_rng_state( - seed, sample.int(.Machine$integer.max, n) - ) + sub_seeds <- .sub_seeds(seed, n) reports <- vector("list", n) for (i in seq_len(n)) { s <- specs[[i]] diff --git a/R/mask_set.R b/R/mask_set.R index f0b895e..cf17047 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -198,7 +198,9 @@ mask_set <- function(input, built <- if (is.null(seed)) { .build_shared_maps(tables, roles, link_groups, mode) } else { - with_rng_state(seed, .build_shared_maps(tables, roles, link_groups, mode)) + with_rng_state( + .link_seed(seed), .build_shared_maps(tables, roles, link_groups, mode) + ) } link_groups <- built$groups shared_by_table <- built$shared_by_table diff --git a/R/rng.R b/R/rng.R index d29840c..2ab8bc7 100644 --- a/R/rng.R +++ b/R/rng.R @@ -46,3 +46,18 @@ with_rng_state <- function(seed, expr) { class = c("masque_bad_seed_refusal", "orchestra_refusal") ) } + +# `n` seeds drawn from the stream of `seed`, one per independent consumer. +.sub_seeds <- function(seed, n) { + with_rng_state(seed, sample.int(.Machine$integer.max, n)) +} + +# Offset that roots the link-map stream away from the tables' own stream. +.LINK_SALT <- 104729L + +# Seed of mask_set()'s link maps, distinct from the seed each table uses. +.link_seed <- function(seed) { + .check_seed(seed) + root <- (as.numeric(seed) + .LINK_SALT) %% .Machine$integer.max + .sub_seeds(root, 1L) +} diff --git a/tests/testthat/test-fix0160-link-seed.R b/tests/testthat/test-fix0160-link-seed.R new file mode 100644 index 0000000..7633513 --- /dev/null +++ b/tests/testthat/test-fix0160-link-seed.R @@ -0,0 +1,53 @@ +# mask_set() link maps draw from their own sub-stream of `seed`. + +# 12 sites linked between a plot table and a site table. +.ls_tables <- function() { + sites <- sprintf("Site%02d", 1:12) + list( + plots = data.frame( + site = rep(sites, each = 4L), rep = rep(1:4, 12L), + yield = round(seq(3, 6, length.out = 48L), 2), + stringsAsFactors = FALSE + ), + meta = data.frame( + site = sites, elev = seq(100, 650, by = 50), stringsAsFactors = FALSE + ) + ) +} + +.ls_map <- function(seed) { + tables <- .ls_tables() + rp <- propose_roles(tables$plots, mode = "collaborate", detect = FALSE) + rp <- set_role(rp, "site", role = "design", action = "alias") + rp <- set_role(rp, "rep", role = "design", action = "keep") + rp <- set_role(rp, "yield", role = "outcome", action = "scramble") + rm <- propose_roles(tables$meta, mode = "collaborate", detect = FALSE) + rm <- set_role(rm, "site", role = "design", action = "alias") + rm <- set_role(rm, "elev", role = "covariate", action = "scramble") + ms <- suppressWarnings(mask_set( + tables, roles = list(plots = rp, meta = rm), mode = "collaborate", + seed = seed, quiet = TRUE + )) + recipe(ms)@links[[1L]]$map +} + +test_that("link maps repeat exactly for a seed", { + expect_identical(.ls_map(11L), .ls_map(11L)) +}) + +test_that("link maps are not drawn from the tables' own stream", { + for (s in 1:5) { + map <- .ls_map(s) + order <- as.integer(sub(".*_K", "", unname(map))) + table_stream <- withr::with_seed(s, sample.int(12L)) + expect_false(identical(order, table_stream), label = paste("seed", s)) + } +}) + +test_that("the link sub-stream differs from the table and coords streams", { + for (s in c(-5L, 0L, 1L, 7L, .Machine$integer.max)) { + link <- masque:::.link_seed(s) + expect_true(link != s) + expect_false(link %in% masque:::.sub_seeds(s, 10L)) + } +}) From bf8603ab06f4ef8e296d56237d5ee3a9d8afa37e Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:37:21 +1030 Subject: [PATCH 52/66] docs: 0.16.0 NEWS, version, vignettes, README, API stability and resubmission notes NEWS 0.16.0 replaces the unreleased 0.15.1 section. Vignettes gain grain, time-to-event, numeric treatment, kept outcome and small-table sections, alias year before masking the met_set, count table-level audit findings, and rename the plan variable that shadowed roles(). Roxygen documents the alias label schemes and what reveal_maps() prints. --- API_STABILITY.md | 17 +++ CITATION.cff | 2 +- DESCRIPTION | 2 +- NEWS.md | 110 ++++++++++++-- R/alias_levels.R | 2 +- R/mask.R | 9 +- R/mask_set.R | 10 +- R/masque-package.R | 7 +- R/masque.R | 3 +- R/recipe_print.R | 7 + README.md | 14 +- cran-comments.md | 36 +++-- inst/WORDLIST | 2 + man/mask.Rd | 9 +- man/mask_set.Rd | 10 +- man/masque-package.Rd | 7 +- man/masque.Rd | 3 +- man/reveal_maps.Rd | 8 + vignettes/confidentiality.Rmd | 274 ++++++++++++++++++++++++++++++---- vignettes/getting_started.Rmd | 70 ++++++--- vignettes/recipe_anatomy.Rmd | 56 +++++-- 21 files changed, 538 insertions(+), 120 deletions(-) diff --git a/API_STABILITY.md b/API_STABILITY.md index f4ff3a8..eafa88b 100644 --- a/API_STABILITY.md +++ b/API_STABILITY.md @@ -77,6 +77,23 @@ existing signatures. The track record so far: `data.table` instead of a tibble, and tibble leaves the dependencies. Signatures and values are unchanged. **Breaking change for code that relies on tibble behaviour, listed first in `NEWS.md`.** +- `0.16.0`: new exports `data_card()` and `roles()`. New arguments, each + with a default: + - `mask()`: `events`, `grain`, `allow_kept_outcome` + - `mask_set()`: `grain`, `allow_kept_outcome`, `sheets`, `info`, `...` + - `masque()`: `detect`, `sheets`, `info`, `...` + - `reveal_maps()`: `table` + + In `mask()` the new arguments sit before `allow_unmasked_coords` and + `quiet`, and in `mask_set()` before `quiet`, so a call that passes those + by position must name them. + + Default clones change for tables with a column constant within a design + unit (`grain = FALSE` restores 0.15.0), for collaborate jitter (no longer + clipped to the observed range), for numeric treatments (aliased in + collaborate mode) and for `mask_set()` link aliases made with a seed. A + kept outcome audits HIGH in collaborate mode. Recipes saved by 0.15.0 + still load and apply. **Behaviour changes, listed first in `NEWS.md`.** Additive by intent is not frozen: before 1.0 a release may break an existing signature when a design flaw surfaces, but every such change must diff --git a/CITATION.cff b/CITATION.cff index 38db2f6..4c222eb 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,7 +20,7 @@ authors: repository-code: "https://github.com/max578/masque" url: "https://max578.github.io/masque/" license: MIT -version: 0.15.1 +version: 0.16.0 keywords: - synthetic-data - data-masking diff --git a/DESCRIPTION b/DESCRIPTION index 1e8c5a7..67a301c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: masque Title: Structurally Faithful Development Surrogates for Tabular Data -Version: 0.15.1 +Version: 0.16.0 Authors@R: c( person("Max", "Moldovan", email = "max.moldovan@gmail.com", diff --git a/NEWS.md b/NEWS.md index baa4b49..77d60e3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,18 +1,98 @@ -# masque 0.15.1 - -* `mask()` gains `events =` for linked time-to-event outcomes: several - event times on one row, in the order the events occur, each with a - status. The times are re-simulated as gaps - within each status pattern, so they stay ordered, a row without the - first event keeps both times equal, and the count of each status pattern - is exact. Status patterns move across rows, within treatment when - `conditional = TRUE`. -* The interactive review in `masque()` now writes the plan table and the - edit menus as messages, so `suppressMessages()` silences them. -* The vignettes restore the `width` option they set. -* In collaborate mode, a double column whose values are all whole numbers - now stays whole-numbered, as an integer column already did. A column - with only two observed values is no longer jittered and keeps both. +# masque 0.16.0 + +## Behaviour changes + +* `mask()` synthesises a column that is constant within a design unit (a + trial-level rainfall, a site soil type) once per unit, so the clone keeps + one value per trial. This changes the default clone of such tables. + `grain = FALSE` gives the row-by-row clone of 0.15.0. In collaborate mode + a design column with fewer than five units falls back to row-by-row + synthesis with a `masque_grain_small_units` warning. +* Collaborate mode no longer clips jittered numbers to the observed range, + which copied the real minimum and maximum into the clone. A value can + now sit up to half a measurement step outside the range. +* A numeric treatment (a dose, an N rate) can be aliased or permuted, and + collaborate mode aliases it by default (`trt_NNN`). `unmask()` and + `apply_recipe()` restore the exact numeric values. +* In collaborate mode an outcome left at `keep` audits HIGH and blocks the + package writers. `allow_kept_outcome = TRUE` allows it after review and + is recorded on the recipe. +* A collaborate clone of fewer than five rows that re-simulates any column + is refused (`masque_too_few_rows_refusal`). Local mode warns. +* `propose_roles()` promotes a column to treatment only when it is crossed + with the design, so a clinical `diagnosis` or a trial-level covariate + stays a covariate. `yield` and similar names are proposed as outcomes, + and `Site Name` or `trial_site` as design. +* In `mask()` and `mask_set()` the new arguments come before `quiet` (and + before `allow_unmasked_coords` in `mask()`), so pass those by name. +* masque needs data.table 1.15.0 or later. Returned tables are data.tables, + as before. +* `mask_set()` link aliases now follow `seed` whatever the caller's random + state, so a set made with a seed in 0.15.0 gets different link aliases + when it is made again. + +## New features + +* `data_card()` writes a data card for a clone, or one per table of a set: + size, design columns and per-column counts, each marked as exact in the + clone or re-simulated. It never lists a level label. +* `roles()` returns the role table of a clone under the clone's column + names. +* `mask()` gains `grain`, `events` (linked time-to-event outcomes kept in + order, with exact counts of each status pattern and no time past the + longest observed follow-up) and + `allow_kept_outcome`. `mask_set()` gains `grain`, `allow_kept_outcome`, + `sheets` and `info` (documentation sheets set aside, not masked). +* `masque()` takes `detect`, `sheets` and `info`, and passes other + arguments to `mask()` or `mask_set()`. An argument the verb does not take + is refused. +* `audit_mask()` grades each column on its excess over a chance baseline + and adds table-level checks (whole-row matches, rare combinations) in + `attr(audit, "joint")`. A column synthesised per unit, or a whole-numbered + covariate such as age, is graded on that excess instead of on raw exact + matches, which it reaches by construction. +* `audit_mask()` grades such a column HIGH when it matches the original + row by row far above chance, as when the real values were copied back. +* `save_recipe()`, `read_recipe()`, `reveal_maps(table = )` and + `audit_mask()` work on a `mask_set()` result. +* The recipe records `conform_table()` decisions, and `apply_recipe()` + replays them on the original. + +## Bug fixes + +* `conform_table(merge_labels = "auto")` merged distinct genotype codes + (`G002` and `G011` into `G001`). Codes that differ only in a run of + digits are never merged now. A label whose merges end on two different + labels is left alone, with a warning. +* `conform_table()` ran a label containing braces as R code when printing + its report. Labels now print as text. +* `masque()` given a roles table made for collaborate mode masked in local + mode and returned real labels. It now uses the plan's mode. +* `print()`, `format()`, `str()` and knitr output of a recipe, a recipe + bundle, a `masque` or a `masque_set` no longer show the seed or the label + maps. +* The interactive review in `masque()` writes the plan and its menus as + messages, so `suppressMessages()` silences them. +* Guided `masque()` on a folder or workbook with column names that needed + repair stopped with an error. Set recipes now keep each table's cleaning, + and `apply_recipe()` keeps a linked key that the roles mark `drop`. +* A coordinate declared in `coords` whose action you set to `drop` or + `scramble` is refused (`masque_coords_action_refusal`). Before, the + declaration silently kept it. A single pair may be passed bare, and its + names must be `lat` and `lon` exactly. +* Whole-numbered columns stay whole in local conditional clones and in + collaborate mode, and a two-valued column is no longer jittered. +* Duplicate input column names are refused once, by name. The + near-duplicate report prints at most 20 pairs. +* `save_recipe()` accepts only a `.rds` path. A seed must be one whole + number, in `jitter_coordinates()` and `synthesise_geospatial()` too. +* A recipe saved by masque 0.15.0 still prints, applies, unmasks and + audits. +* `apply_recipe()` maps a numeric column that has an alias map instead of + passing real values through. +* The guided summary counts columns as the audit does, and the + non-interactive notice no longer suggests an argument already passed. +* The vignettes restore the `options()` they set. # masque 0.15.0 diff --git a/R/alias_levels.R b/R/alias_levels.R index 36ef8da..04268d6 100644 --- a/R/alias_levels.R +++ b/R/alias_levels.R @@ -1,7 +1,7 @@ #' Collaborate-mode opaque aliasing of categorical levels #' #' Replaces the level vocabulary with opaque aliases of the form -#' `_001`, `_002`, ..., assigned to the levels by a +#' `001`, `002`, ... (`trt_001`, `site_D001`), assigned to the levels by a #' **random permutation** drawn from the RNG stream `mask()` has already #' seeded. The alias vocabulary itself is fixed and sorted; which level #' receives which alias is not. diff --git a/R/mask.R b/R/mask.R index 57c09d7..993f5db 100644 --- a/R/mask.R +++ b/R/mask.R @@ -35,10 +35,12 @@ #' the assignment structure never moves.} #' \item{`alias`}{As `scramble` where applicable, plus opaque label #' substitution: treatments become `trt_NNN` (`_trt_NNN` when -#' two or more treatment factors are aliased), categorical +#' two or more treatments are aliased. A numeric treatment becomes +#' text, and [unmask()] restores its exact values), categorical #' covariates `_LNNN`, design labels `_DNNN` (in place - #' structure intact), ids `_INNN` (in place - row linkage -#' intact), text values `_TNNN`.} +#' intact), text values `_TNNN`. A key [mask_set()] shares across +#' tables becomes `_KNNN`. `NNN` has at least three digits.} #' \item{`drop`}{Column excluded from the synthetic, both modes.} #' } #' @@ -231,7 +233,8 @@ #' head(synthetic(m)) #' #' @seealso [propose_roles()], [set_role()], [roles_validate()], -#' [synthetic()], [recipe()], [reveal_maps()]. +#' [synthetic()], [recipe()], [roles()], [reveal_maps()], [audit_mask()], +#' [data_card()]. #' #' @export mask <- function(df, diff --git a/R/mask_set.R b/R/mask_set.R index f0b895e..2820475 100644 --- a/R/mask_set.R +++ b/R/mask_set.R @@ -15,9 +15,10 @@ #' `mask_set()` proposes links automatically and prints them; pass #' `links` to override. Linked columns are aliased in place with a shared #' map (built over the union of values across all tables), so identical -#' original values map to identical aliases in every table. Set a linked -#' column's action to `keep` in its roles table to pass it through -#' unmasked instead. +#' original values map to identical aliases in every table. The aliases +#' read `_K001`, `_K002`, ..., assigned at random and, when `seed` +#' is set, the same for the same seed. Set a linked column's action to +#' `keep` in its roles table to pass it through unmasked instead. #' #' @param input A folder path, an `.xlsx` / `.xls` path, or a named list #' of data frames - anything [read_set()] accepts. A list element may also @@ -87,7 +88,8 @@ #' synthetic(m)$plots$site #' synthetic(m)$sites$site # same aliases -> the join still works #' -#' @seealso [mask()], [read_set()], [write_set()]. +#' @seealso [mask()], [read_set()], [write_set()], [audit_mask()], +#' [data_card()], [reveal_maps()]. #' @export mask_set <- function(input, roles = NULL, diff --git a/R/masque-package.R b/R/masque-package.R index f0e0664..6fe7f13 100644 --- a/R/masque-package.R +++ b/R/masque-package.R @@ -18,7 +18,9 @@ #' workbook, or named list flows through [mask_set()], which aliases #' shared keys identically across tables so the synthetic set still #' joins. [apply_recipe()] and [unmask()] re-target a finished pipeline -#' onto the original data. +#' onto the original data. [audit_mask()] grades what the clone discloses, +#' and [data_card()] describes the clone for a collaborator without +#' listing any value. #' #' @section Scope and limits: #' `masque` is **not** a differential-privacy or anonymisation tool. Its outputs @@ -32,7 +34,8 @@ #' level vocabularies; may emit observed values; not for external sharing. #' - `mode = "collaborate"`: opaque aliasing for treatment and #' categorical-covariate levels; within-resolution jitter on numerics; -#' integers stochastically rounded; `audit_mask()` auto-runs at construction. +#' whole numbers stochastically rounded; `audit_mask()` auto-runs at +#' construction. #' #' @keywords internal "_PACKAGE" diff --git a/R/masque.R b/R/masque.R index 39d373c..9a28f03 100644 --- a/R/masque.R +++ b/R/masque.R @@ -102,7 +102,8 @@ #' #' @seealso [mask()], [mask_set()], [propose_roles()], [set_role()], #' [apply_recipe()] and [unmask()] for the round trip, [save_recipe()] to -#' keep the recipe, [write_set()]. +#' keep the recipe, [audit_mask()], [data_card()] and [roles()] to describe +#' the clone, [write_set()]. #' @export masque <- function(input, roles = NULL, diff --git a/R/recipe_print.R b/R/recipe_print.R index ccd5529..0fda768 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -249,6 +249,13 @@ local({ #' methods of recipes, recipe bundles, `masque` and `masque_set` objects. #' `saveRDS()` and `dput()` write the maps by design. #' +#' For each recipe it prints the column-name map (when names were +#' aliased), every level map as `original -> alias`, and the seed. The +#' alias labels follow the scheme described under `alias` in [mask()]: +#' `trt_NNN` or `_trt_NNN` for treatments, `_LNNN`, `_DNNN`, +#' `_INNN` and `_TNNN` for covariates, design columns, ids and +#' text, and `_KNNN` for a key shared across the tables of a set. +#' #' @param rec A `masque_recipe` object (e.g., from `recipe(m)`), or the #' `masque_recipe_set` bundle of a [mask_set()] result. #' @param table For a bundle, the names of the tables whose maps to reveal. diff --git a/README.md b/README.md index df64ddf..00aa6f6 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,10 @@ fit <- lm(yield ~ gen + rep, data = synth) # ... and the custodian re-targets it to the original data. preds <- predict(fit, newdata = apply_recipe(df, rec)) + +# A description of the clone the analyst can plan with: sizes, roles and +# counts, never a value. +card <- data_card(m, name = "alpha_trial") ``` A folder of files or a multi-sheet workbook works the same way. Pass the @@ -106,8 +110,14 @@ before using it. - Preserves enough structure for pipelines to run unchanged. - Provides two explicit modes: `local` for owner-only realistic surrogates, - and `collaborate` for controlled sharing with opaque aliasing, numeric - jitter, and an automatic leakage audit. + and `collaborate` for controlled sharing with opaque aliasing (numeric + doses included), numeric jitter, and an automatic leakage audit. + Collaborate mode drops identifiers and free text (character columns in + which most values are distinct), and refuses to re-simulate a table of + fewer than five rows. +- Synthesises a column that is constant within a trial or site once per + unit, so the clone keeps one value per trial, and keeps linked event + times in order (`events =`). - Keeps treatment, block and environment means on request (`conditional = TRUE`). Treatment means on the clone then track those of the real data, and block and environment effects stay detectable. diff --git a/cran-comments.md b/cran-comments.md index 41e3bae..e27b016 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -1,23 +1,31 @@ ## Resubmission -This is a resubmission of a new package. In response to the review of -0.15.0: - -* The text that `R/masque.R` wrote to the console with `print()` and - `cat()` during the interactive review now goes through `message()` - (via cli), so it can be suppressed. The output that remains on the - console comes from `print()` and `summary()` methods. -* Each vignette saves the user's `options()` before changing `width` and - restores them in its last chunk. +This is a resubmission of a new package, now version 0.16.0. The review of +0.15.0 asked for two changes: + +* **Console output that cannot be suppressed.** The interactive review in + `R/masque.R` wrote the plan table and its menus with `print()` and + `cat()`. It now captures that text and writes it with + `cli::cli_verbatim()`, which signals a message, so `suppressMessages()` + silences it. The `print()` and `cat()` calls left in + the package are inside `print()`, `format()`, `str()` and `knit_print()` + methods, and in `reveal_maps()`, whose only purpose is to print the + label maps a user asks to see. +* **Vignettes that change `options()`.** Each of the four vignettes saves + the user's options with `old_options <- options(width = 100)` in its + setup chunk and restores them with `options(old_options)` in its last + chunk. Rendering every vignette leaves `options()` unchanged. + +Since 0.15.0 the package has also gained `data_card()` and `roles()`, +synthesis of unit-constant columns once per design unit, linked +time-to-event outcomes, and fixes to the leakage audit and the +multi-table round trip. `NEWS.md` lists them. ## Test environments * Local: macOS (Apple Silicon), R 4.6.1. -* GitHub Actions: Ubuntu R-devel, R-release and R-oldrel-1; macOS - R-release; Windows R-release. -* R-hub: Linux R-devel, Ubuntu R-patched, Windows R-devel, macOS arm64 - R-devel. -* win-builder: Windows R-devel. +* [TO COMPLETE AFTER CI: GitHub Actions, R-hub and win-builder runs for + 0.16.0, with the R versions and platforms.] ## R CMD check results diff --git a/inst/WORDLIST b/inst/WORDLIST index 6678b36..5ffa407 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -55,9 +55,11 @@ handoff interpretability jitter's joinable +knitr maintainer's masqued matchable +multiset oldrel orchestrator passthrough diff --git a/man/mask.Rd b/man/mask.Rd index b5a8963..3f683b9 100644 --- a/man/mask.Rd +++ b/man/mask.Rd @@ -258,10 +258,12 @@ preserved. Treatment columns get a seeded label permutation - the assignment structure never moves.} \item{\code{alias}}{As \code{scramble} where applicable, plus opaque label substitution: treatments become \code{trt_NNN} (\verb{_trt_NNN} when -two or more treatment factors are aliased), categorical +two or more treatments are aliased. A numeric treatment becomes +text, and \code{\link[=unmask]{unmask()}} restores its exact values), categorical covariates \verb{_LNNN}, design labels \verb{_DNNN} (in place - structure intact), ids \verb{_INNN} (in place - row linkage -intact), text values \verb{_TNNN}.} +intact), text values \verb{_TNNN}. A key \code{\link[=mask_set]{mask_set()}} shares across +tables becomes \verb{_KNNN}. \code{NNN} has at least three digits.} \item{\code{drop}}{Column excluded from the synthetic, both modes.} } @@ -278,5 +280,6 @@ head(synthetic(m)) } \seealso{ \code{\link[=propose_roles]{propose_roles()}}, \code{\link[=set_role]{set_role()}}, \code{\link[=roles_validate]{roles_validate()}}, -\code{\link[=synthetic]{synthetic()}}, \code{\link[=recipe]{recipe()}}, \code{\link[=reveal_maps]{reveal_maps()}}. +\code{\link[=synthetic]{synthetic()}}, \code{\link[=recipe]{recipe()}}, \code{\link[=roles]{roles()}}, \code{\link[=reveal_maps]{reveal_maps()}}, \code{\link[=audit_mask]{audit_mask()}}, +\code{\link[=data_card]{data_card()}}. } diff --git a/man/mask_set.Rd b/man/mask_set.Rd index 3ca7e20..9e0270e 100644 --- a/man/mask_set.Rd +++ b/man/mask_set.Rd @@ -105,9 +105,10 @@ compatible kind and overlapping values - the join keys of the set. \code{mask_set()} proposes links automatically and prints them; pass \code{links} to override. Linked columns are aliased in place with a shared map (built over the union of values across all tables), so identical -original values map to identical aliases in every table. Set a linked -column's action to \code{keep} in its roles table to pass it through -unmasked instead. +original values map to identical aliases in every table. The aliases +read \verb{_K001}, \verb{_K002}, ..., assigned at random and, when \code{seed} +is set, the same for the same seed. Set a linked column's action to +\code{keep} in its roles table to pass it through unmasked instead. } \examples{ @@ -130,5 +131,6 @@ synthetic(m)$sites$site # same aliases -> the join still works } \seealso{ -\code{\link[=mask]{mask()}}, \code{\link[=read_set]{read_set()}}, \code{\link[=write_set]{write_set()}}. +\code{\link[=mask]{mask()}}, \code{\link[=read_set]{read_set()}}, \code{\link[=write_set]{write_set()}}, \code{\link[=audit_mask]{audit_mask()}}, +\code{\link[=data_card]{data_card()}}, \code{\link[=reveal_maps]{reveal_maps()}}. } diff --git a/man/masque-package.Rd b/man/masque-package.Rd index 0315af1..778c13a 100644 --- a/man/masque-package.Rd +++ b/man/masque-package.Rd @@ -24,7 +24,9 @@ writes the result. A single table flows through \code{\link[=mask]{mask()}}; a f workbook, or named list flows through \code{\link[=mask_set]{mask_set()}}, which aliases shared keys identically across tables so the synthetic set still joins. \code{\link[=apply_recipe]{apply_recipe()}} and \code{\link[=unmask]{unmask()}} re-target a finished pipeline -onto the original data. +onto the original data. \code{\link[=audit_mask]{audit_mask()}} grades what the clone discloses, +and \code{\link[=data_card]{data_card()}} describes the clone for a collaborator without +listing any value. } \section{Scope and limits}{ @@ -43,7 +45,8 @@ the full threat model and limitations. level vocabularies; may emit observed values; not for external sharing. \item \code{mode = "collaborate"}: opaque aliasing for treatment and categorical-covariate levels; within-resolution jitter on numerics; -integers stochastically rounded; \code{audit_mask()} auto-runs at construction. +whole numbers stochastically rounded; \code{audit_mask()} auto-runs at +construction. } } diff --git a/man/masque.Rd b/man/masque.Rd index ea4cb46..95a64c1 100644 --- a/man/masque.Rd +++ b/man/masque.Rd @@ -151,5 +151,6 @@ head(synthetic(m)) \seealso{ \code{\link[=mask]{mask()}}, \code{\link[=mask_set]{mask_set()}}, \code{\link[=propose_roles]{propose_roles()}}, \code{\link[=set_role]{set_role()}}, \code{\link[=apply_recipe]{apply_recipe()}} and \code{\link[=unmask]{unmask()}} for the round trip, \code{\link[=save_recipe]{save_recipe()}} to -keep the recipe, \code{\link[=write_set]{write_set()}}. +keep the recipe, \code{\link[=audit_mask]{audit_mask()}}, \code{\link[=data_card]{data_card()}} and \code{\link[=roles]{roles()}} to describe +the clone, \code{\link[=write_set]{write_set()}}. } diff --git a/man/reveal_maps.Rd b/man/reveal_maps.Rd index 5d6b973..ab94d78 100644 --- a/man/reveal_maps.Rd +++ b/man/reveal_maps.Rd @@ -25,6 +25,14 @@ built into the \code{print()}, \code{format()}, \code{str()} and \code{knitr::kn methods of recipes, recipe bundles, \code{masque} and \code{masque_set} objects. \code{saveRDS()} and \code{dput()} write the maps by design. } +\details{ +For each recipe it prints the column-name map (when names were +aliased), every level map as \code{original -> alias}, and the seed. The +alias labels follow the scheme described under \code{alias} in \code{\link[=mask]{mask()}}: +\code{trt_NNN} or \verb{_trt_NNN} for treatments, \verb{_LNNN}, \verb{_DNNN}, +\verb{_INNN} and \verb{_TNNN} for covariates, design columns, ids and +text, and \verb{_KNNN} for a key shared across the tables of a set. +} \examples{ r <- propose_roles(iris) r$role[r$col == "Sepal.Length"] <- "outcome" diff --git a/vignettes/confidentiality.Rmd b/vignettes/confidentiality.Rmd index 19a0afb..7056980 100644 --- a/vignettes/confidentiality.Rmd +++ b/vignettes/confidentiality.Rmd @@ -67,12 +67,13 @@ per-column work: | Knob | `mode = "local"` | `mode = "collaborate"` | |---|---|---| -| Treatment labels | kept | aliased (`trt_001`) | +| Treatment labels, numeric doses included | kept | aliased (`trt_001`) | | Categorical covariate labels | kept | aliased (`_L001`) | | Date / time columns | row-permuted with class preserved | row-permuted with class preserved | | Identifiers (`id`) | kept | dropped | -| Free text (`text`) | kept | dropped | -| Numeric synthesis | Gaussian copula with empirical-quantile marginals (may emit observed values) | the same, plus within-resolution jitter, with integers stochastically rounded | +| Free text (`text`: a character column in which more than half the values are distinct) | kept | dropped | +| Numeric synthesis | Gaussian copula with empirical-quantile marginals (may emit observed values) | the same, plus within-resolution jitter, with whole numbers stochastically rounded. A value can fall up to half a measurement step outside the observed range | +| Columns constant within a design unit | one value per unit, drawn from the real unit values | one value per unit, jittered, or row by row when there are fewer than five units | | NA mask | preserved cell-by-cell | preserved cell-by-cell | | `audit_mask()` | on demand | automatic at `mask()` time | | `print(recipe)` | redacted | redacted, with explicit `reveal_maps()` to inspect | @@ -92,9 +93,9 @@ df <- data.frame( rep = rep(seq_len(3L), 6L), yield = rnorm(18L) ) -roles <- propose_roles(df, detect = FALSE) -roles <- set_role(roles, "site", role = "design", action = "alias") -s <- synthetic(mask(df, roles, mode = "collaborate", seed = 1L)) +plan <- propose_roles(df, detect = FALSE) +plan <- set_role(plan, "site", role = "design", action = "alias") +s <- synthetic(mask(df, plan, mode = "collaborate", seed = 1L)) site_names_gone <- !any(as.character(s$site) %in% as.character(df$site)) site_names_gone knitr::kable( @@ -143,6 +144,35 @@ An alias therefore raises the cost of re-identification. It cannot be relied on against a determined attacker, and none of it is a differential-privacy guarantee. +### Numeric treatments + +A dose or a nitrogen rate is a numeric treatment. Its values are often the +most recognisable part of a trial, so collaborate mode aliases them by +default, as it does treatment labels. The clone carries `trt_001`, +`trt_002`, ... as text, and the recipe turns them back into the exact +numbers: + +```{r numeric-treatment} +rates <- data.frame( + block = factor(rep(seq_len(4L), each = 5L)), + nitro = rep(c(0, 0.2, 0.4, 0.6, 0.8), 4L) +) +rates$yield <- round(80 + 40 * rates$nitro + rnorm(20L, sd = 5), 1) +plan_rates <- propose_roles(rates, mode = "collaborate") +m_rates <- mask(rates, plan_rates, mode = "collaborate", seed = 1L) +knitr::kable( + head(synthetic(m_rates), 5L), + digits = 1, + caption = "The first block of the clone: each N rate is an opaque label." +) +identical(unmask(synthetic(m_rates), recipe(m_rates))$nitro, rates$nitro) +``` + +Every block still holds each rate once, so the design is intact. With +`action = "scramble"` the rates stay numbers and are permuted among +themselves instead. A numeric treatment left at `keep` in collaborate mode +audits HIGH. + ### The conditional clone: preserving the treatment effect The default numeric synthesis re-simulates every scrambled numeric column @@ -172,9 +202,9 @@ arm <- factor(rep(c("ctrl", "treat"), each = n / 2L)) yield <- 10 + 5 * (arm == "treat") + rnorm(n, sd = 2) trial <- data.frame(genotype = arm, yield = yield) -roles <- propose_roles(trial, detect = FALSE) -roles <- set_role(roles, "genotype", role = "treatment", action = "scramble") -roles <- set_role(roles, "yield", role = "outcome") +plan <- propose_roles(trial, detect = FALSE) +plan <- set_role(plan, "genotype", role = "treatment", action = "scramble") +plan <- set_role(plan, "yield", role = "outcome") true_effect <- coef(lm(yield ~ genotype, trial))[["genotypetreat"]] true_effect @@ -184,8 +214,8 @@ Cloning both ways from the same seed, then re-estimating the effect on each clone: ```{r conditional-contrast} -marg <- mask(trial, roles, mode = "local", seed = 1L, conditional = FALSE) -cond <- mask(trial, roles, mode = "local", seed = 1L, conditional = TRUE) +marg <- mask(trial, plan, mode = "local", seed = 1L, conditional = FALSE) +cond <- mask(trial, plan, mode = "local", seed = 1L, conditional = TRUE) effect_of <- function(m) { coef(lm(yield ~ genotype, synthetic(m)))[["genotypetreat"]] @@ -299,7 +329,7 @@ it asked for: ```r withCallingHandlers( - mask(trial, roles, seed = 1L, conditional = TRUE), + mask(trial, plan, seed = 1L, conditional = TRUE), masque_conditional_degraded = function(w) stop(conditionMessage(w)) ) ``` @@ -360,6 +390,115 @@ original. A good fit there is not evidence that the step behaves correctly on the real data. Validate such a step by running it on the original through the recipe. +### Columns constant within a trial: the grain + +Some columns describe a whole unit rather than a row: the rainfall of a +trial, the soil type of a site, a sowing date. Re-simulated row by row, +such a column would vary inside the trial, which no real trial does. +`mask()` finds the design unit within which a column holds a single value +and synthesises the column once per unit, writing each unit's new value to +all its rows. This is the column's *grain*: + +```{r grain} +set.seed(3) +trials <- data.frame( + trial = factor(rep(sprintf("T%d", seq_len(8L)), each = 12L)), + rep = factor(rep(rep(seq_len(3L), each = 4L), 8L)), + gen = factor(rep(sprintf("G%d", seq_len(4L)), 24L)) +) +trial_rain <- round(runif(8L, 250, 550)) +trials$rainfall <- trial_rain[as.integer(trials$trial)] +trials$yield <- round(2 + trials$rainfall / 200 + rnorm(96L, sd = 0.4), 2) + +plan_trials <- propose_roles(trials, mode = "collaborate") +by_unit <- mask(trials, plan_trials, mode = "collaborate", seed = 1L) +by_row <- mask(trials, plan_trials, mode = "collaborate", seed = 1L, + grain = FALSE) +values_per_trial <- function(m) { + s <- synthetic(m) + as.vector(tapply(s$rainfall, s$trial, function(x) length(unique(x)))) +} +knitr::kable( + data.frame( + clone = c("grain = TRUE (default)", "grain = FALSE"), + distinct_rainfall_per_trial = c( + paste(values_per_trial(by_unit), collapse = " "), + paste(values_per_trial(by_row), collapse = " ") + ) + ), + caption = "Distinct rainfall values in each of the eight trials of the clone." +) +``` + +`propose_roles()` notes such a column ("Constant within trial; synthesised +per trial"), and the recipe records each grain. The clone keeps one value +per unit, the frequencies of a categorical column across units, and the +correlation among columns of the same grain. It does not keep the +correlation between a grained column and the row-level columns, so the +link between rainfall and yield above is lost, and grained columns are +not conditioned on treatment when `conditional = TRUE`. + +In local mode the unit values are drawn from the real unit values. In +collaborate mode they are jittered, and a grain with fewer than five units +is synthesised row by row instead, with a `masque_grain_small_units` +warning, because resampling so few real values would disclose them. The +audit then notes that the clone does not keep that grain. `grain = FALSE` +gives the row-by-row clone of masque 0.15.0. + +### Time-to-event outcomes + +A clinical table often holds several event times on one row, such as time +to recurrence and time to death, each with a status. Re-simulated as +independent columns, the times would fall out of order. Declaring them to +`events` keeps them linked. The times are listed in the order the events +occur: + +```{r events} +set.seed(5) +n_pat <- 300L +arm <- rep(c("A", "B"), length.out = n_pat) +t_rec <- rexp(n_pat, ifelse(arm == "A", 1 / 900, 1 / 1400)) +t_death <- rexp(n_pat, 1 / 2500) +censor <- runif(n_pat, 30, 3000) +status_death <- as.integer(t_death <= censor) +time_death <- round(pmin(t_death, censor)) +status_rec <- as.integer(t_rec < pmin(t_death, censor)) +time_rec <- ifelse(status_rec == 1L, pmin(round(t_rec), time_death), + time_death) +patients <- data.frame(arm, time_rec, status_rec, time_death, status_death) + +plan_pat <- propose_roles(patients, mode = "collaborate", detect = FALSE) +plan_pat <- set_role(plan_pat, "arm", role = "treatment") +for (cn in c("time_rec", "status_rec", "time_death", "status_death")) { + plan_pat <- set_role(plan_pat, cn, role = "outcome", action = "scramble") +} +m_pat <- mask( + patients, plan_pat, mode = "collaborate", seed = 1L, + events = list(time = c("time_rec", "time_death"), + status = c("status_rec", "status_death")) +) +s_pat <- synthetic(m_pat) +all(s_pat$time_rec <= s_pat$time_death) +pattern <- function(d) paste(d$status_rec, d$status_death, sep = "/") +knitr::kable( + data.frame( + pattern = names(table(pattern(patients))), + original = as.vector(table(pattern(patients))), + clone = as.vector(table(pattern(s_pat))) + ), + caption = "Rows in each recurrence/death status pattern." +) +``` + +Within each status pattern the gaps between events are re-simulated and +summed back into times, so the times stay in order, a patient without a +recurrence keeps both times equal, and the count of each pattern is exact. +No final time runs past the longest follow-up observed in its pattern. The +status patterns move across rows, within each arm when +`conditional = TRUE`, so no real patient keeps their own outcome. No +hazard model is fitted, and the clone keeps no relation between a +covariate and the events. + ### The leakage audit, and a refused write `audit_mask()` inspects the synthetic against the original and grades @@ -381,9 +520,9 @@ df <- data.frame( stringsAsFactors = FALSE ) -roles <- propose_roles(df, mode = "collaborate") +plan <- propose_roles(df, mode = "collaborate") knitr::kable( - roles[, c("col", "role", "action", "pii_suspected")], + plan[, c("col", "role", "action", "pii_suspected")], caption = paste0( "The proposed plan: `contact_email` is auto-flagged ", "`pii_suspected` and dropped." @@ -395,10 +534,10 @@ Suppose the custodian overrides the flag, keeps `contact_email`, and makes the rare column a covariate: ```{r leaky-mask} -roles <- set_role(roles, "yield", role = "outcome") -roles <- set_role(roles, "contact_email", role = "covariate", action = "keep") -roles <- set_role(roles, "rare_treatment", role = "covariate") -m <- mask(df, roles, mode = "collaborate", seed = 1L) +plan <- set_role(plan, "yield", role = "outcome") +plan <- set_role(plan, "contact_email", role = "covariate", action = "keep") +plan <- set_role(plan, "rare_treatment", role = "covariate") +m <- mask(df, plan, mode = "collaborate", seed = 1L) knitr::kable( audit_mask(m)[, c("col", "leakage_class", "notes")], caption = paste0( @@ -416,7 +555,7 @@ HIGH finding stands, the package will not write the synthetic to disk: ```{r leaky-write, error = TRUE} out_dir <- tempfile() -masque(df, roles = roles, out = out_dir, mode = "collaborate", seed = 1L, +masque(df, roles = plan, out = out_dir, mode = "collaborate", seed = 1L, quiet = TRUE) dir.exists(out_dir) ``` @@ -429,6 +568,71 @@ has reviewed the findings and still wants to proceed can pass `allow_high warning and recorded in the recipe's warnings, so the exception stays auditable rather than silent. +The audit also checks the table as a whole. Each column is graded on how +far its row-by-row match with the original exceeds what chance gives, and +`attr(audit, "joint")` holds table-level checks: rows whose every re-simulated +column equals the original, and rare combinations of retained values. A +HIGH table-level finding blocks the writers as a column finding does. + +An outcome left at `keep` in collaborate mode hands over the real +responses, so it audits HIGH: + +```{r kept-outcome} +trial_kept <- data.frame( + block = factor(rep(seq_len(4L), each = 6L)), + variety = factor(rep(sprintf("V%d", seq_len(6L)), 4L)), + height = round(rnorm(24L, 90, 8)), + yield = round(rnorm(24L, 5, 1), 2) +) +plan_kept <- propose_roles(trial_kept, mode = "collaborate") +plan_kept <- set_role(plan_kept, "yield", role = "outcome", action = "keep") +m_kept <- mask(trial_kept, plan_kept, mode = "collaborate", seed = 1L) +knitr::kable( + audit_mask(m_kept, print = FALSE)[col == "yield", c("col", "leakage_class")], + caption = "A kept outcome in a collaborate clone." +) +``` + +If the custodian has decided the real outcome may be shared, passing +`allow_kept_outcome = TRUE` grades the column medium and records the +choice on the recipe: + +```{r kept-outcome-allowed} +m_allowed <- mask(trial_kept, plan_kept, mode = "collaborate", seed = 1L, + allow_kept_outcome = TRUE) +knitr::kable( + audit_mask(m_allowed, print = FALSE)[ + col == "yield", c("col", "leakage_class") + ], + caption = "The same outcome, allowed by the custodian." +) +recipe(m_allowed)@allow_kept_outcome +``` + +The table-level checks still apply. When the kept outcome is the only +column that would otherwise be re-simulated, the clone matches the +original row for row, the whole-row check stays HIGH, and the writers stay +blocked. + +### Small tables + +Re-simulating a column of a few rows returns the original values in a new +order. Collaborate mode therefore refuses a table of fewer than five rows +when any column would be re-simulated: + +```{r small-table, error = TRUE} +tiny <- trial_kept[1:4, c("block", "yield")] +mask(tiny, propose_roles(tiny, mode = "collaborate"), mode = "collaborate", + seed = 1L) +``` + +Local mode warns instead. Kept, aliased and dropped columns are not +re-simulated, so a short lookup table of site codes masks in either mode. +Five rows is a floor, not a guarantee. A column of a few dozen rows still +carries values close to the real ones, and jitter within a measurement +step does little to hide them. The same applies to small strata of a +conditional clone, which is why its cells hold at least five rows. + Beyond that gate the release decision stays with the custodian. Whether a synthetic table is appropriate for a given collaborator, environment, or jurisdiction depends on context the package cannot see. `masque` informs @@ -443,15 +647,26 @@ linked key is the join surface, aliased consistently rather than permuted ```{r set} set_dir <- system.file("extdata", "met_set", package = "masque") -ms <- mask_set(set_dir, mode = "collaborate", seed = 1L, quiet = TRUE) +tabs <- read_set(set_dir) +tabs$agronomy[, year := factor(year)] +ms <- mask_set(tabs, mode = "collaborate", seed = 1L, quiet = TRUE) ag <- synthetic(ms)$agronomy qa <- synthetic(ms)$quality join_survives <- setequal(unique(ag$gen), unique(qa$gen)) join_survives +head(unique(ag$gen), 3L) ``` +`year` is converted to a factor first. As a number it would be kept +with its real values, with a warning, because a numeric environment column +cannot be aliased. A linked key gets aliases of the form `_K001`, +such as `gen_K001`, from one map shared by every table. + The recipe bundle is private exactly as a single recipe is; `write_set()` -never writes it. +never writes it. `save_recipe()` and `read_recipe()` keep it as one +`.rds` file, `audit_mask(ms)` returns one audit per table, and +`reveal_maps(recipe(ms), table = "agronomy")` reveals the maps of one +table. ### Geographic coordinates @@ -554,7 +769,7 @@ Declaring the pair to `mask()` applies the same coarsening as part of masking, so the coordinates never pass through the copula: ```{r geo-mask, eval = FALSE} -m <- mask(df, roles, mode = "collaborate", +m <- mask(df, plan, mode = "collaborate", coords = list(list(lat = "GPS_S", lon = "GPS_E", by = "TRIAL")), seed = 1L) ``` @@ -592,9 +807,14 @@ fidelity would not notice. The conditional clone keeps it (`r round(effect_cond, 2)`). Neither keeps a non-monotone relationship between two numeric columns. +A column constant within a trial stays constant within it, and linked +event times stay in order with exact pattern counts. + The audit caught both planted problems and the write was refused: -`dir.exists(out_dir)` is `r dir.exists(out_dir)`. Linked tables share one -alias map for their key, so they still join. +`dir.exists(out_dir)` is `r dir.exists(out_dir)`. A kept outcome audits +HIGH unless the custodian allows it, and a collaborate table of fewer than +five rows is refused. Linked tables share one alias map for their key, so +they still join. A synthetic clone therefore protects the labels and the raw values behind them. It does not hide the design footprint or the level frequencies, and @@ -634,8 +854,10 @@ original. `set.seed(1)` is set once for the whole document; the conditional-clone fixture reseeds explicitly with `set.seed(42)` and the copula-shape fixture with `set.seed(7)`, so both sections are self-contained regardless of -chunk execution order. Every `mask()`/`mask_set()`/`jitter_coordinates()` -call also passes `seed = 1L` explicitly. Package versions follow. +chunk execution order. The grain fixture reseeds with `set.seed(3)` and the +events fixture with `set.seed(5)`. Every +`mask()`/`mask_set()`/`jitter_coordinates()` call also passes `seed = 1L` +explicitly. Package versions follow. ```{r session-info} sessionInfo() diff --git a/vignettes/getting_started.Rmd b/vignettes/getting_started.Rmd index 4973d20..6d3f0f1 100644 --- a/vignettes/getting_started.Rmd +++ b/vignettes/getting_started.Rmd @@ -74,7 +74,8 @@ The `mode` argument sets the defaults for both axes at once: * `local` is for the custodian's own development and keeps labels intact. * `collaborate` is for handing the synthetic to someone else. It aliases treatment and categorical labels, jitters numerics, drops identifiers and - free text, and runs the leakage audit. + free text (a character column in which more than half the values are + distinct), and runs the leakage audit. A per-column `action` always overrides the mode's default. @@ -117,10 +118,10 @@ knitr::kable( Under the one-call path, `masque()` built the roles table without asking. Building and reviewing it directly gives full control: -```{r roles} -roles <- propose_roles(df, mode = "collaborate") +```{r plan} +plan <- propose_roles(df, mode = "collaborate") knitr::kable( - roles[, c("col", "role", "action", "kind")], + plan[, c("col", "role", "action", "kind")], caption = paste0( "The proposed masking plan for the alpha-design trial ", "(collaborate mode)." @@ -130,15 +131,17 @@ knitr::kable( `propose_roles()` fills in a sensible action for each column given the mode, so the table reviewed here is the plan that will run. `yield` was -proposed as a `covariate`. Naming it the trial's `outcome` does not change -how it is masked. It records intent, and the *conditional* clone (see -*Confidentiality and the threat model*) reads it to find the response: +proposed as the trial's `outcome` from its name. The role records intent, +and the *conditional* clone (see *Confidentiality and the threat model*) +reads it to find the response. Any role can be changed with `set_role()`. +Here the plot number is re-roled as an identifier, which collaborate mode +drops: ```{r set-role} -roles <- set_role(roles, "yield", role = "outcome") +plan_id <- set_role(plan, "plot", role = "id") knitr::kable( - roles[, c("col", "role", "action")], - caption = "The plan after naming `yield` as the trial outcome." + plan_id[, c("col", "role", "action")], + caption = "The plan after re-roling `plot` as an identifier." ) ``` @@ -153,7 +156,7 @@ guided prompt opens when the custodian chooses `e` edits the same table. `set_role()` is vectorised over columns: ```{r edit-vectorised} -r2 <- set_role(roles, c("row", "col"), action = "drop") +r2 <- set_role(plan, c("row", "col"), action = "drop") ``` Direct edits work too. A direct `role` edit leaves `action` untouched, so @@ -195,14 +198,14 @@ scan reads names, not content, so when a harmlessly named column holds sensitive values, flagging it by hand is the correct fix: ```{r pii-by-hand, eval = FALSE} -roles[col == "comments", pii_suspected := TRUE] +plan[col == "comments", pii_suspected := TRUE] ``` Passing the edited table back to `mask()` (or to `masque(df, roles = -roles)`) runs the plan: +plan)`) runs the plan: ```{r mask} -m <- mask(df, roles, mode = "collaborate", seed = 1L) +m <- mask(df, plan, mode = "collaborate", seed = 1L) synth <- synthetic(m) byte_identical_plot <- identical(synth$plot, df$plot) gen_levels_changed <- !setequal(levels(synth$gen), levels(df$gen)) @@ -217,8 +220,8 @@ head(levels(synth$gen)) and `mask()` refuses: ```{r fail-closed, error = TRUE} -bad_roles <- set_role(roles, "plot", role = "design", action = "scramble") -mask(df, bad_roles, mode = "collaborate", seed = 1L) +bad_plan <- set_role(plan, "plot", role = "design", action = "scramble") +mask(df, bad_plan, mode = "collaborate", seed = 1L) ``` The call stops before any masking, names the column and the combination at @@ -242,8 +245,26 @@ ms ``` The genotype column `gen` appears in both tables and is masked to the same -codes in each, so the field and laboratory tables still join. See -*Confidentiality and the threat model* for the set-level controls. +codes in each, so the field and laboratory tables still join. + +The warning is about `year`. It is an environment column, but it is +numeric, so collaborate mode keeps its real values. To alias it, read the +tables, convert `year` to a factor, and mask the list: + +```{r set-year} +tabs <- read_set(set_dir) +tabs$agronomy[, year := factor(year)] +ms <- masque(tabs, mode = "collaborate", seed = 1L, ask = FALSE, quiet = TRUE) +knitr::kable( + head(synthetic(ms)$agronomy[, c("env", "loc", "year", "gen")], 3L), + caption = "The first rows of the clone, with `year` now aliased." +) +``` + +For a workbook, `sheets =` picks which sheets to read, and `info =` names +documentation sheets, such as a data dictionary, to set aside: they are +not masked and not written. See *Confidentiality and the threat model* for +the set-level controls. ### Describing the clone: the data card @@ -381,14 +402,15 @@ scientific inference. ## Results ```{r audit-tally, include = FALSE} -audit_class <- audit_mask(m)$leakage_class -n_high <- sum(audit_class == "HIGH") -n_medium <- sum(audit_class == "medium") +audit <- audit_mask(m, print = FALSE) +grades <- c(audit$leakage_class, attr(audit, "joint")$leakage_class) +n_high <- sum(grades == "high") +n_medium <- sum(grades == "medium") ``` -On this trial the audit graded `r n_high` columns HIGH and `r n_medium` -medium. A HIGH finding would stop the package writing the synthetic to -disk. A design column left at `keep` comes back unchanged +On this trial the audit, counting its column and table-level checks +together, gave `r n_high` HIGH and `r n_medium` medium findings. A HIGH +finding would stop the package writing the synthetic to disk. A design column left at `keep` comes back unchanged and an aliased treatment comes back as `trt_001` to `trt_024`; a plan that asks for something the package cannot do is refused before it runs. Related tables are masked together and still join. diff --git a/vignettes/recipe_anatomy.Rmd b/vignettes/recipe_anatomy.Rmd index b500fb8..f6741f4 100644 --- a/vignettes/recipe_anatomy.Rmd +++ b/vignettes/recipe_anatomy.Rmd @@ -25,8 +25,7 @@ set.seed(1) f <- system.file("extdata", "john_alpha.csv", package = "masque") df <- read.csv(f, stringsAsFactors = TRUE) -roles <- propose_roles(df, mode = "collaborate") -roles <- set_role(roles, "yield", role = "outcome") +plan <- propose_roles(df, mode = "collaborate") ``` ## Why @@ -56,7 +55,7 @@ withholds. ### What a recipe holds ```{r accessors} -m <- mask(df, roles, mode = "collaborate", seed = 1L) +m <- mask(df, plan, mode = "collaborate", seed = 1L) rec <- recipe(m) class(rec) ``` @@ -73,7 +72,13 @@ A recipe holds: conditional clone asked for, the stratum it got, the columns it dropped or carried as shifts, and the share of rows pooled. - `level_maps`: the per-column original-to-alias maps. The sensitive - part. + part. Aliases follow one scheme: `trt_001`, `trt_002`, ... for a single + aliased treatment, factor or numeric (`_trt_001` when several + treatments are aliased), `_L001` for a categorical + covariate, `_D001` for a design column, `_I001` for an + identifier, `_T001` for free text, and `_K001` for a key + shared across the tables of a set. The numbers have at least three + digits, more when a column has over 999 levels. - `column_name_map`: the column-name aliases, when `alias_names` was used (otherwise `NULL`). - `storage_classes`, `factor_meta`: the original column classes, factor @@ -81,11 +86,20 @@ A recipe holds: - `cleaning`: the tidying done before masking (column names made valid R names, whitespace trimmed), so `apply_recipe()` re-applies the same tidying. +- `conform`: the label merges and type changes `conform_table()` made, so + `apply_recipe()` replays them on the original. +- `grain`: each design unit within which columns were synthesised once per + unit, with the columns, the number of units and whether the clone kept + the grain or fell back to rows. +- `events`: the linked time-to-event specs and how many final times were + capped. - `coords`: one entry per coordinate pair declared to `mask()`'s `coords` argument: the jitter method and radius, the site grouping it was masked under, and how many sites the grouping produced. - `allow_unmasked_coords`: `TRUE` only when the caller deliberately wrote a real coordinate through unmasked. +- `allow_kept_outcome`: `TRUE` only when the caller allowed a kept outcome + in a collaborate clone. - `warnings`: the warnings raised while masking, including any HIGH audit finding. - `integrity_fp`: a SHA-256 of `is.na(original)`, an integrity @@ -93,7 +107,8 @@ A recipe holds: A recipe written by an older version has no record for the slots added since, for example `coords` before 0.10.0 and `allow_unmasked_coords` -before 0.11.0. The changelog lists when each slot was added. +before 0.11.0. It still prints, applies and unmasks. The changelog lists +when each slot was added. It does not hold the copula covariance, the raw observed values, or any file paths or usernames. @@ -105,11 +120,10 @@ an audit of the recipe does not need the original data back: df_geo <- df df_geo$lat <- -34.9 + stats::runif(nrow(df_geo), -0.05, 0.05) df_geo$lon <- 138.6 + stats::runif(nrow(df_geo), -0.05, 0.05) -roles_geo <- propose_roles(df_geo, mode = "collaborate") -roles_geo <- set_role(roles_geo, "yield", role = "outcome") -roles_geo <- set_role(roles_geo, c("lat", "lon"), action = "keep") +plan_geo <- propose_roles(df_geo, mode = "collaborate") +plan_geo <- set_role(plan_geo, c("lat", "lon"), action = "keep") -m_geo <- mask(df_geo, roles_geo, mode = "collaborate", seed = 1L, +m_geo <- mask(df_geo, plan_geo, mode = "collaborate", seed = 1L, coords = list(list(lat = "lat", lon = "lon"))) n_sites_recorded <- recipe(m_geo)@coords[[1]]$n_sites recipe(m_geo)@coords[[1]][c("method", "min_km", "max_km", "n_sites")] @@ -136,8 +150,10 @@ themselves: rec ``` -Anyone holding the recipe can print the maps with an explicit call, which -shows a warning first: +The same holds for `format()`, `str()` and knitr output of a recipe and of +a `masque` object, so the seed and the maps do not end up in a log or a +rendered report by accident. Anyone holding the recipe can print the maps +with an explicit call, which shows a warning first: ```{r reveal, results = "hide"} reveal_maps(rec) @@ -168,8 +184,10 @@ labels_recovered <- identical(as.character(back$gen), as.character(df$gen)) labels_recovered ``` -Columns whose action is `keep`, numeric columns and numeric predictions -pass through both functions unchanged. `apply_recipe()` returns an aliased +Columns whose action is `keep`, re-simulated numeric columns and numeric +predictions pass through both functions unchanged. A numeric treatment +that was aliased is mapped both ways, and `unmask()` returns its exact +original values. `apply_recipe()` returns an aliased factor with the clone's levels in the clone's order, so a model fitted on the translated original has the same reference level as on the clone. @@ -194,7 +212,7 @@ one. ### Saving the recipe `save_recipe()` writes a single small `.rds`, to be stored next to the -original at the same security class. `read_recipe()` +original at the same security class. The path must end in `.rds`. `read_recipe()` validates the file and notes a version mismatch without raising an error. ```{r save} @@ -210,16 +228,22 @@ fingerprint_survives_roundtrip A `mask_set()` result carries a recipe *bundle*: one recipe per table plus the shared cross-table link maps. The same `apply_recipe()` and `unmask()` verbs dispatch on it, operating table by table over a named -list: +list. `year` is made a factor before masking so that collaborate mode can +alias it (see *Getting started with masque*): ```{r set-bundle} set_dir <- system.file("extdata", "met_set", package = "masque") -ms <- mask_set(set_dir, mode = "collaborate", seed = 1L, quiet = TRUE) originals <- read_set(set_dir) +originals$agronomy[, year := factor(year)] +ms <- mask_set(originals, mode = "collaborate", seed = 1L, quiet = TRUE) fwd_set <- apply_recipe(originals, recipe(ms)) names(fwd_set) ``` +`save_recipe()` and `read_recipe()` keep a bundle as one file, and +`reveal_maps(recipe(ms), table = "quality")` reveals the maps of the named +tables only. + Because the shared keys were aliased consistently at masking time, the re-targeted tables join on the same keys the synthetic tables do. From f7c308920d48916d0ea0fed5e766d5343fdf3049 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:37:27 +1030 Subject: [PATCH 53/66] docs: plainer audit line in NEWS --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 77d60e3..3d4c20e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -49,8 +49,8 @@ * `audit_mask()` grades each column on its excess over a chance baseline and adds table-level checks (whole-row matches, rare combinations) in `attr(audit, "joint")`. A column synthesised per unit, or a whole-numbered - covariate such as age, is graded on that excess instead of on raw exact - matches, which it reaches by construction. + covariate (age in years), is graded on that excess instead of on raw + exact matches, which it reaches by construction. * `audit_mask()` grades such a column HIGH when it matches the original row by row far above chance, as when the real values were copied back. * `save_recipe()`, `read_recipe()`, `reveal_maps(table = )` and From 4ad2d39ecd48336e5be4b91c15283a0f3329fd25 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:39:00 +1030 Subject: [PATCH 54/66] fix: replication tile skips with a warning when no block column exists x@block_cols[1L] is NA on an empty vector, so the length guard never fired and table() errored instead. --- R/detect_plot.R | 2 +- tests/testthat/test-fix0160-warning-classes.R | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/R/detect_plot.R b/R/detect_plot.R index fd8db3e..7541b39 100644 --- a/R/detect_plot.R +++ b/R/detect_plot.R @@ -255,7 +255,7 @@ local(S7::method(plot, design_summary) <- function( .plot_replication_tile <- function(x, df, engine) { trt <- x@treatment_col[1L] block <- x@block_cols[1L] - if (is.null(trt) || is.na(trt) || length(block) == 0L) { + if (is.null(trt) || is.na(trt) || is.na(block)) { cli::cli_warn( "Replication tile needs treatment + block; skipping.", class = "masque_plot_skipped" diff --git a/tests/testthat/test-fix0160-warning-classes.R b/tests/testthat/test-fix0160-warning-classes.R index a7cf2ed..a0afd1b 100644 --- a/tests/testthat/test-fix0160-warning-classes.R +++ b/tests/testthat/test-fix0160-warning-classes.R @@ -52,4 +52,10 @@ test_that("a design plot panel that cannot be drawn warns by class", { masque:::.plot_replication_tile(ds, df, "base"), class = "masque_plot_skipped" ) + ds@treatment_col <- "trt" + ds@block_cols <- character() + expect_warning( + masque:::.plot_replication_tile(ds, df, "base"), + class = "masque_plot_skipped" + ) }) From b75aec5e03634e0b3301d2c8b3090ea1bf09968c Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 12:39:38 +1030 Subject: [PATCH 55/66] docs: NEWS lines for the calibrated audit rule, count grading, warning classes, link stream and tile guard --- NEWS.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/NEWS.md b/NEWS.md index 3d4c20e..67eef78 100644 --- a/NEWS.md +++ b/NEWS.md @@ -28,8 +28,10 @@ * masque needs data.table 1.15.0 or later. Returned tables are data.tables, as before. * `mask_set()` link aliases now follow `seed` whatever the caller's random - state, so a set made with a seed in 0.15.0 gets different link aliases - when it is made again. + state, from their own stream derived from `seed`, so a set made with a + seed in 0.15.0 gets different link aliases when it is made again. +* A whole-numbered outcome (a count) is graded on its excess over chance: + an ordinary clone audits low instead of medium. ## New features @@ -51,8 +53,14 @@ `attr(audit, "joint")`. A column synthesised per unit, or a whole-numbered covariate (age in years), is graded on that excess instead of on raw exact matches, which it reaches by construction. -* `audit_mask()` grades such a column HIGH when it matches the original - row by row far above chance, as when the real values were copied back. +* `audit_mask()` grades such a column HIGH when it matches its own row or + design unit far above chance (50 points and 4 standard deviations), and a + continuous per-unit column HIGH when its unit values are the real ones, + even moved between units. With 5 to 8 units and few distinct values a + copy cannot be told from chance, and the audit grades it low. +* Every warning carries a `masque_` class, including + `masque_roles_upgraded`, `masque_roles_mode_mismatch`, + `masque_plot_skipped` and `masque_geo_anchor_missing`. * `save_recipe()`, `read_recipe()`, `reveal_maps(table = )` and `audit_mask()` work on a `mask_set()` result. * The recipe records `conform_table()` decisions, and `apply_recipe()` @@ -93,6 +101,8 @@ * The guided summary counts columns as the audit does, and the non-interactive notice no longer suggests an argument already passed. * The vignettes restore the `options()` they set. +* The design plot skips the replication panel with a warning, instead of + an error, when there is no block column. # masque 0.15.0 From a0a802c7dc53e742558f3d07f6e9c15eb665b374 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 14:15:15 +1030 Subject: [PATCH 56/66] fix: reveal_maps() prints through cli, so suppressMessages() silences it --- R/recipe_print.R | 6 +++--- tests/testthat/test-recipe-redaction.R | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/R/recipe_print.R b/R/recipe_print.R index 0fda768..c993fc4 100644 --- a/R/recipe_print.R +++ b/R/recipe_print.R @@ -327,7 +327,7 @@ reveal_maps <- function(rec, table = NULL) { lines <- sprintf( " %s -> %s", names(cmap), unname(unlist(cmap)) ) - cat(lines, sep = "\n") + cli::cli_verbatim(lines) } if (length(rec@level_maps) == 0L) { @@ -341,12 +341,12 @@ reveal_maps <- function(rec, table = NULL) { cli::cli_h3(col) map <- rec@level_maps[[col]] lines <- sprintf(" %s -> %s", names(map), unname(map)) - cat(lines, sep = "\n") + cli::cli_verbatim(lines) } if (length(rec@seed)) { cli::cli_h3("seed") - cat(sprintf(" %d\n", rec@seed)) + cli::cli_verbatim(sprintf(" %d", rec@seed)) } invisible(rec) } diff --git a/tests/testthat/test-recipe-redaction.R b/tests/testthat/test-recipe-redaction.R index d088f7d..fcca6a1 100644 --- a/tests/testthat/test-recipe-redaction.R +++ b/tests/testthat/test-recipe-redaction.R @@ -74,6 +74,15 @@ test_that("reveal_maps prints a warning banner before maps", { expect_match(joined, "Revealing sensitive") }) +test_that("reveal_maps writes nothing to stdout, so it can be silenced", { + r <- propose_roles(iris) + r$role[r$col == "Sepal.Length"] <- "outcome" + r$role[r$col == "Species"] <- "covariate" + m <- mask(iris, r, mode = "collaborate", seed = 1) + out <- utils::capture.output(suppressMessages(reveal_maps(recipe(m)))) + expect_identical(out, character()) +}) + test_that("reveal_maps on an empty-maps recipe says so", { r <- propose_roles(iris) r$role[r$col == "Sepal.Length"] <- "outcome" From ca8f599479d05775f2ddd2225f2b0c4456680c22 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 14:18:11 +1030 Subject: [PATCH 57/66] fix: import utils::str so the namespace loads with only base attached --- NAMESPACE | 1 + R/masque-package.R | 1 + 2 files changed, 2 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 62cfbc0..977337d 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -27,3 +27,4 @@ export(synthesise_geospatial) export(synthetic) export(unmask) export(write_set) +importFrom(utils,str) diff --git a/R/masque-package.R b/R/masque-package.R index 6fe7f13..b026621 100644 --- a/R/masque-package.R +++ b/R/masque-package.R @@ -37,6 +37,7 @@ #' whole numbers stochastically rounded; `audit_mask()` auto-runs at #' construction. #' +#' @importFrom utils str #' @keywords internal "_PACKAGE" From fe07ef3b1dc65a4feed88d858a50846bd553acb1 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 14:19:51 +1030 Subject: [PATCH 58/66] fix: import data.table's := for the merge helpers --- NAMESPACE | 1 + R/masque-package.R | 1 + 2 files changed, 2 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 977337d..16104aa 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -27,4 +27,5 @@ export(synthesise_geospatial) export(synthetic) export(unmask) export(write_set) +importFrom(data.table,":=") importFrom(utils,str) diff --git a/R/masque-package.R b/R/masque-package.R index b026621..1244c65 100644 --- a/R/masque-package.R +++ b/R/masque-package.R @@ -37,6 +37,7 @@ #' whole numbers stochastically rounded; `audit_mask()` auto-runs at #' construction. #' +#' @importFrom data.table := #' @importFrom utils str #' @keywords internal "_PACKAGE" From ce71974ed5ce80ab2202864b3a00061d0ce61630 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 15:51:59 +1030 Subject: [PATCH 59/66] test: hash clones at 10 significant digits so the pinned digests hold on x86_64 The 15-digit text form differs between arm64 and x86_64 in the last digit, so the reproducibility and grain = FALSE digests failed on every Linux and Windows CI job. Local-mode digests at 10 digits are unchanged from 12ead24. --- tests/testthat/test-fix0160-grain.R | 11 ++++++----- tests/testthat/test-fix0160-reproducibility.R | 13 +++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R index 3af956a..3c9cf78 100644 --- a/tests/testthat/test-fix0160-grain.R +++ b/tests/testthat/test-fix0160-grain.R @@ -79,13 +79,14 @@ r } -# Text form of a clone: doubles to 15 significant digits, then the level maps. +# Text form of a clone: doubles to 10 significant digits (15 is not stable +# across arm64 and x86_64), then the level maps. .fx_canon <- function(m) { syn <- as.data.frame(synthetic(m)) cols <- vapply(names(syn), function(nm) { x <- syn[[nm]] v <- if (is.double(x) && !inherits(x, "Date")) { - sprintf("%.15g", x) + sprintf("%.10g", x) } else { as.character(x) } @@ -300,11 +301,11 @@ test_that("grain = FALSE is the row-level clone of masque 0.15", { local_FALSE = "65154f4d80860124e54c67e3fcccc934d3757823dac158f5d2dda8ac98764601", local_TRUE = - "789a764980d220b4924e1367dd6e47fe8db31c903d9b02196d5635390a689808", + "73d686ac8878aa5b73cddb8bfed9d29a5151cd582316206e00d496d74da2307e", collaborate_FALSE = - "684cc07c5fa22d2c70bd897250250a0f79fde0b2091b149b539623f688ac5de1", + "a14edc6096ebd4e41aee38b4bf105fa085423a0ffd7f0a907f7af243ef59a6fc", collaborate_TRUE = - "154490c0135a592e49167c306127438a6723266fb96142762682674673db10e6" + "d5cae870dc9308415ce59945dc98bc00998fdb882bfffe243f10c1848f5022ba" ) df <- .fx_met() for (cb in .fx_combos) { diff --git a/tests/testthat/test-fix0160-reproducibility.R b/tests/testthat/test-fix0160-reproducibility.R index 1a2adfa..40de284 100644 --- a/tests/testthat/test-fix0160-reproducibility.R +++ b/tests/testthat/test-fix0160-reproducibility.R @@ -36,14 +36,15 @@ r } -# Text form of a clone: 15 significant digits per double, so the digest is -# independent of serialisation format and of last-bit platform noise. +# Text form of a clone: 10 significant digits per double, so the digest is +# independent of serialisation format and of last-bit platform noise (at 15 +# digits arm64 and x86_64 disagree). .repro_canon <- function(m) { syn <- as.data.frame(synthetic(m)) cols <- vapply(names(syn), function(nm) { x <- syn[[nm]] v <- if (is.double(x) && !inherits(x, "Date")) { - sprintf("%.15g", x) + sprintf("%.10g", x) } else { as.character(x) } @@ -65,11 +66,11 @@ test_that("a table with no grained column clones as on the 0.16.0 base", { local_FALSE = "ba60bae634f27d436a003f1ab22bc4146a77e64aea643d9716349d0325499097", local_TRUE = - "dbecf2f681904bbef71d17f1169cddca7eb78ff67e85040611a038d0046c75c3", + "50992aff8e147e21a46c7aeee0276b2e3a47e140a67998d679a3869ecbc24b52", collaborate_FALSE = - "3c4422bc3cbe1ceed6e709e396459ed71e9dbb1a0b32cc7716f2d5b8d2a7fe73", + "816c6e8d7c466a000bed73c43b4b5d4a2a7c70a252e70c79d7a670d99f7ad0a8", collaborate_TRUE = - "51faf1085e3f60405f8a21003c09b273a759cb83238197acfa53406185b5863a" + "de986487ca13bd4e9e76d2eed909d7925bd5dcf7a0c81dabc0aafcd582121d4b" ) df <- .repro_fixture() for (mode in c("local", "collaborate")) { From cca53c48de3654730975084f902a0b8a452610da Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 19:11:25 +1030 Subject: [PATCH 60/66] fix: refuse a declared coordinate whose proposed action is not keep (N3) A proposed drop is part of the reviewed plan; coords no longer overrides it. --- R/jitter_coordinates.R | 64 ++++++++-------------- R/mask.R | 11 ++-- man/mask.Rd | 7 +-- tests/testthat/test-coord-guard.R | 1 + tests/testthat/test-coord-site-invariant.R | 3 + tests/testthat/test-fix0160-coords.R | 19 +++++-- tests/testthat/test-jitter-coordinates.R | 2 + vignettes/confidentiality.Rmd | 1 + 8 files changed, 51 insertions(+), 57 deletions(-) diff --git a/R/jitter_coordinates.R b/R/jitter_coordinates.R index 98d5d5d..f862c1b 100644 --- a/R/jitter_coordinates.R +++ b/R/jitter_coordinates.R @@ -550,48 +550,32 @@ jitter_coordinates <- function(df, lat_col, lon_col, unique(unlist(lapply(specs, function(s) c(s$lat, s$lon)), use.names = FALSE)) } -# Declared coordinates get action keep; an explicit or unknown non-keep -# action is refused, a proposed default yields with a recipe note. -.coords_take_keep <- function(roles, coord_cols) { +# A declared coordinate must already have action keep in the reviewed roles; +# any other action, proposed or chosen, is refused rather than overridden. +.coords_require_keep <- function(roles, coord_cols) { idx <- which(roles$col %in% coord_cols & roles$action != "keep") if (!length(idx)) { - return(list(roles = roles, note = character())) - } - cols <- roles$col[idx] - acts <- roles$action[idx] - pa <- attr(roles, "proposed_actions") - proposed <- if (is.null(pa)) { - rep(FALSE, length(idx)) - } else { - cols %in% names(pa) & acts == unname(pa[cols]) - } - if (!all(proposed)) { - bad <- cols[!proposed] - bad_act <- unique(acts[!proposed]) - noun <- if (length(bad) == 1L) "column" else "columns" - verb <- if (length(bad) == 1L) "has" else "have" - bad_code <- paste0("c(", paste0("\"", bad, "\"", collapse = ", "), ")") - cli::cli_abort(c( - paste0( - "Declared coordinate {noun} {.field {bad}} {verb} the action ", - "{.val {bad_act}} in {.arg roles}." - ), - x = paste0( - "A coordinate declared to {.arg coords} is coarsened and kept, which ", - "would override the reviewed action." - ), - i = paste0( - "Set the action to {.val keep} ({.code set_role(roles, {bad_code}, ", - "action = \"keep\")}), or remove the pair from {.arg coords}." - ) - ), class = c("masque_coords_action_refusal", "orchestra_refusal")) - } - roles$action[idx] <- "keep" - note <- sprintf( - "coords: %s had the proposed action %s; set to keep for the geomask.", - paste(cols, collapse = ", "), paste(unique(acts), collapse = "/") - ) - list(roles = roles, note = note) + return(invisible(roles)) + } + bad <- roles$col[idx] + bad_act <- unique(roles$action[idx]) + noun <- if (length(bad) == 1L) "column" else "columns" + verb <- if (length(bad) == 1L) "has" else "have" + bad_code <- paste0("c(", paste0("\"", bad, "\"", collapse = ", "), ")") + cli::cli_abort(c( + paste0( + "Declared coordinate {noun} {.field {bad}} {verb} the action ", + "{.val {bad_act}} in {.arg roles}." + ), + x = paste0( + "A coordinate declared to {.arg coords} is coarsened and kept, which ", + "would override the reviewed action." + ), + i = paste0( + "Set the action to {.val keep} ({.code set_role(roles, {bad_code}, ", + "action = \"keep\")}), or remove the pair from {.arg coords}." + ) + ), class = c("masque_coords_action_refusal", "orchestra_refusal")) } # Jitter each spec in place and return the frame with one report per spec. diff --git a/R/mask.R b/R/mask.R index 993f5db..0bcc50c 100644 --- a/R/mask.R +++ b/R/mask.R @@ -136,10 +136,9 @@ #' (`by`, `method`, `min_km`, `max_km`, `sd_km`, `on_land`); a single pair #' may be passed bare, and several pairs as a list of pairs. Defaults to a #' donut of 5-20 km on land. A declared column must have the action `keep` -#' in `roles`, or still carry the action [propose_roles()] proposed, which -#' the declaration replaces with `keep` (noted on the recipe); an -#' explicitly chosen `drop` or `scramble` is refused with a -#' `masque_coords_action_refusal` error. One displacement +#' in `roles`; any other action, including the `drop` that +#' [propose_roles()] proposes for a coordinate-named column, is refused +#' with a `masque_coords_action_refusal` error. One displacement #' is drawn per **site** and broadcast to that site's rows: `by` names the #' columns of `df` that identify a site, and is omitted when rows sharing an #' identical input coordinate already identify one. A declared pair always @@ -390,9 +389,7 @@ mask <- function(df, coord_specs <- .normalise_coords(coords, df) coord_cols <- .coord_cols(coord_specs) if (length(coord_cols)) { - taken <- .coords_take_keep(roles, coord_cols) - roles <- taken$roles - warnings_acc <- c(warnings_acc, taken$note) + .coords_require_keep(roles, coord_cols) } # Linked event outcomes skip the copula; they are re-simulated as gaps. diff --git a/man/mask.Rd b/man/mask.Rd index 3f683b9..d382d92 100644 --- a/man/mask.Rd +++ b/man/mask.Rd @@ -121,10 +121,9 @@ implausible locations. A pair is a named vector \code{c(lat = "lat_col", lon = " (\code{by}, \code{method}, \code{min_km}, \code{max_km}, \code{sd_km}, \code{on_land}); a single pair may be passed bare, and several pairs as a list of pairs. Defaults to a donut of 5-20 km on land. A declared column must have the action \code{keep} -in \code{roles}, or still carry the action \code{\link[=propose_roles]{propose_roles()}} proposed, which -the declaration replaces with \code{keep} (noted on the recipe); an -explicitly chosen \code{drop} or \code{scramble} is refused with a -\code{masque_coords_action_refusal} error. One displacement +in \code{roles}; any other action, including the \code{drop} that +\code{\link[=propose_roles]{propose_roles()}} proposes for a coordinate-named column, is refused +with a \code{masque_coords_action_refusal} error. One displacement is drawn per \strong{site} and broadcast to that site's rows: \code{by} names the columns of \code{df} that identify a site, and is omitted when rows sharing an identical input coordinate already identify one. A declared pair always diff --git a/tests/testthat/test-coord-guard.R b/tests/testthat/test-coord-guard.R index fd9d636..b9994a8 100644 --- a/tests/testthat/test-coord-guard.R +++ b/tests/testthat/test-coord-guard.R @@ -137,6 +137,7 @@ test_that("a coarsened coordinate audits as medium, described correctly", { y = stats::rnorm(n * k) ) r <- suppressWarnings(propose_roles(d, mode = "collaborate")) + r <- set_role(r, c("gps_s", "gps_e"), action = "keep") m <- suppressWarnings(mask( d, r, mode = "collaborate", seed = 1, coords = list(list(lat = "gps_s", lon = "gps_e", by = "trial", diff --git a/tests/testthat/test-coord-site-invariant.R b/tests/testthat/test-coord-site-invariant.R index 5ff4235..ac2876c 100644 --- a/tests/testthat/test-coord-site-invariant.R +++ b/tests/testthat/test-coord-site-invariant.R @@ -183,6 +183,8 @@ test_that("`by` rejects a non-grouping", { .masked_sites <- function(df, coords, ...) { roles <- suppressWarnings(propose_roles(df, detect = FALSE)) + cols <- unlist(lapply(coords, function(s) c(s[["lat"]], s[["lon"]]))) + roles <- set_role(roles, cols, action = "keep") m <- suppressWarnings(mask(df, roles, seed = 1, coords = coords, ...)) synthetic(m) } @@ -207,6 +209,7 @@ test_that("mask(coords=) accepts `by` and groups on the original table", { y = stats::rnorm(12) ) roles <- suppressWarnings(propose_roles(df, mode = "collaborate")) + roles <- set_role(roles, c("lat", "lon"), action = "keep") # `trial` is aliased in the synthetic; the grouping must still hold, because # it is taken from the original table rather than the masked one. m <- suppressWarnings(mask(df, roles, mode = "collaborate", seed = 1, diff --git a/tests/testthat/test-fix0160-coords.R b/tests/testthat/test-fix0160-coords.R index aa7d4a1..59a85b7 100644 --- a/tests/testthat/test-fix0160-coords.R +++ b/tests/testthat/test-fix0160-coords.R @@ -114,19 +114,26 @@ test_that("a direct edit away from the proposal is also explicit", { expect_match(conditionMessage(err), "keep") }) -test_that("the proposed default yields to the declaration and says so", { +test_that("the proposed default is a reviewed action and is refused too", { df <- .fx_sites() roles <- .fx_site_roles(df, coord_action = NULL) # propose_roles() proposes drop for a coordinate-named column. - expect_true(all(roles$action[roles$col %in% c("lat", "lon")] != "keep")) + expect_true(all(roles$action[roles$col %in% c("lat", "lon")] == "drop")) + err <- tryCatch( + suppressWarnings(mask( + df, roles, mode = "local", seed = 1, + coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) + )), + masque_coords_action_refusal = function(e) e + ) + expect_s3_class(err, "masque_coords_action_refusal") + expect_match(conditionMessage(err), "drop") + # The documented escape runs and keeps one displacement per site. + roles <- set_role(roles, c("lat", "lon"), action = "keep") m <- suppressWarnings(mask( df, roles, mode = "local", seed = 1, coords = list(list(lat = "lat", lon = "lon", on_land = FALSE)) )) - rec <- recipe(m) - expect_identical(rec@roles$action[rec@roles$col %in% c("lat", "lon")], - c("keep", "keep")) - expect_true(any(grepl("set to keep for the geomask", rec@warnings))) expect_true(.fx_one_displacement_per_site(df, synthetic(m))) }) diff --git a/tests/testthat/test-jitter-coordinates.R b/tests/testthat/test-jitter-coordinates.R index 5fac5ef..150a236 100644 --- a/tests/testthat/test-jitter-coordinates.R +++ b/tests/testthat/test-jitter-coordinates.R @@ -89,6 +89,7 @@ test_that("mask(coords=) coarsens a declared pair on land and records it", { df$gps_s[1] <- NA df$gps_e[1] <- NA roles <- suppressWarnings(propose_roles(df, mode = "collaborate")) + roles <- set_role(roles, c("gps_s", "gps_e"), action = "keep") m <- suppressWarnings(mask(df, roles, mode = "collaborate", seed = 1, coords = list(c(lat = "gps_s", lon = "gps_e")))) syn <- synthetic(m) @@ -119,6 +120,7 @@ test_that("coords accepts the c() form with numeric params (string-coerced)", { df <- data.frame(gps_s = lat[onl][1:40], gps_e = lon[onl][1:40], y = stats::rnorm(40)) roles <- suppressWarnings(propose_roles(df, detect = FALSE)) + roles <- set_role(roles, c("gps_s", "gps_e"), action = "keep") # c() coerces 10 / 18 to "10" / "18"; the coarsening must still honour them m <- suppressWarnings(mask(df, roles, seed = 1, mode = "local", coords = list(c(lat = "gps_s", lon = "gps_e", min_km = 10, max_km = 18)))) diff --git a/vignettes/confidentiality.Rmd b/vignettes/confidentiality.Rmd index 7056980..6410289 100644 --- a/vignettes/confidentiality.Rmd +++ b/vignettes/confidentiality.Rmd @@ -769,6 +769,7 @@ Declaring the pair to `mask()` applies the same coarsening as part of masking, so the coordinates never pass through the copula: ```{r geo-mask, eval = FALSE} +plan <- set_role(plan, c("GPS_S", "GPS_E"), action = "keep") m <- mask(df, plan, mode = "collaborate", coords = list(list(lat = "GPS_S", lon = "GPS_E", by = "TRIAL")), seed = 1L) From 318897081d3db8b878a9934c500c84f7a686f60a Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 19:11:25 +1030 Subject: [PATCH 61/66] test: check grain-free clones portably; run base digests only on macOS arm64 --- tests/testthat/helper-platform.R | 7 ++++ tests/testthat/test-fix0160-grain.R | 4 +-- tests/testthat/test-fix0160-reproducibility.R | 32 +++++++++++++++---- 3 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 tests/testthat/helper-platform.R diff --git a/tests/testthat/helper-platform.R b/tests/testthat/helper-platform.R new file mode 100644 index 0000000..a020f93 --- /dev/null +++ b/tests/testthat/helper-platform.R @@ -0,0 +1,7 @@ +# Clone digests were recorded on macOS arm64; floating-point output is not +# bit-stable across platforms, so the digest checks run only there. +skip_unless_digest_platform <- function() { + on_it <- identical(Sys.info()[["sysname"]], "Darwin") && + identical(R.version$arch, "aarch64") + testthat::skip_if_not(on_it, "clone digests were recorded on macOS arm64") +} diff --git a/tests/testthat/test-fix0160-grain.R b/tests/testthat/test-fix0160-grain.R index 3c9cf78..01deef1 100644 --- a/tests/testthat/test-fix0160-grain.R +++ b/tests/testthat/test-fix0160-grain.R @@ -79,8 +79,7 @@ r } -# Text form of a clone: doubles to 10 significant digits (15 is not stable -# across arm64 and x86_64), then the level maps. +# Text form of a clone: doubles to 10 significant digits, then the level maps. .fx_canon <- function(m) { syn <- as.data.frame(synthetic(m)) cols <- vapply(names(syn), function(nm) { @@ -318,6 +317,7 @@ test_that("grain = FALSE is the row-level clone of masque 0.15", { expect_identical(gr[[1]]$synthesised, "row", label = key) expect_identical(gr[[1]]$reason, "grain = FALSE", label = key) skip_on_cran() + skip_unless_digest_platform() expect_identical(.fx_canon(m), base[[key]], label = key) } }) diff --git a/tests/testthat/test-fix0160-reproducibility.R b/tests/testthat/test-fix0160-reproducibility.R index 40de284..6af4bbb 100644 --- a/tests/testthat/test-fix0160-reproducibility.R +++ b/tests/testthat/test-fix0160-reproducibility.R @@ -36,9 +36,7 @@ r } -# Text form of a clone: 10 significant digits per double, so the digest is -# independent of serialisation format and of last-bit platform noise (at 15 -# digits arm64 and x86_64 disagree). +# Text form of a clone, doubles at 10 significant digits. .repro_canon <- function(m) { syn <- as.data.frame(synthetic(m)) cols <- vapply(names(syn), function(nm) { @@ -60,7 +58,30 @@ serialize = FALSE) } -test_that("a table with no grained column clones as on the 0.16.0 base", { +test_that("a table with no grained column clones as with grain = FALSE", { + df <- .repro_fixture() + for (mode in c("local", "collaborate")) { + roles <- .repro_roles(df, mode) + expect_length(.detect_grain(df, roles), 0L) + for (cond in c(FALSE, TRUE)) { + key <- sprintf("%s_%s", mode, cond) + m <- suppressWarnings(suppressMessages(mask( + df, roles, mode = mode, seed = 1L, conditional = cond + ))) + expect_length(recipe(m)@grain, 0L) + m0 <- suppressWarnings(suppressMessages(mask( + df, roles, mode = mode, seed = 1L, conditional = cond, grain = FALSE + ))) + expect_identical(synthetic(m), synthetic(m0), label = key) + expect_identical(recipe(m)@level_maps, recipe(m0)@level_maps, + label = key) + } + } +}) + +test_that("on the recording platform the clones match the 0.16.0 base", { + skip_on_cran() + skip_unless_digest_platform() # Base 12ead24 digests; collaborate re-taken after the jitter clip went. base <- c( local_FALSE = @@ -75,14 +96,11 @@ test_that("a table with no grained column clones as on the 0.16.0 base", { df <- .repro_fixture() for (mode in c("local", "collaborate")) { roles <- .repro_roles(df, mode) - expect_length(.detect_grain(df, roles), 0L) for (cond in c(FALSE, TRUE)) { key <- sprintf("%s_%s", mode, cond) m <- suppressWarnings(suppressMessages(mask( df, roles, mode = mode, seed = 1L, conditional = cond ))) - expect_length(recipe(m)@grain, 0L) - skip_on_cran() expect_identical(.repro_canon(m), base[[key]], label = key) } } From 08fa3c926c3d17ec59e0d0814656c463227759f5 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 19:11:25 +1030 Subject: [PATCH 62/66] docs: NEWS 0.16.0 to one screen, with the F6 block fix and strict N3 --- NEWS.md | 147 ++++++++++++++++++++++---------------------------------- 1 file changed, 57 insertions(+), 90 deletions(-) diff --git a/NEWS.md b/NEWS.md index 67eef78..c31d076 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,105 +4,72 @@ * `mask()` synthesises a column that is constant within a design unit (a trial-level rainfall, a site soil type) once per unit, so the clone keeps - one value per trial. This changes the default clone of such tables. - `grain = FALSE` gives the row-by-row clone of 0.15.0. In collaborate mode - a design column with fewer than five units falls back to row-by-row - synthesis with a `masque_grain_small_units` warning. + one value per trial. `grain = FALSE` restores the row-by-row clone. In + collaborate mode, fewer than five units fall back to row-by-row with a + `masque_grain_small_units` warning. * Collaborate mode no longer clips jittered numbers to the observed range, - which copied the real minimum and maximum into the clone. A value can - now sit up to half a measurement step outside the range. + which copied the real extremes into the clone. * A numeric treatment (a dose, an N rate) can be aliased or permuted, and - collaborate mode aliases it by default (`trt_NNN`). `unmask()` and - `apply_recipe()` restore the exact numeric values. -* In collaborate mode an outcome left at `keep` audits HIGH and blocks the - package writers. `allow_kept_outcome = TRUE` allows it after review and - is recorded on the recipe. -* A collaborate clone of fewer than five rows that re-simulates any column - is refused (`masque_too_few_rows_refusal`). Local mode warns. -* `propose_roles()` promotes a column to treatment only when it is crossed - with the design, so a clinical `diagnosis` or a trial-level covariate - stays a covariate. `yield` and similar names are proposed as outcomes, - and `Site Name` or `trial_site` as design. -* In `mask()` and `mask_set()` the new arguments come before `quiet` (and - before `allow_unmasked_coords` in `mask()`), so pass those by name. -* masque needs data.table 1.15.0 or later. Returned tables are data.tables, - as before. -* `mask_set()` link aliases now follow `seed` whatever the caller's random - state, from their own stream derived from `seed`, so a set made with a - seed in 0.15.0 gets different link aliases when it is made again. -* A whole-numbered outcome (a count) is graded on its excess over chance: - an ordinary clone audits low instead of medium. + collaborate mode aliases it by default; `unmask()` restores the values. +* In collaborate mode a kept outcome audits HIGH and blocks the package + writers unless `allow_kept_outcome = TRUE`, which the recipe records. + A collaborate clone of fewer than five rows is refused. +* A coordinate declared in `coords` must have the action `keep` in `roles`. + Any other action, including the `drop` that `propose_roles()` proposes, + is refused (`masque_coords_action_refusal`) instead of being overridden. +* `propose_roles()` proposes a treatment only when it is crossed with the + design, `yield`-like names as outcomes, and `Site Name`-like names as + design. +* New arguments of `mask()` and `mask_set()` come before `quiet`; pass + `quiet` and `allow_unmasked_coords` by name. +* `mask_set()` link aliases follow `seed` from their own stream, so a seeded + set from 0.15.0 gets different link aliases when made again. +* masque needs data.table 1.15.0 or later. ## New features -* `data_card()` writes a data card for a clone, or one per table of a set: - size, design columns and per-column counts, each marked as exact in the - clone or re-simulated. It never lists a level label. -* `roles()` returns the role table of a clone under the clone's column - names. -* `mask()` gains `grain`, `events` (linked time-to-event outcomes kept in - order, with exact counts of each status pattern and no time past the - longest observed follow-up) and - `allow_kept_outcome`. `mask_set()` gains `grain`, `allow_kept_outcome`, - `sheets` and `info` (documentation sheets set aside, not masked). -* `masque()` takes `detect`, `sheets` and `info`, and passes other - arguments to `mask()` or `mask_set()`. An argument the verb does not take - is refused. -* `audit_mask()` grades each column on its excess over a chance baseline - and adds table-level checks (whole-row matches, rare combinations) in - `attr(audit, "joint")`. A column synthesised per unit, or a whole-numbered - covariate (age in years), is graded on that excess instead of on raw - exact matches, which it reaches by construction. -* `audit_mask()` grades such a column HIGH when it matches its own row or - design unit far above chance (50 points and 4 standard deviations), and a - continuous per-unit column HIGH when its unit values are the real ones, - even moved between units. With 5 to 8 units and few distinct values a - copy cannot be told from chance, and the audit grades it low. -* Every warning carries a `masque_` class, including - `masque_roles_upgraded`, `masque_roles_mode_mismatch`, - `masque_plot_skipped` and `masque_geo_anchor_missing`. +* `data_card()` describes a clone or a set: size, design, and per-column + counts marked exact or re-simulated. It never lists a level label. + `roles()` returns a clone's role table under the clone's column names. +* `mask()` gains `events`: linked time-to-event outcomes keep their order, + the exact count of each status pattern, and no time past the longest + observed follow-up. +* `mask_set()` and `masque()` gain `sheets` and `info` (documentation + sheets set aside, not masked). `masque()` takes `detect` and passes other + arguments on to `mask()` or `mask_set()`, refusing any they do not take. +* `audit_mask()` grades each column on its excess over chance and adds + whole-row and rare-combination checks in `attr(audit, "joint")`. A + per-unit column is HIGH when its unit values are the real ones, even moved + between units. With 5 to 8 units and few distinct values a copy cannot be + told from chance, and grades low. * `save_recipe()`, `read_recipe()`, `reveal_maps(table = )` and - `audit_mask()` work on a `mask_set()` result. -* The recipe records `conform_table()` decisions, and `apply_recipe()` - replays them on the original. + `audit_mask()` accept a `mask_set()` result. The recipe records + `conform_table()` decisions, and `apply_recipe()` replays them. +* Every warning carries a `masque_` class. ## Bug fixes -* `conform_table(merge_labels = "auto")` merged distinct genotype codes - (`G002` and `G011` into `G001`). Codes that differ only in a run of - digits are never merged now. A label whose merges end on two different - labels is left alone, with a warning. -* `conform_table()` ran a label containing braces as R code when printing - its report. Labels now print as text. -* `masque()` given a roles table made for collaborate mode masked in local - mode and returned real labels. It now uses the plan's mode. -* `print()`, `format()`, `str()` and knitr output of a recipe, a recipe - bundle, a `masque` or a `masque_set` no longer show the seed or the label - maps. -* The interactive review in `masque()` writes the plan and its menus as - messages, so `suppressMessages()` silences them. -* Guided `masque()` on a folder or workbook with column names that needed - repair stopped with an error. Set recipes now keep each table's cleaning, - and `apply_recipe()` keeps a linked key that the roles mark `drop`. -* A coordinate declared in `coords` whose action you set to `drop` or - `scramble` is refused (`masque_coords_action_refusal`). Before, the - declaration silently kept it. A single pair may be passed bare, and its - names must be `lat` and `lon` exactly. -* Whole-numbered columns stay whole in local conditional clones and in - collaborate mode, and a two-valued column is no longer jittered. -* Duplicate input column names are refused once, by name. The - near-duplicate report prints at most 20 pairs. -* `save_recipe()` accepts only a `.rds` path. A seed must be one whole - number, in `jitter_coordinates()` and `synthesise_geospatial()` too. -* A recipe saved by masque 0.15.0 still prints, applies, unmasks and - audits. -* `apply_recipe()` maps a numeric column that has an alias map instead of - passing real values through. -* The guided summary counts columns as the audit does, and the - non-interactive notice no longer suggests an argument already passed. -* The vignettes restore the `options()` they set. -* The design plot skips the replication panel with a warning, instead of - an error, when there is no block column. +* `print()`, `format()`, `str()` and knitr output of recipes and clones no + longer show the seed or the label maps. +* `masque()` used local mode with a collaborate plan and returned real + labels; it now follows the plan's mode. +* `conform_table(merge_labels = "auto")` no longer merges codes that differ + in a run of digits (`G002`, `G011`), and prints labels as text instead of + evaluating braces in them. +* `detect_design()` no longer names an environment column (a year) as the + block of a multi-environment trial. +* Guided `masque()` no longer stops on a folder or workbook whose column + names need repair, and `apply_recipe()` keeps a linked key marked `drop`. +* Whole-numbered columns stay whole in conditional and collaborate clones, + and two-valued columns are no longer jittered. +* `apply_recipe()` maps an aliased numeric column instead of passing real + values through. +* Duplicate input column names are refused. The near-duplicate report + prints at most 20 pairs. `save_recipe()` accepts only `.rds`; a seed must + be one whole number. +* Recipes saved by 0.15.0 still print, apply, unmask and audit. +* Console output of the guided review is written as messages, and the + vignettes restore the `options()` they set. # masque 0.15.0 From 01821793df7c510e785fc08eadee538447b75f65 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 19:16:44 +1030 Subject: [PATCH 63/66] docs: point the r-universe badge at its final URL --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 00aa6f6..04d36e8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Lifecycle: experimental](https://img.shields.io/badge/lifecycle-experimental-orange.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental) [![R-CMD-check](https://github.com/max578/masque/actions/workflows/R-CMD-check.yaml/badge.svg)](https://github.com/max578/masque/actions/workflows/R-CMD-check.yaml) [![Codecov test coverage](https://codecov.io/gh/max578/masque/graph/badge.svg)](https://app.codecov.io/gh/max578/masque) -[![r-universe](https://max578.r-universe.dev/badges/masque)](https://max578.r-universe.dev/masque) +[![r-universe](https://max578.r-universe.dev/masque/badges/version)](https://max578.r-universe.dev/masque) [![pkgdown](https://github.com/max578/masque/actions/workflows/pkgdown.yaml/badge.svg)](https://max578.github.io/masque/) From 6879fc10e51fb1fb1b31798e8502e6882fb06803 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Fri, 30 Oct 2026 19:29:21 +1030 Subject: [PATCH 64/66] docs: cran-comments lists the GitHub Actions platforms --- cran-comments.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cran-comments.md b/cran-comments.md index e27b016..98f29f9 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -24,8 +24,8 @@ multi-table round trip. `NEWS.md` lists them. ## Test environments * Local: macOS (Apple Silicon), R 4.6.1. -* [TO COMPLETE AFTER CI: GitHub Actions, R-hub and win-builder runs for - 0.16.0, with the R versions and platforms.] +* GitHub Actions: Ubuntu (R-devel, R-release, R-oldrel-1), Windows + (R-release), macOS (R-release). ## R CMD check results From d84b35f7cf9bc0dd7dd2aff842abdb6f746bcb39 Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Sat, 31 Oct 2026 09:43:17 +1030 Subject: [PATCH 65/66] docs: cran-comments lists the R-hub platforms --- cran-comments.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cran-comments.md b/cran-comments.md index 98f29f9..f6a61ca 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -26,6 +26,8 @@ multi-table round trip. `NEWS.md` lists them. * Local: macOS (Apple Silicon), R 4.6.1. * GitHub Actions: Ubuntu (R-devel, R-release, R-oldrel-1), Windows (R-release), macOS (R-release). +* R-hub: Linux (R-devel), Ubuntu next, Windows (R-devel), macOS arm64 + (R-devel). ## R CMD check results From 84ed3af633cba3e88da18dc20981fe7b32421c8e Mon Sep 17 00:00:00 2001 From: Max Moldovan Date: Sat, 31 Oct 2026 10:04:17 +1030 Subject: [PATCH 66/66] docs: cran-comments lists the win-builder run --- cran-comments.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cran-comments.md b/cran-comments.md index f6a61ca..75844c0 100644 --- a/cran-comments.md +++ b/cran-comments.md @@ -28,6 +28,7 @@ multi-table round trip. `NEWS.md` lists them. (R-release), macOS (R-release). * R-hub: Linux (R-devel), Ubuntu next, Windows (R-devel), macOS arm64 (R-devel). +* win-builder: Windows, R-devel (2026-10-28 r90783 ucrt). ## R CMD check results