From f50a629bcd57696653ec9134675d4e4a2d8e001a Mon Sep 17 00:00:00 2001 From: javier-gracia-tabuenca-tuni Date: Tue, 10 Nov 2026 13:59:43 +0200 Subject: [PATCH] Check sourceName_fi uniqueness per sourceConceptId, not per sourceCode A sourceCode can have several sourceConceptIds, each valid for a different period and with its own English and Finnish name (e.g. REIMB 307). The previous check flagged these valid files as ERROR, which stopped buildVocabulariesAll. Each sourceConceptId must now have one sourceName_fi, so each concept gets exactly one Finnish synonym. Co-Authored-By: Claude Opus 5.5 --- NEWS.md | 2 +- R/validateUsagiFile.R | 27 +++--- man/validateUsagiFile.Rd | 2 +- plan_add_finnish_names.md | 104 ------------------------ tests/testthat/test-validateUsagiFile.R | 42 ++++++++-- 5 files changed, 52 insertions(+), 125 deletions(-) delete mode 100644 plan_add_finnish_names.md diff --git a/NEWS.md b/NEWS.md index a8d1eed..e26e8e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # ROMOPMappingTools (development version) - Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. -- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceConceptId; a warning is given if it is empty for APPROVED or UNCHECKED rows. - `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. - Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 60d49fc..bf2e2d4 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -16,7 +16,7 @@ #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary #' If usagi file has the ADD_INFO:sourceName_fi column: #' - Check if sourceName_fi is less than 1000 characters -#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Check if sourceName_fi is the same for all the rows of a sourceConceptId #' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate @@ -364,27 +364,30 @@ validateUsagiFile <- function( usagiTibble <- result$fileTibble validationLogR6 <- result$validationLogR6 - # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) - sourceCodesWithDifferentNameFi <- usagiTibble |> - dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> - dplyr::count(sourceCode) |> + # check all rows of a sourceConceptId have the same sourceName_fi (multi-mapped codes repeat the source row), + # each sourceConceptId gets one Finnish synonym. A sourceCode can have several sourceConceptIds, valid for + # different periods, each with its own name, so the check is not done per sourceCode + sourceConceptIdsWithDifferentNameFi <- usagiTibble |> + dplyr::filter(!is.na(`ADD_INFO:sourceConceptId`)) |> + dplyr::distinct(`ADD_INFO:sourceConceptId`, `ADD_INFO:sourceName_fi`) |> + dplyr::count(`ADD_INFO:sourceConceptId`) |> dplyr::filter(n > 1) |> - dplyr::pull(sourceCode) + dplyr::pull(`ADD_INFO:sourceConceptId`) - if (length(sourceCodesWithDifferentNameFi) > 0) { + if (length(sourceConceptIdsWithDifferentNameFi) > 0) { validationLogR6$ERROR( - "SourceNameFi is not the same for all rows of the sourceCode", - paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + "SourceNameFi is not the same for all rows of the sourceConceptId", + paste0("Found ", length(sourceConceptIdsWithDifferentNameFi), " sourceConceptIds with different sourceName_fi") ) usagiTibble <- usagiTibble |> dplyr::mutate(tmpvalidationMessages = dplyr::if_else( - sourceCode %in% sourceCodesWithDifferentNameFi, - paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + `ADD_INFO:sourceConceptId` %in% sourceConceptIdsWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceConceptId"), tmpvalidationMessages )) } else { - validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceConceptId", "") } # warn if sourceName_fi is empty, the column is optional so this is not an error diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index a1f9363..a8b5ad7 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -39,7 +39,7 @@ If usagi file has C&CR columns: \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary If usagi file has the ADD_INFO:sourceName_fi column: \item Check if sourceName_fi is less than 1000 characters -\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Check if sourceName_fi is the same for all the rows of a sourceConceptId \item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md deleted file mode 100644 index 0291d04..0000000 --- a/plan_add_finnish_names.md +++ /dev/null @@ -1,104 +0,0 @@ -# Finnish names from Usagi files to `CONCEPT_SYNONYM` - -The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. - -## Decisions -- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. -- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. -- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. -- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. -- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. -- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. -- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. - -## Data flow - -``` -Usagi CSV STCM (extended) CDM -ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM - concept_id = source_concept_id - concept_synonym_name = source_code_description_fi - language_concept_id = 4181730 (Finnish) -``` - -- The language is hardcoded to Finnish (`4181730`). -- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. -- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). - -## Changes by step - -### 1. Reading and writing Usagi files (`R/usagiFile.R`) -- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. -- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. - -### 2. Validation (`R/validateUsagiFile.R`) -- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. -- New checks: - -| Step in the log | Level | Rule | -|---|---|---| -| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | -| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | -| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | - -- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. - -### 3. STCM table -- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. -- `R/appendUsagiFileToSTCMtable.R`: - - The column is part of the extended STCM columns. - - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. -- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. - -### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) -- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. -- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: - - `source_concept_id = 0` - - the name is NULL or blank - - the name equals `source_code_description` -- Re-running it gives the same result. - -### 5. Post-load check (`R/buildVocabuliesAll.R`) -- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: - - the expected names in the STCM table, using the same filters as the SQL; - - the Finnish rows in `CONCEPT_SYNONYM`. -- It logs SUCCESS if the counts match and ERROR otherwise. - -### 6. Other -- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. -- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. - -## Tests -- `test-validateUsagiFile.R`: - - The too-long, differing-within-sourceCode and empty checks each fire. - - The C&CR columns are required when `sourceName_fi` is present. -- `test-appendUsagiFileToSTCMTable.R`: - - The new column appears in the extended STCM table and gets its value. - - It is NA when the Usagi file has no Finnish column. -- `test-STCMToCDMTable.R`: - - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. - - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. - - Running it twice gives the same result. - - An existing English synonym is preserved. -- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. -- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. -- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. - -Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. - -## Files changed -- `R/`: - - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` - - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` -- `inst/`: - - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` - - `sql/sql_server/STCMExtendedToCDM.sql` - - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` - - `testdata/createTestData.R` -- `man/`: - - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` - - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` -- `tests/testthat/`: - - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` - - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` -- `NEWS.md` diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index fcb4111..7127132 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -193,8 +193,9 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { # SourceConceptId is not unique validationsSummary |> dplyr::filter(step == "SourceConceptId is not unique") |> nrow() |> expect_equal(1) validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> nrow() |> expect_equal(1) - validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal("ERROR: SourceConceptId is not unique") + # the row shares sourceConceptId with another sourceCode that has a different sourceName_fi + validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceConceptId is not unique | ERROR: SourceNameFi is not the same for all rows of the sourceConceptId") validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") @@ -324,7 +325,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { sourceCode == "A01.0+G01" ~ strrep("a", 1001), # empty sourceCode == "A01.0+I39.8" ~ NA_character_, - # different within a multi-mapped sourceCode + # different within a multi-mapped sourceCode, all rows share the same sourceConceptId sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", TRUE ~ `ADD_INFO:sourceName_fi` ) @@ -352,7 +353,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> expect_setequal(c( "SourceNameFi is more than 1000 characters", - "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is not the same for all rows of the sourceConceptId", "SourceNameFi is empty" )) @@ -362,10 +363,10 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { expect_equal("ERROR: SourceNameFi is more than 1000 characters") validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") - # SourceNameFi is not the same for all rows of the sourceCode - validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + # SourceNameFi is not the same for all rows of the sourceConceptId + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceConceptId", 2)) validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) # SourceNameFi is empty, only a warning, mappingStatus is not changed @@ -395,3 +396,30 @@ test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") }) + +test_that("test validateUsagiFile allows different ADD_INFO:sourceName_fi for different sourceConceptIds of the same sourceCode", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + # a sourceCode with two sourceConceptIds valid in different periods, each with its own names + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + secondRow <- which(usagiTibble$sourceCode == "A02.2+H22.0")[2] + usagiTibble$`ADD_INFO:sourceConceptId`[secondRow] <- 2000599999 + usagiTibble$sourceName[secondRow] <- "Other name for a later period" + usagiTibble$`ADD_INFO:sourceName_fi`[secondRow] <- "Toinen nimi myöhemmälle jaksolle" + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> + dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + validationsSummary |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) +})