diff --git a/NEWS.md b/NEWS.md index a8d1eed..e26e8e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # ROMOPMappingTools (development version) - Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. -- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceConceptId; a warning is given if it is empty for APPROVED or UNCHECKED rows. - `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. - Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 60d49fc..bf2e2d4 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -16,7 +16,7 @@ #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary #' If usagi file has the ADD_INFO:sourceName_fi column: #' - Check if sourceName_fi is less than 1000 characters -#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Check if sourceName_fi is the same for all the rows of a sourceConceptId #' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate @@ -364,27 +364,30 @@ validateUsagiFile <- function( usagiTibble <- result$fileTibble validationLogR6 <- result$validationLogR6 - # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) - sourceCodesWithDifferentNameFi <- usagiTibble |> - dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> - dplyr::count(sourceCode) |> + # check all rows of a sourceConceptId have the same sourceName_fi (multi-mapped codes repeat the source row), + # each sourceConceptId gets one Finnish synonym. A sourceCode can have several sourceConceptIds, valid for + # different periods, each with its own name, so the check is not done per sourceCode + sourceConceptIdsWithDifferentNameFi <- usagiTibble |> + dplyr::filter(!is.na(`ADD_INFO:sourceConceptId`)) |> + dplyr::distinct(`ADD_INFO:sourceConceptId`, `ADD_INFO:sourceName_fi`) |> + dplyr::count(`ADD_INFO:sourceConceptId`) |> dplyr::filter(n > 1) |> - dplyr::pull(sourceCode) + dplyr::pull(`ADD_INFO:sourceConceptId`) - if (length(sourceCodesWithDifferentNameFi) > 0) { + if (length(sourceConceptIdsWithDifferentNameFi) > 0) { validationLogR6$ERROR( - "SourceNameFi is not the same for all rows of the sourceCode", - paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + "SourceNameFi is not the same for all rows of the sourceConceptId", + paste0("Found ", length(sourceConceptIdsWithDifferentNameFi), " sourceConceptIds with different sourceName_fi") ) usagiTibble <- usagiTibble |> dplyr::mutate(tmpvalidationMessages = dplyr::if_else( - sourceCode %in% sourceCodesWithDifferentNameFi, - paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + `ADD_INFO:sourceConceptId` %in% sourceConceptIdsWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceConceptId"), tmpvalidationMessages )) } else { - validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceConceptId", "") } # warn if sourceName_fi is empty, the column is optional so this is not an error diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index a1f9363..a8b5ad7 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -39,7 +39,7 @@ If usagi file has C&CR columns: \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary If usagi file has the ADD_INFO:sourceName_fi column: \item Check if sourceName_fi is less than 1000 characters -\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Check if sourceName_fi is the same for all the rows of a sourceConceptId \item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md deleted file mode 100644 index 0291d04..0000000 --- a/plan_add_finnish_names.md +++ /dev/null @@ -1,104 +0,0 @@ -# Finnish names from Usagi files to `CONCEPT_SYNONYM` - -The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. - -## Decisions -- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. -- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. -- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. -- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. -- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. -- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. -- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. - -## Data flow - -``` -Usagi CSV STCM (extended) CDM -ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM - concept_id = source_concept_id - concept_synonym_name = source_code_description_fi - language_concept_id = 4181730 (Finnish) -``` - -- The language is hardcoded to Finnish (`4181730`). -- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. -- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). - -## Changes by step - -### 1. Reading and writing Usagi files (`R/usagiFile.R`) -- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. -- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. - -### 2. Validation (`R/validateUsagiFile.R`) -- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. -- New checks: - -| Step in the log | Level | Rule | -|---|---|---| -| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | -| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | -| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | - -- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. - -### 3. STCM table -- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. -- `R/appendUsagiFileToSTCMtable.R`: - - The column is part of the extended STCM columns. - - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. -- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. - -### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) -- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. -- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: - - `source_concept_id = 0` - - the name is NULL or blank - - the name equals `source_code_description` -- Re-running it gives the same result. - -### 5. Post-load check (`R/buildVocabuliesAll.R`) -- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: - - the expected names in the STCM table, using the same filters as the SQL; - - the Finnish rows in `CONCEPT_SYNONYM`. -- It logs SUCCESS if the counts match and ERROR otherwise. - -### 6. Other -- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. -- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. - -## Tests -- `test-validateUsagiFile.R`: - - The too-long, differing-within-sourceCode and empty checks each fire. - - The C&CR columns are required when `sourceName_fi` is present. -- `test-appendUsagiFileToSTCMTable.R`: - - The new column appears in the extended STCM table and gets its value. - - It is NA when the Usagi file has no Finnish column. -- `test-STCMToCDMTable.R`: - - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. - - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. - - Running it twice gives the same result. - - An existing English synonym is preserved. -- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. -- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. -- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. - -Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. - -## Files changed -- `R/`: - - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` - - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` -- `inst/`: - - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` - - `sql/sql_server/STCMExtendedToCDM.sql` - - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` - - `testdata/createTestData.R` -- `man/`: - - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` - - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` -- `tests/testthat/`: - - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` - - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` -- `NEWS.md` diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index fcb4111..7127132 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -193,8 +193,9 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { # SourceConceptId is not unique validationsSummary |> dplyr::filter(step == "SourceConceptId is not unique") |> nrow() |> expect_equal(1) validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> nrow() |> expect_equal(1) - validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal("ERROR: SourceConceptId is not unique") + # the row shares sourceConceptId with another sourceCode that has a different sourceName_fi + validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceConceptId is not unique | ERROR: SourceNameFi is not the same for all rows of the sourceConceptId") validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") @@ -324,7 +325,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { sourceCode == "A01.0+G01" ~ strrep("a", 1001), # empty sourceCode == "A01.0+I39.8" ~ NA_character_, - # different within a multi-mapped sourceCode + # different within a multi-mapped sourceCode, all rows share the same sourceConceptId sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", TRUE ~ `ADD_INFO:sourceName_fi` ) @@ -352,7 +353,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> expect_setequal(c( "SourceNameFi is more than 1000 characters", - "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is not the same for all rows of the sourceConceptId", "SourceNameFi is empty" )) @@ -362,10 +363,10 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { expect_equal("ERROR: SourceNameFi is more than 1000 characters") validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") - # SourceNameFi is not the same for all rows of the sourceCode - validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + # SourceNameFi is not the same for all rows of the sourceConceptId + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceConceptId", 2)) validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) # SourceNameFi is empty, only a warning, mappingStatus is not changed @@ -395,3 +396,30 @@ test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") }) + +test_that("test validateUsagiFile allows different ADD_INFO:sourceName_fi for different sourceConceptIds of the same sourceCode", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + # a sourceCode with two sourceConceptIds valid in different periods, each with its own names + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + secondRow <- which(usagiTibble$sourceCode == "A02.2+H22.0")[2] + usagiTibble$`ADD_INFO:sourceConceptId`[secondRow] <- 2000599999 + usagiTibble$sourceName[secondRow] <- "Other name for a later period" + usagiTibble$`ADD_INFO:sourceName_fi`[secondRow] <- "Toinen nimi myöhemmälle jaksolle" + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> + dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + validationsSummary |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) +})