diff --git a/NEWS.md b/NEWS.md index 4b57762..a8d1eed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# ROMOPMappingTools (development version) + +- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. +- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. + # ROMOPMappingTools 2.1.3 - Added rule to validate usagi file: sourceConceptId must be unique diff --git a/R/appendUsagiFileToSTCMtable.R b/R/appendUsagiFileToSTCMtable.R index 8b2aacb..3072e82 100644 --- a/R/appendUsagiFileToSTCMtable.R +++ b/R/appendUsagiFileToSTCMtable.R @@ -12,11 +12,13 @@ #' - source_concept_class #' - source_domain #' - source_parents_concept_ids +#' - source_code_description_fi #' #' Extended Usagi columns: #' - ADD_INFO:sourceConceptId #' - ADD_INFO:sourceConceptClass #' - ADD_INFO:sourceDomain +#' - ADD_INFO:sourceName_fi (optional, NA if missing) #' #' If formats don't match (e.g., extended Usagi with default STCM), the extended information #' will be ignored with a warning. @@ -83,7 +85,8 @@ appendUsagiFileToSTCMtable <- function( stcmExtendedColumns <- c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", - "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids" + "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids", + "source_code_description_fi" ) if (all(stcmExtendedColumns %in% stcmTableColumns)) { @@ -150,6 +153,10 @@ appendUsagiFileToSTCMtable <- function( if (!"ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceValidEndDate` = lubridate::as_date(ymd("2099-12-31"))) } + # if colum `ADD_INFO:sourceName_fi` is not present, set it to NA + if (!"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceName_fi` = NA_character_) + } # if colums `ADD_INFO:sourceParents` and `ADD_INFO:sourceParentVocabulary` are present and any of the `ADD_INFO:sourceParents` is not NA if (all(c("ADD_INFO:sourceParents", "ADD_INFO:sourceParentVocabulary") %in% usagiTibbleColumns) && @@ -227,7 +234,8 @@ appendUsagiFileToSTCMtable <- function( invalid_reason = NA_character_, source_concept_class = `ADD_INFO:sourceConceptClass`, source_domain = `ADD_INFO:sourceDomain`, - source_parents_concept_ids = sourceParentConceptIds + source_parents_concept_ids = sourceParentConceptIds, + source_code_description_fi = `ADD_INFO:sourceName_fi` ) } diff --git a/R/buildVocabuliesAll.R b/R/buildVocabuliesAll.R index d5b1344..f566fd3 100644 --- a/R/buildVocabuliesAll.R +++ b/R/buildVocabuliesAll.R @@ -111,6 +111,48 @@ buildVocabulariesAll <- function( return(validationLogTibble) } + # check all the Finnish names in the SourceToConceptMap table are in the CONCEPT_SYNONYM table + nFinnishNamesInSTCM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM ( + SELECT DISTINCT source_concept_id, source_code_description_fi + FROM @vocabulary_database_schema.@source_to_concept_map_table + WHERE source_concept_id != 0 + AND source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(source_code_description_fi)) != '' + AND (source_code_description IS NULL OR source_code_description_fi != source_code_description) + ) AS t", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + nFinnishSynonymsInCDM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM @vocabulary_database_schema.CONCEPT_SYNONYM + WHERE language_concept_id = 4181730 + AND concept_id IN (SELECT DISTINCT source_concept_id FROM @vocabulary_database_schema.@source_to_concept_map_table)", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + + if (nFinnishNamesInSTCM != nFinnishSynonymsInCDM) { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "ERROR", + step = "Finnish names in CONCEPT_SYNONYM", + message = paste0( + "Found ", nFinnishSynonymsInCDM, " Finnish synonyms in CONCEPT_SYNONYM but ", + nFinnishNamesInSTCM, " Finnish names in the SourceToConceptMap table" + ) + )) + } else { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "SUCCESS", + step = "Finnish names in CONCEPT_SYNONYM", + message = "" + )) + } + # create the ancestor tables message("Creating the ancestor tables") errorMessage <- "" diff --git a/R/calculateMappingStatus.R b/R/calculateMappingStatus.R index acee366..8294e81 100644 --- a/R/calculateMappingStatus.R +++ b/R/calculateMappingStatus.R @@ -90,7 +90,7 @@ calculateMappingStatus <- function( ON c.concept_id = cr.concept_id_1 LEFT JOIN ( SELECT concept_id, concept_synonym_name FROM @vocabulary_database_schema.CONCEPT_SYNONYM - WHERE language_concept_id == 4181730 + WHERE language_concept_id = 4181730 ) AS cs ON c.concept_id = cs.concept_id " diff --git a/R/createSourceToConceptMapExtended.R b/R/createSourceToConceptMapExtended.R index d20a236..4ba52dc 100644 --- a/R/createSourceToConceptMapExtended.R +++ b/R/createSourceToConceptMapExtended.R @@ -1,7 +1,7 @@ #' Create Extended Source to Concept Map Table #' #' Creates an extended version of the Source to Concept Map table that includes additional columns -#' for source concept class, domain, and parent relationships. +#' for source concept class, domain, parent relationships, and the Finnish source code description. #' #' @param connection A DatabaseConnector connection object #' @param vocabularyDatabaseSchema Schema name where the vocabulary tables are stored diff --git a/R/usagiFile.R b/R/usagiFile.R index 5c2c0ff..c5cb850 100644 --- a/R/usagiFile.R +++ b/R/usagiFile.R @@ -30,6 +30,7 @@ #' - sourceValidEndDate (date): End date of validity #' - sourceParents (character): Parent concepts #' - sourceParentVocabulary (character): Vocabulary of parent concepts +#' - sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM #' - validationMessagess (character): Validation messages #' #' @param pathToUsagiFile Path to the Usagi mapping file to read @@ -69,6 +70,7 @@ readUsagiFile <- function(pathToUsagiFile) { `ADD_INFO:sourceValidEndDate` = readr::col_date(), `ADD_INFO:sourceParents` = readr::col_character(), `ADD_INFO:sourceParentVocabulary` = readr::col_character(), + `ADD_INFO:sourceName_fi` = readr::col_character(), `ADD_INFO:validationMessages` = readr::col_character(), # .default = readr::col_character() @@ -110,6 +112,7 @@ readUsagiFile <- function(pathToUsagiFile) { #' - ADD_INFO:sourceValidEndDate (date): Source valid end date #' - ADD_INFO:sourceParents (character): Source parents #' - ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +#' - ADD_INFO:sourceName_fi (character): Source name in Finnish #' - ADD_INFO:validationMessages (character): Validation messages #' - ADD_INFO:autoUpdatingInfo (character): Auto updating info #' @@ -147,7 +150,8 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { "ADD_INFO:sourceValidStartDate", "ADD_INFO:sourceValidEndDate", "ADD_INFO:sourceParents", - "ADD_INFO:sourceParentVocabulary" + "ADD_INFO:sourceParentVocabulary", + "ADD_INFO:sourceName_fi" ) lastColNames <- c( @@ -164,7 +168,7 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { dplyr::any_of(c(firstColNames, midColNames, lastColNames)) ) - # force "ADD_INFO:validationMessages" and "ADD_INFO:autoUpdatingInfo" to be NA if they are empty, if they exist + # force "ADD_INFO:validationMessages", "ADD_INFO:autoUpdatingInfo" and "ADD_INFO:sourceName_fi" to be NA if they are empty, if they exist if ("ADD_INFO:validationMessages" %in% colNames) { usagiTibble <- usagiTibble |> dplyr::mutate( @@ -177,6 +181,12 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { `ADD_INFO:autoUpdatingInfo` = ifelse(`ADD_INFO:autoUpdatingInfo` == "", NA, `ADD_INFO:autoUpdatingInfo`) ) } + if ("ADD_INFO:sourceName_fi" %in% colNames) { + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = ifelse(`ADD_INFO:sourceName_fi` == "", NA, `ADD_INFO:sourceName_fi`) + ) + } readr::write_csv(usagiTibble, pathToUsagiFile, na = "") } diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 13ba240..60d49fc 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -14,6 +14,10 @@ #' - Check codes with mapping to more than one domain are mapped to compatible domains #' - Check if sourceValidStartDate is before sourceValidEndDate #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +#' If usagi file has the ADD_INFO:sourceName_fi column: +#' - Check if sourceName_fi is less than 1000 characters +#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate #' @param connection A DatabaseConnector connection object @@ -202,7 +206,8 @@ validateUsagiFile <- function( # if it has any of the 3 columns it must have the other 2 if ("ADD_INFO:sourceConceptId" %in% usagiTibbleColumns || "ADD_INFO:sourceConceptClass" %in% usagiTibbleColumns || "ADD_INFO:sourceDomain" %in% usagiTibbleColumns || "ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns || - "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns) { + "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns || + "ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { missingColumns <- c( "ADD_INFO:sourceConceptId", @@ -347,6 +352,61 @@ validateUsagiFile <- function( } } + # - Check ADD_INFO:sourceName_fi, it will be used as Finnish CONCEPT_SYNONYM + # (whitespace-only names are read as NA by readUsagiFile, so they are reported as empty) + if ("ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + validationRules <- validate::validator( + SourceNameFi.is.more.than.1000.characters = field_length(`ADD_INFO:sourceName_fi`, min = 0, max = 1000) + ) + validations <- validate::confront(usagiTibble, validationRules) + + result <- .applyValidationRules(usagiTibble, validations, validationLogR6) + usagiTibble <- result$fileTibble + validationLogR6 <- result$validationLogR6 + + # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) + sourceCodesWithDifferentNameFi <- usagiTibble |> + dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> + dplyr::count(sourceCode) |> + dplyr::filter(n > 1) |> + dplyr::pull(sourceCode) + + if (length(sourceCodesWithDifferentNameFi) > 0) { + validationLogR6$ERROR( + "SourceNameFi is not the same for all rows of the sourceCode", + paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + sourceCode %in% sourceCodesWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + } + + # warn if sourceName_fi is empty, the column is optional so this is not an error + isEmptyNameFi <- usagiTibble$mappingStatus %in% c("APPROVED", "UNCHECKED") & is.na(usagiTibble$`ADD_INFO:sourceName_fi`) + + if (any(isEmptyNameFi)) { + validationLogR6$WARNING( + "SourceNameFi is empty", + paste0("Found ", sum(isEmptyNameFi), " APPROVED or UNCHECKED rows with empty sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + isEmptyNameFi, + paste0(tmpvalidationMessages, " | ", "WARNING: SourceNameFi is empty"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is empty", "") + } + } + # - Check if ADD_INFO:sourceValidStartDate and ADD_INFO:sourceValidEndDate are used if ("ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { missingColumns <- diff --git a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql index 1693a3a..e964d5a 100644 --- a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql +++ b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql @@ -18,5 +18,6 @@ CREATE TABLE @vocabularyDatabaseSchema.@sourceToConceptMapTable ( invalid_reason varchar(1) NULL , source_concept_class varchar(20) NULL, source_domain varchar(20) NULL, - source_parents_concept_ids varchar(255) NULL + source_parents_concept_ids varchar(255) NULL, + source_code_description_fi varchar(1000) NULL ); \ No newline at end of file diff --git a/inst/sql/sql_server/STCMExtendedToCDM.sql b/inst/sql/sql_server/STCMExtendedToCDM.sql index c2328bf..f011e88 100644 --- a/inst/sql/sql_server/STCMExtendedToCDM.sql +++ b/inst/sql/sql_server/STCMExtendedToCDM.sql @@ -300,4 +300,42 @@ SELECT DISTINCT ) AS valid_end_date, NULL AS invalid_reason FROM split_parents AS sp -WHERE sp.source_parents_concept_ids IS NOT NULL +WHERE sp.source_parents_concept_ids IS NOT NULL; + + +-- 5. CONCEPT_SYNONYM table +-- Only Finnish synonyms (language_concept_id = 4181730) are managed here, synonyms in other languages are not touched +-- Delete previous Finnish rows with the same concept_id as in the source_to_concept_map.source_concept_id +-- Insert one row for each **unique** source_concept_id with a non-empty source_code_description_fi +-- that is different from source_code_description (same name as concept_name adds nothing as synonym) +-- - concept_id = source_to_concept_map.source_concept_id +-- - concept_synonym_name = source_to_concept_map.source_code_description_fi +-- - language_concept_id = 4181730 (Finnish) +DELETE FROM + @vocabularyDatabaseSchema.CONCEPT_SYNONYM +WHERE + language_concept_id = 4181730 + AND concept_id IN ( + SELECT + DISTINCT stcm.source_concept_id + FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm + ); + +INSERT INTO + @vocabularyDatabaseSchema.CONCEPT_SYNONYM ( + concept_id, + concept_synonym_name, + language_concept_id + ) +SELECT DISTINCT + stcm.source_concept_id AS concept_id, + stcm.source_code_description_fi AS concept_synonym_name, + 4181730 AS language_concept_id +FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm +WHERE + stcm.source_concept_id != 0 + AND stcm.source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(stcm.source_code_description_fi)) != '' + AND (stcm.source_code_description IS NULL OR stcm.source_code_description_fi != stcm.source_code_description); diff --git a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb index 9e8753d..61e9209 100644 Binary files a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb and b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb differ diff --git a/inst/testdata/createTestData.R b/inst/testdata/createTestData.R index fc964da..1ff72b6 100644 --- a/inst/testdata/createTestData.R +++ b/inst/testdata/createTestData.R @@ -42,7 +42,8 @@ allUsagiFiles <- dplyr::bind_rows(ICD10fiUsagiFile, UNITfiUsagiFile, ICD10fiUsag conceptIdsToMap <- allUsagiFiles |> dplyr::pull(conceptId) |> unique() # add extra conceptIds to map -conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219) +# 4181730 is the Finnish language concept, used as language_concept_id in CONCEPT_SYNONYM +conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219, 4181730) # For each table we have to add also the concepts to the concept table diff --git a/man/appendUsagiFileToSTCMtable.Rd b/man/appendUsagiFileToSTCMtable.Rd index c2503ea..1281765 100644 --- a/man/appendUsagiFileToSTCMtable.Rd +++ b/man/appendUsagiFileToSTCMtable.Rd @@ -57,6 +57,7 @@ Extended STCM columns (includes all default columns plus): \item source_concept_class \item source_domain \item source_parents_concept_ids +\item source_code_description_fi } Extended Usagi columns: @@ -64,6 +65,7 @@ Extended Usagi columns: \item ADD_INFO:sourceConceptId \item ADD_INFO:sourceConceptClass \item ADD_INFO:sourceDomain +\item ADD_INFO:sourceName_fi (optional, NA if missing) } If formats don't match (e.g., extended Usagi with default STCM), the extended information diff --git a/man/createSourceToConceptMapExtended.Rd b/man/createSourceToConceptMapExtended.Rd index 14ae818..2e4b035 100644 --- a/man/createSourceToConceptMapExtended.Rd +++ b/man/createSourceToConceptMapExtended.Rd @@ -22,5 +22,5 @@ createSourceToConceptMapExtended( } \description{ Creates an extended version of the Source to Concept Map table that includes additional columns -for source concept class, domain, and parent relationships. +for source concept class, domain, parent relationships, and the Finnish source code description. } diff --git a/man/readUsagiFile.Rd b/man/readUsagiFile.Rd index cc63ab9..796130a 100644 --- a/man/readUsagiFile.Rd +++ b/man/readUsagiFile.Rd @@ -47,6 +47,7 @@ Optional ADD_INFO columns: \item sourceValidEndDate (date): End date of validity \item sourceParents (character): Parent concepts \item sourceParentVocabulary (character): Vocabulary of parent concepts +\item sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM \item validationMessagess (character): Validation messages } } diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index 06f74cc..a1f9363 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -37,5 +37,9 @@ If usagi file has C&CR columns: \item Check codes with mapping to more than one domain are mapped to compatible domains \item Check if sourceValidStartDate is before sourceValidEndDate \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +If usagi file has the ADD_INFO:sourceName_fi column: +\item Check if sourceName_fi is less than 1000 characters +\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/man/writeUsagiFile.Rd b/man/writeUsagiFile.Rd index 9441d51..7a941dd 100644 --- a/man/writeUsagiFile.Rd +++ b/man/writeUsagiFile.Rd @@ -44,6 +44,7 @@ The function writes a Usagi mapping file with the following columns: \item ADD_INFO:sourceValidEndDate (date): Source valid end date \item ADD_INFO:sourceParents (character): Source parents \item ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +\item ADD_INFO:sourceName_fi (character): Source name in Finnish \item ADD_INFO:validationMessages (character): Validation messages \item ADD_INFO:autoUpdatingInfo (character): Auto updating info } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md new file mode 100644 index 0000000..0291d04 --- /dev/null +++ b/plan_add_finnish_names.md @@ -0,0 +1,104 @@ +# Finnish names from Usagi files to `CONCEPT_SYNONYM` + +The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. + +## Decisions +- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. +- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. +- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. +- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. +- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. +- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. +- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. + +## Data flow + +``` +Usagi CSV STCM (extended) CDM +ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM + concept_id = source_concept_id + concept_synonym_name = source_code_description_fi + language_concept_id = 4181730 (Finnish) +``` + +- The language is hardcoded to Finnish (`4181730`). +- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. +- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). + +## Changes by step + +### 1. Reading and writing Usagi files (`R/usagiFile.R`) +- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. +- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. + +### 2. Validation (`R/validateUsagiFile.R`) +- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. +- New checks: + +| Step in the log | Level | Rule | +|---|---|---| +| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | +| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | +| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | + +- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. + +### 3. STCM table +- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. +- `R/appendUsagiFileToSTCMtable.R`: + - The column is part of the extended STCM columns. + - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. +- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. + +### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) +- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. +- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: + - `source_concept_id = 0` + - the name is NULL or blank + - the name equals `source_code_description` +- Re-running it gives the same result. + +### 5. Post-load check (`R/buildVocabuliesAll.R`) +- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: + - the expected names in the STCM table, using the same filters as the SQL; + - the Finnish rows in `CONCEPT_SYNONYM`. +- It logs SUCCESS if the counts match and ERROR otherwise. + +### 6. Other +- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. +- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. + +## Tests +- `test-validateUsagiFile.R`: + - The too-long, differing-within-sourceCode and empty checks each fire. + - The C&CR columns are required when `sourceName_fi` is present. +- `test-appendUsagiFileToSTCMTable.R`: + - The new column appears in the extended STCM table and gets its value. + - It is NA when the Usagi file has no Finnish column. +- `test-STCMToCDMTable.R`: + - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. + - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. + - Running it twice gives the same result. + - An existing English synonym is preserved. +- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. +- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. +- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. + +Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. + +## Files changed +- `R/`: + - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` + - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` +- `inst/`: + - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` + - `sql/sql_server/STCMExtendedToCDM.sql` + - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` + - `testdata/createTestData.R` +- `man/`: + - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` + - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` +- `tests/testthat/`: + - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` + - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` +- `NEWS.md` diff --git a/tests/testthat/test-STCMToCDMTable.R b/tests/testthat/test-STCMToCDMTable.R index 6584227..4042d07 100644 --- a/tests/testthat/test-STCMToCDMTable.R +++ b/tests/testthat/test-STCMToCDMTable.R @@ -25,6 +25,7 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre "code9", 2000000009, "TestVocab", "Test Code 9 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002", "code10", 2000000010, "TestVocab", "Test Code 10 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002|2000000003" ) + testData <- testData |> dplyr::mutate(source_code_description_fi = NA_character_) # Create connection to test database connection <- DatabaseConnector::connect( @@ -113,3 +114,80 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre res |> dplyr::pull(concept_id_2) |> expect_equal(c(2000000001, 2000000001, 2000000002, 2000000001, 2000000002, 2000000003)) }) + +test_that("STCMToCDMTables creates Finnish CONCEPT_SYNONYM entries from source_code_description_fi", { + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + sourceToConceptMapTable <- "source_to_concept_map_extended" + + testData <- tibble::tribble( + ~source_code, ~source_concept_id, ~source_vocabulary_id, ~source_code_description, ~target_concept_id, ~target_vocabulary_id, ~valid_start_date, ~valid_end_date, ~invalid_reason, ~source_concept_class, ~source_domain, ~source_parents_concept_ids, ~source_code_description_fi, + # multi-mapped code, one synonym + "code1", 2000000001, "TestVocab", "Test Code 1", 141797, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + "code1", 2000000001, "TestVocab", "Test Code 1", 36713461, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + # unmapped code also gets synonym + "code2", 2000000002, "TestVocab", "Test Code 2", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 2", + # same name as concept_name is not added as synonym + "code3", 2000000003, "TestVocab", "Test Code 3", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Test Code 3", + # no synonym + "code4", 2000000004, "TestVocab", "Test Code 4 no fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, NA_character_, + "code5", 2000000005, "TestVocab", "Test Code 5 blank fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, " ", + # source_concept_id 0 is skipped + "code6", 0, "TestVocab", "Test Code 6 no source concept id", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 6" + ) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + createSourceToConceptMapExtended( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable, + overwrite = TRUE + ) + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = sourceToConceptMapTable, + data = testData + ) + + # existing synonyms: an old Finnish one to be replaced and an English one to be kept + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = "CONCEPT_SYNONYM", + data = tibble::tribble( + ~concept_id, ~concept_synonym_name, ~language_concept_id, + 2000000001, "Vanha nimi", 4181730, + 2000000001, "Old English synonym", 4180186 + ), + dropTableIfExists = FALSE, + createTable = FALSE + ) + + # run twice to check it is idempotent + for (i in 1:2) { + STCMToCDMTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + } + + res <- dplyr::tbl(connection, "CONCEPT_SYNONYM") |> + dplyr::filter(concept_id >= 2000000000 | concept_id == 0) |> + dplyr::arrange(concept_id, language_concept_id) |> + dplyr::collect() + + finnish <- res |> dplyr::filter(language_concept_id == 4181730) + finnish |> dplyr::pull(concept_id) |> expect_equal(c(2000000001, 2000000002)) + finnish |> dplyr::pull(concept_synonym_name) |> expect_equal(c("Testikoodi 1", "Testikoodi 2")) + + # other languages are not touched + res |> dplyr::filter(language_concept_id != 4181730) |> dplyr::pull(concept_synonym_name) |> expect_equal("Old English synonym") +}) diff --git a/tests/testthat/test-appendUsagiFileToSTCMTable.R b/tests/testthat/test-appendUsagiFileToSTCMTable.R index 6749c9c..2fb6846 100644 --- a/tests/testthat/test-appendUsagiFileToSTCMTable.R +++ b/tests/testthat/test-appendUsagiFileToSTCMTable.R @@ -81,13 +81,19 @@ test_that("test appendUsagiFileToSTCMTable appends the usagi file to the sourceT expect_equal(c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", "invalid_reason", "source_concept_class", - "source_domain", "source_parents_concept_ids")) + "source_domain", "source_parents_concept_ids", "source_code_description_fi")) stcmTable |> dplyr::filter(is.na(source_parents_concept_ids)) |> nrow() |> expect_equal(0) + # source_code_description_fi comes from ADD_INFO:sourceName_fi + stcmTable |> + dplyr::filter(source_code == "A01.0+G01") |> + dplyr::pull(source_code_description_fi) |> + expect_equal("Lavantautiin liittyvä aivokalvotulehdus") + }) test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the sourceToConceptMapTable with ICD10 parent information", { @@ -130,3 +136,34 @@ test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the expect_equal("2000503725") }) + +test_that("test appendUsagiFileToSTCMTable sets source_code_description_fi to NA when the usagi file has no ADD_INFO:sourceName_fi", { + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-`ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + sourceToConceptMapTable <- "source_to_concept_map_extended" + createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) + + appendUsagiFileToSTCMtable( + vocabularyId = "ICD10fi", + pathToUsagiFile = pathToUsagiFile, + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + + stcmTable <- DBI::dbReadTable(connection, sourceToConceptMapTable) |> tibble::as_tibble() + stcmTable |> nrow() |> expect_equal(nrow(usagiTibble)) + stcmTable |> dplyr::pull(source_code_description_fi) |> is.na() |> all() |> expect_true() +}) diff --git a/tests/testthat/test-buildVocabulariesAll.R b/tests/testthat/test-buildVocabulariesAll.R index c0fcf28..8e0416a 100644 --- a/tests/testthat/test-buildVocabulariesAll.R +++ b/tests/testthat/test-buildVocabulariesAll.R @@ -28,6 +28,9 @@ test_that("buildVocabulariesAll works", { # Check results validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) + # Finnish names were added to CONCEPT_SYNONYM + validationLogTibble |> dplyr::filter(step == "Finnish names in CONCEPT_SYNONYM") |> dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + # check the validation results folder expect_true(!file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) }) diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index 8a5d44b..fcb4111 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -306,3 +306,92 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { + +test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + pathToValidatedUsagiFile <- tempfile(fileext = ".csv") + vocabularyDatabaseSchema = "main" + sourceConceptIdOffset = 2000500000 + + # make a copy of a valid usagi file with errors in ADD_INFO:sourceName_fi + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = dplyr::case_when( + # too long + sourceCode == "A01.0+G01" ~ strrep("a", 1001), + # empty + sourceCode == "A01.0+I39.8" ~ NA_character_, + # different within a multi-mapped sourceCode + sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", + TRUE ~ `ADD_INFO:sourceName_fi` + ) + ) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile( + pathToUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile, + sourceConceptIdOffset + ) + + validatedUsagiFile <- readUsagiFile(pathToValidatedUsagiFile) + + # only the sourceName_fi validations fail + validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> + expect_setequal(c( + "SourceNameFi is more than 1000 characters", + "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is empty" + )) + + # SourceNameFi is more than 1000 characters + validationsSummary |> dplyr::filter(step == "SourceNameFi is more than 1000 characters") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceNameFi is more than 1000 characters") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") + + # SourceNameFi is not the same for all rows of the sourceCode + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) + + # SourceNameFi is empty, only a warning, mappingStatus is not changed + validationsSummary |> dplyr::filter(step == "SourceNameFi is empty") |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("WARNING: SourceNameFi is empty") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(mappingStatus) |> expect_equal("APPROVED") +}) + +test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName_fi is present", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-dplyr::starts_with("ADD_INFO:"), `ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") +}) diff --git a/tests/testthat/test-validateVocabularyFolder.R b/tests/testthat/test-validateVocabularyFolder.R index 597b028..95aa62b 100644 --- a/tests/testthat/test-validateVocabularyFolder.R +++ b/tests/testthat/test-validateVocabularyFolder.R @@ -23,7 +23,12 @@ test_that("validateVocabularyFolder returns all SUCCESS for valid vocabulary fol ) # Check validation results - validationsLogTibble |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) + # only warning: UNITfi has the ADD_INFO:sourceName_fi column but all values are empty + notSuccess <- validationsLogTibble |> dplyr::filter(type != "SUCCESS") + notSuccess |> nrow() |> expect_equal(1) + notSuccess |> dplyr::pull(context) |> as.character() |> expect_equal("UNITfi") + notSuccess |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + notSuccess |> dplyr::pull(step) |> expect_equal("SourceNameFi is empty") # Check that validated files were created expect_true(dir.exists(pathToValidatedVocabularyFolder))