diff --git a/DESCRIPTION b/DESCRIPTION index 3dced7b..ff07bde 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: ROMOPMappingTools Title: Tools for Working with OMOP CDM Mappings -Version: 2.1.3 +Version: 2.2.0 Authors@R: person("Javier", "Gracia-Tabuenca", , "javier.graciatabuenca@tuni.fi", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-2455-0598")) diff --git a/NEWS.md b/NEWS.md index 4b57762..fdcf4a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# ROMOPMappingTools 2.2.0 + +- Added `ADD_INFO:sourceName_fi` Usagi column, validated and loaded into `CONCEPT_SYNONYM` as Finnish synonyms. + # ROMOPMappingTools 2.1.3 - Added rule to validate usagi file: sourceConceptId must be unique diff --git a/R/appendUsagiFileToSTCMtable.R b/R/appendUsagiFileToSTCMtable.R index 8b2aacb..3072e82 100644 --- a/R/appendUsagiFileToSTCMtable.R +++ b/R/appendUsagiFileToSTCMtable.R @@ -12,11 +12,13 @@ #' - source_concept_class #' - source_domain #' - source_parents_concept_ids +#' - source_code_description_fi #' #' Extended Usagi columns: #' - ADD_INFO:sourceConceptId #' - ADD_INFO:sourceConceptClass #' - ADD_INFO:sourceDomain +#' - ADD_INFO:sourceName_fi (optional, NA if missing) #' #' If formats don't match (e.g., extended Usagi with default STCM), the extended information #' will be ignored with a warning. @@ -83,7 +85,8 @@ appendUsagiFileToSTCMtable <- function( stcmExtendedColumns <- c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", - "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids" + "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids", + "source_code_description_fi" ) if (all(stcmExtendedColumns %in% stcmTableColumns)) { @@ -150,6 +153,10 @@ appendUsagiFileToSTCMtable <- function( if (!"ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceValidEndDate` = lubridate::as_date(ymd("2099-12-31"))) } + # if colum `ADD_INFO:sourceName_fi` is not present, set it to NA + if (!"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceName_fi` = NA_character_) + } # if colums `ADD_INFO:sourceParents` and `ADD_INFO:sourceParentVocabulary` are present and any of the `ADD_INFO:sourceParents` is not NA if (all(c("ADD_INFO:sourceParents", "ADD_INFO:sourceParentVocabulary") %in% usagiTibbleColumns) && @@ -227,7 +234,8 @@ appendUsagiFileToSTCMtable <- function( invalid_reason = NA_character_, source_concept_class = `ADD_INFO:sourceConceptClass`, source_domain = `ADD_INFO:sourceDomain`, - source_parents_concept_ids = sourceParentConceptIds + source_parents_concept_ids = sourceParentConceptIds, + source_code_description_fi = `ADD_INFO:sourceName_fi` ) } diff --git a/R/buildVocabuliesAll.R b/R/buildVocabuliesAll.R index d5b1344..f566fd3 100644 --- a/R/buildVocabuliesAll.R +++ b/R/buildVocabuliesAll.R @@ -111,6 +111,48 @@ buildVocabulariesAll <- function( return(validationLogTibble) } + # check all the Finnish names in the SourceToConceptMap table are in the CONCEPT_SYNONYM table + nFinnishNamesInSTCM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM ( + SELECT DISTINCT source_concept_id, source_code_description_fi + FROM @vocabulary_database_schema.@source_to_concept_map_table + WHERE source_concept_id != 0 + AND source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(source_code_description_fi)) != '' + AND (source_code_description IS NULL OR source_code_description_fi != source_code_description) + ) AS t", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + nFinnishSynonymsInCDM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM @vocabulary_database_schema.CONCEPT_SYNONYM + WHERE language_concept_id = 4181730 + AND concept_id IN (SELECT DISTINCT source_concept_id FROM @vocabulary_database_schema.@source_to_concept_map_table)", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + + if (nFinnishNamesInSTCM != nFinnishSynonymsInCDM) { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "ERROR", + step = "Finnish names in CONCEPT_SYNONYM", + message = paste0( + "Found ", nFinnishSynonymsInCDM, " Finnish synonyms in CONCEPT_SYNONYM but ", + nFinnishNamesInSTCM, " Finnish names in the SourceToConceptMap table" + ) + )) + } else { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "SUCCESS", + step = "Finnish names in CONCEPT_SYNONYM", + message = "" + )) + } + # create the ancestor tables message("Creating the ancestor tables") errorMessage <- "" diff --git a/R/calculateMappingStatus.R b/R/calculateMappingStatus.R index acee366..8294e81 100644 --- a/R/calculateMappingStatus.R +++ b/R/calculateMappingStatus.R @@ -90,7 +90,7 @@ calculateMappingStatus <- function( ON c.concept_id = cr.concept_id_1 LEFT JOIN ( SELECT concept_id, concept_synonym_name FROM @vocabulary_database_schema.CONCEPT_SYNONYM - WHERE language_concept_id == 4181730 + WHERE language_concept_id = 4181730 ) AS cs ON c.concept_id = cs.concept_id " diff --git a/R/createSourceToConceptMapExtended.R b/R/createSourceToConceptMapExtended.R index d20a236..4ba52dc 100644 --- a/R/createSourceToConceptMapExtended.R +++ b/R/createSourceToConceptMapExtended.R @@ -1,7 +1,7 @@ #' Create Extended Source to Concept Map Table #' #' Creates an extended version of the Source to Concept Map table that includes additional columns -#' for source concept class, domain, and parent relationships. +#' for source concept class, domain, parent relationships, and the Finnish source code description. #' #' @param connection A DatabaseConnector connection object #' @param vocabularyDatabaseSchema Schema name where the vocabulary tables are stored diff --git a/R/usagiFile.R b/R/usagiFile.R index 5c2c0ff..c5cb850 100644 --- a/R/usagiFile.R +++ b/R/usagiFile.R @@ -30,6 +30,7 @@ #' - sourceValidEndDate (date): End date of validity #' - sourceParents (character): Parent concepts #' - sourceParentVocabulary (character): Vocabulary of parent concepts +#' - sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM #' - validationMessagess (character): Validation messages #' #' @param pathToUsagiFile Path to the Usagi mapping file to read @@ -69,6 +70,7 @@ readUsagiFile <- function(pathToUsagiFile) { `ADD_INFO:sourceValidEndDate` = readr::col_date(), `ADD_INFO:sourceParents` = readr::col_character(), `ADD_INFO:sourceParentVocabulary` = readr::col_character(), + `ADD_INFO:sourceName_fi` = readr::col_character(), `ADD_INFO:validationMessages` = readr::col_character(), # .default = readr::col_character() @@ -110,6 +112,7 @@ readUsagiFile <- function(pathToUsagiFile) { #' - ADD_INFO:sourceValidEndDate (date): Source valid end date #' - ADD_INFO:sourceParents (character): Source parents #' - ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +#' - ADD_INFO:sourceName_fi (character): Source name in Finnish #' - ADD_INFO:validationMessages (character): Validation messages #' - ADD_INFO:autoUpdatingInfo (character): Auto updating info #' @@ -147,7 +150,8 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { "ADD_INFO:sourceValidStartDate", "ADD_INFO:sourceValidEndDate", "ADD_INFO:sourceParents", - "ADD_INFO:sourceParentVocabulary" + "ADD_INFO:sourceParentVocabulary", + "ADD_INFO:sourceName_fi" ) lastColNames <- c( @@ -164,7 +168,7 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { dplyr::any_of(c(firstColNames, midColNames, lastColNames)) ) - # force "ADD_INFO:validationMessages" and "ADD_INFO:autoUpdatingInfo" to be NA if they are empty, if they exist + # force "ADD_INFO:validationMessages", "ADD_INFO:autoUpdatingInfo" and "ADD_INFO:sourceName_fi" to be NA if they are empty, if they exist if ("ADD_INFO:validationMessages" %in% colNames) { usagiTibble <- usagiTibble |> dplyr::mutate( @@ -177,6 +181,12 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { `ADD_INFO:autoUpdatingInfo` = ifelse(`ADD_INFO:autoUpdatingInfo` == "", NA, `ADD_INFO:autoUpdatingInfo`) ) } + if ("ADD_INFO:sourceName_fi" %in% colNames) { + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = ifelse(`ADD_INFO:sourceName_fi` == "", NA, `ADD_INFO:sourceName_fi`) + ) + } readr::write_csv(usagiTibble, pathToUsagiFile, na = "") } diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 13ba240..bf2e2d4 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -14,6 +14,10 @@ #' - Check codes with mapping to more than one domain are mapped to compatible domains #' - Check if sourceValidStartDate is before sourceValidEndDate #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +#' If usagi file has the ADD_INFO:sourceName_fi column: +#' - Check if sourceName_fi is less than 1000 characters +#' - Check if sourceName_fi is the same for all the rows of a sourceConceptId +#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate #' @param connection A DatabaseConnector connection object @@ -202,7 +206,8 @@ validateUsagiFile <- function( # if it has any of the 3 columns it must have the other 2 if ("ADD_INFO:sourceConceptId" %in% usagiTibbleColumns || "ADD_INFO:sourceConceptClass" %in% usagiTibbleColumns || "ADD_INFO:sourceDomain" %in% usagiTibbleColumns || "ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns || - "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns) { + "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns || + "ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { missingColumns <- c( "ADD_INFO:sourceConceptId", @@ -347,6 +352,64 @@ validateUsagiFile <- function( } } + # - Check ADD_INFO:sourceName_fi, it will be used as Finnish CONCEPT_SYNONYM + # (whitespace-only names are read as NA by readUsagiFile, so they are reported as empty) + if ("ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + validationRules <- validate::validator( + SourceNameFi.is.more.than.1000.characters = field_length(`ADD_INFO:sourceName_fi`, min = 0, max = 1000) + ) + validations <- validate::confront(usagiTibble, validationRules) + + result <- .applyValidationRules(usagiTibble, validations, validationLogR6) + usagiTibble <- result$fileTibble + validationLogR6 <- result$validationLogR6 + + # check all rows of a sourceConceptId have the same sourceName_fi (multi-mapped codes repeat the source row), + # each sourceConceptId gets one Finnish synonym. A sourceCode can have several sourceConceptIds, valid for + # different periods, each with its own name, so the check is not done per sourceCode + sourceConceptIdsWithDifferentNameFi <- usagiTibble |> + dplyr::filter(!is.na(`ADD_INFO:sourceConceptId`)) |> + dplyr::distinct(`ADD_INFO:sourceConceptId`, `ADD_INFO:sourceName_fi`) |> + dplyr::count(`ADD_INFO:sourceConceptId`) |> + dplyr::filter(n > 1) |> + dplyr::pull(`ADD_INFO:sourceConceptId`) + + if (length(sourceConceptIdsWithDifferentNameFi) > 0) { + validationLogR6$ERROR( + "SourceNameFi is not the same for all rows of the sourceConceptId", + paste0("Found ", length(sourceConceptIdsWithDifferentNameFi), " sourceConceptIds with different sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + `ADD_INFO:sourceConceptId` %in% sourceConceptIdsWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceConceptId"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceConceptId", "") + } + + # warn if sourceName_fi is empty, the column is optional so this is not an error + isEmptyNameFi <- usagiTibble$mappingStatus %in% c("APPROVED", "UNCHECKED") & is.na(usagiTibble$`ADD_INFO:sourceName_fi`) + + if (any(isEmptyNameFi)) { + validationLogR6$WARNING( + "SourceNameFi is empty", + paste0("Found ", sum(isEmptyNameFi), " APPROVED or UNCHECKED rows with empty sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + isEmptyNameFi, + paste0(tmpvalidationMessages, " | ", "WARNING: SourceNameFi is empty"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is empty", "") + } + } + # - Check if ADD_INFO:sourceValidStartDate and ADD_INFO:sourceValidEndDate are used if ("ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { missingColumns <- diff --git a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql index 1693a3a..e964d5a 100644 --- a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql +++ b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql @@ -18,5 +18,6 @@ CREATE TABLE @vocabularyDatabaseSchema.@sourceToConceptMapTable ( invalid_reason varchar(1) NULL , source_concept_class varchar(20) NULL, source_domain varchar(20) NULL, - source_parents_concept_ids varchar(255) NULL + source_parents_concept_ids varchar(255) NULL, + source_code_description_fi varchar(1000) NULL ); \ No newline at end of file diff --git a/inst/sql/sql_server/STCMExtendedToCDM.sql b/inst/sql/sql_server/STCMExtendedToCDM.sql index c2328bf..f011e88 100644 --- a/inst/sql/sql_server/STCMExtendedToCDM.sql +++ b/inst/sql/sql_server/STCMExtendedToCDM.sql @@ -300,4 +300,42 @@ SELECT DISTINCT ) AS valid_end_date, NULL AS invalid_reason FROM split_parents AS sp -WHERE sp.source_parents_concept_ids IS NOT NULL +WHERE sp.source_parents_concept_ids IS NOT NULL; + + +-- 5. CONCEPT_SYNONYM table +-- Only Finnish synonyms (language_concept_id = 4181730) are managed here, synonyms in other languages are not touched +-- Delete previous Finnish rows with the same concept_id as in the source_to_concept_map.source_concept_id +-- Insert one row for each **unique** source_concept_id with a non-empty source_code_description_fi +-- that is different from source_code_description (same name as concept_name adds nothing as synonym) +-- - concept_id = source_to_concept_map.source_concept_id +-- - concept_synonym_name = source_to_concept_map.source_code_description_fi +-- - language_concept_id = 4181730 (Finnish) +DELETE FROM + @vocabularyDatabaseSchema.CONCEPT_SYNONYM +WHERE + language_concept_id = 4181730 + AND concept_id IN ( + SELECT + DISTINCT stcm.source_concept_id + FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm + ); + +INSERT INTO + @vocabularyDatabaseSchema.CONCEPT_SYNONYM ( + concept_id, + concept_synonym_name, + language_concept_id + ) +SELECT DISTINCT + stcm.source_concept_id AS concept_id, + stcm.source_code_description_fi AS concept_synonym_name, + 4181730 AS language_concept_id +FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm +WHERE + stcm.source_concept_id != 0 + AND stcm.source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(stcm.source_code_description_fi)) != '' + AND (stcm.source_code_description IS NULL OR stcm.source_code_description_fi != stcm.source_code_description); diff --git a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb index 9e8753d..61e9209 100644 Binary files a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb and b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb differ diff --git a/inst/testdata/createTestData.R b/inst/testdata/createTestData.R index fc964da..1ff72b6 100644 --- a/inst/testdata/createTestData.R +++ b/inst/testdata/createTestData.R @@ -42,7 +42,8 @@ allUsagiFiles <- dplyr::bind_rows(ICD10fiUsagiFile, UNITfiUsagiFile, ICD10fiUsag conceptIdsToMap <- allUsagiFiles |> dplyr::pull(conceptId) |> unique() # add extra conceptIds to map -conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219) +# 4181730 is the Finnish language concept, used as language_concept_id in CONCEPT_SYNONYM +conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219, 4181730) # For each table we have to add also the concepts to the concept table diff --git a/man/appendUsagiFileToSTCMtable.Rd b/man/appendUsagiFileToSTCMtable.Rd index c2503ea..1281765 100644 --- a/man/appendUsagiFileToSTCMtable.Rd +++ b/man/appendUsagiFileToSTCMtable.Rd @@ -57,6 +57,7 @@ Extended STCM columns (includes all default columns plus): \item source_concept_class \item source_domain \item source_parents_concept_ids +\item source_code_description_fi } Extended Usagi columns: @@ -64,6 +65,7 @@ Extended Usagi columns: \item ADD_INFO:sourceConceptId \item ADD_INFO:sourceConceptClass \item ADD_INFO:sourceDomain +\item ADD_INFO:sourceName_fi (optional, NA if missing) } If formats don't match (e.g., extended Usagi with default STCM), the extended information diff --git a/man/createSourceToConceptMapExtended.Rd b/man/createSourceToConceptMapExtended.Rd index 14ae818..2e4b035 100644 --- a/man/createSourceToConceptMapExtended.Rd +++ b/man/createSourceToConceptMapExtended.Rd @@ -22,5 +22,5 @@ createSourceToConceptMapExtended( } \description{ Creates an extended version of the Source to Concept Map table that includes additional columns -for source concept class, domain, and parent relationships. +for source concept class, domain, parent relationships, and the Finnish source code description. } diff --git a/man/readUsagiFile.Rd b/man/readUsagiFile.Rd index cc63ab9..796130a 100644 --- a/man/readUsagiFile.Rd +++ b/man/readUsagiFile.Rd @@ -47,6 +47,7 @@ Optional ADD_INFO columns: \item sourceValidEndDate (date): End date of validity \item sourceParents (character): Parent concepts \item sourceParentVocabulary (character): Vocabulary of parent concepts +\item sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM \item validationMessagess (character): Validation messages } } diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index 06f74cc..a8b5ad7 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -37,5 +37,9 @@ If usagi file has C&CR columns: \item Check codes with mapping to more than one domain are mapped to compatible domains \item Check if sourceValidStartDate is before sourceValidEndDate \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +If usagi file has the ADD_INFO:sourceName_fi column: +\item Check if sourceName_fi is less than 1000 characters +\item Check if sourceName_fi is the same for all the rows of a sourceConceptId +\item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/man/writeUsagiFile.Rd b/man/writeUsagiFile.Rd index 9441d51..7a941dd 100644 --- a/man/writeUsagiFile.Rd +++ b/man/writeUsagiFile.Rd @@ -44,6 +44,7 @@ The function writes a Usagi mapping file with the following columns: \item ADD_INFO:sourceValidEndDate (date): Source valid end date \item ADD_INFO:sourceParents (character): Source parents \item ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +\item ADD_INFO:sourceName_fi (character): Source name in Finnish \item ADD_INFO:validationMessages (character): Validation messages \item ADD_INFO:autoUpdatingInfo (character): Auto updating info } diff --git a/tests/testthat/test-STCMToCDMTable.R b/tests/testthat/test-STCMToCDMTable.R index 6584227..4042d07 100644 --- a/tests/testthat/test-STCMToCDMTable.R +++ b/tests/testthat/test-STCMToCDMTable.R @@ -25,6 +25,7 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre "code9", 2000000009, "TestVocab", "Test Code 9 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002", "code10", 2000000010, "TestVocab", "Test Code 10 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002|2000000003" ) + testData <- testData |> dplyr::mutate(source_code_description_fi = NA_character_) # Create connection to test database connection <- DatabaseConnector::connect( @@ -113,3 +114,80 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre res |> dplyr::pull(concept_id_2) |> expect_equal(c(2000000001, 2000000001, 2000000002, 2000000001, 2000000002, 2000000003)) }) + +test_that("STCMToCDMTables creates Finnish CONCEPT_SYNONYM entries from source_code_description_fi", { + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + sourceToConceptMapTable <- "source_to_concept_map_extended" + + testData <- tibble::tribble( + ~source_code, ~source_concept_id, ~source_vocabulary_id, ~source_code_description, ~target_concept_id, ~target_vocabulary_id, ~valid_start_date, ~valid_end_date, ~invalid_reason, ~source_concept_class, ~source_domain, ~source_parents_concept_ids, ~source_code_description_fi, + # multi-mapped code, one synonym + "code1", 2000000001, "TestVocab", "Test Code 1", 141797, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + "code1", 2000000001, "TestVocab", "Test Code 1", 36713461, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + # unmapped code also gets synonym + "code2", 2000000002, "TestVocab", "Test Code 2", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 2", + # same name as concept_name is not added as synonym + "code3", 2000000003, "TestVocab", "Test Code 3", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Test Code 3", + # no synonym + "code4", 2000000004, "TestVocab", "Test Code 4 no fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, NA_character_, + "code5", 2000000005, "TestVocab", "Test Code 5 blank fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, " ", + # source_concept_id 0 is skipped + "code6", 0, "TestVocab", "Test Code 6 no source concept id", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 6" + ) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + createSourceToConceptMapExtended( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable, + overwrite = TRUE + ) + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = sourceToConceptMapTable, + data = testData + ) + + # existing synonyms: an old Finnish one to be replaced and an English one to be kept + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = "CONCEPT_SYNONYM", + data = tibble::tribble( + ~concept_id, ~concept_synonym_name, ~language_concept_id, + 2000000001, "Vanha nimi", 4181730, + 2000000001, "Old English synonym", 4180186 + ), + dropTableIfExists = FALSE, + createTable = FALSE + ) + + # run twice to check it is idempotent + for (i in 1:2) { + STCMToCDMTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + } + + res <- dplyr::tbl(connection, "CONCEPT_SYNONYM") |> + dplyr::filter(concept_id >= 2000000000 | concept_id == 0) |> + dplyr::arrange(concept_id, language_concept_id) |> + dplyr::collect() + + finnish <- res |> dplyr::filter(language_concept_id == 4181730) + finnish |> dplyr::pull(concept_id) |> expect_equal(c(2000000001, 2000000002)) + finnish |> dplyr::pull(concept_synonym_name) |> expect_equal(c("Testikoodi 1", "Testikoodi 2")) + + # other languages are not touched + res |> dplyr::filter(language_concept_id != 4181730) |> dplyr::pull(concept_synonym_name) |> expect_equal("Old English synonym") +}) diff --git a/tests/testthat/test-appendUsagiFileToSTCMTable.R b/tests/testthat/test-appendUsagiFileToSTCMTable.R index 6749c9c..2fb6846 100644 --- a/tests/testthat/test-appendUsagiFileToSTCMTable.R +++ b/tests/testthat/test-appendUsagiFileToSTCMTable.R @@ -81,13 +81,19 @@ test_that("test appendUsagiFileToSTCMTable appends the usagi file to the sourceT expect_equal(c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", "invalid_reason", "source_concept_class", - "source_domain", "source_parents_concept_ids")) + "source_domain", "source_parents_concept_ids", "source_code_description_fi")) stcmTable |> dplyr::filter(is.na(source_parents_concept_ids)) |> nrow() |> expect_equal(0) + # source_code_description_fi comes from ADD_INFO:sourceName_fi + stcmTable |> + dplyr::filter(source_code == "A01.0+G01") |> + dplyr::pull(source_code_description_fi) |> + expect_equal("Lavantautiin liittyvä aivokalvotulehdus") + }) test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the sourceToConceptMapTable with ICD10 parent information", { @@ -130,3 +136,34 @@ test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the expect_equal("2000503725") }) + +test_that("test appendUsagiFileToSTCMTable sets source_code_description_fi to NA when the usagi file has no ADD_INFO:sourceName_fi", { + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-`ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + sourceToConceptMapTable <- "source_to_concept_map_extended" + createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) + + appendUsagiFileToSTCMtable( + vocabularyId = "ICD10fi", + pathToUsagiFile = pathToUsagiFile, + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + + stcmTable <- DBI::dbReadTable(connection, sourceToConceptMapTable) |> tibble::as_tibble() + stcmTable |> nrow() |> expect_equal(nrow(usagiTibble)) + stcmTable |> dplyr::pull(source_code_description_fi) |> is.na() |> all() |> expect_true() +}) diff --git a/tests/testthat/test-buildVocabulariesAll.R b/tests/testthat/test-buildVocabulariesAll.R index c0fcf28..8e0416a 100644 --- a/tests/testthat/test-buildVocabulariesAll.R +++ b/tests/testthat/test-buildVocabulariesAll.R @@ -28,6 +28,9 @@ test_that("buildVocabulariesAll works", { # Check results validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) + # Finnish names were added to CONCEPT_SYNONYM + validationLogTibble |> dplyr::filter(step == "Finnish names in CONCEPT_SYNONYM") |> dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + # check the validation results folder expect_true(!file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) }) diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index 8a5d44b..7127132 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -193,8 +193,9 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { # SourceConceptId is not unique validationsSummary |> dplyr::filter(step == "SourceConceptId is not unique") |> nrow() |> expect_equal(1) validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> nrow() |> expect_equal(1) - validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal("ERROR: SourceConceptId is not unique") + # the row shares sourceConceptId with another sourceCode that has a different sourceName_fi + validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceConceptId is not unique | ERROR: SourceNameFi is not the same for all rows of the sourceConceptId") validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") @@ -306,3 +307,119 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { + +test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + pathToValidatedUsagiFile <- tempfile(fileext = ".csv") + vocabularyDatabaseSchema = "main" + sourceConceptIdOffset = 2000500000 + + # make a copy of a valid usagi file with errors in ADD_INFO:sourceName_fi + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = dplyr::case_when( + # too long + sourceCode == "A01.0+G01" ~ strrep("a", 1001), + # empty + sourceCode == "A01.0+I39.8" ~ NA_character_, + # different within a multi-mapped sourceCode, all rows share the same sourceConceptId + sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", + TRUE ~ `ADD_INFO:sourceName_fi` + ) + ) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile( + pathToUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile, + sourceConceptIdOffset + ) + + validatedUsagiFile <- readUsagiFile(pathToValidatedUsagiFile) + + # only the sourceName_fi validations fail + validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> + expect_setequal(c( + "SourceNameFi is more than 1000 characters", + "SourceNameFi is not the same for all rows of the sourceConceptId", + "SourceNameFi is empty" + )) + + # SourceNameFi is more than 1000 characters + validationsSummary |> dplyr::filter(step == "SourceNameFi is more than 1000 characters") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceNameFi is more than 1000 characters") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") + + # SourceNameFi is not the same for all rows of the sourceConceptId + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceConceptId", 2)) + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) + + # SourceNameFi is empty, only a warning, mappingStatus is not changed + validationsSummary |> dplyr::filter(step == "SourceNameFi is empty") |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("WARNING: SourceNameFi is empty") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(mappingStatus) |> expect_equal("APPROVED") +}) + +test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName_fi is present", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-dplyr::starts_with("ADD_INFO:"), `ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") +}) + +test_that("test validateUsagiFile allows different ADD_INFO:sourceName_fi for different sourceConceptIds of the same sourceCode", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + # a sourceCode with two sourceConceptIds valid in different periods, each with its own names + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + secondRow <- which(usagiTibble$sourceCode == "A02.2+H22.0")[2] + usagiTibble$`ADD_INFO:sourceConceptId`[secondRow] <- 2000599999 + usagiTibble$sourceName[secondRow] <- "Other name for a later period" + usagiTibble$`ADD_INFO:sourceName_fi`[secondRow] <- "Toinen nimi myöhemmälle jaksolle" + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> + dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + validationsSummary |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) +}) diff --git a/tests/testthat/test-validateVocabularyFolder.R b/tests/testthat/test-validateVocabularyFolder.R index 597b028..95aa62b 100644 --- a/tests/testthat/test-validateVocabularyFolder.R +++ b/tests/testthat/test-validateVocabularyFolder.R @@ -23,7 +23,12 @@ test_that("validateVocabularyFolder returns all SUCCESS for valid vocabulary fol ) # Check validation results - validationsLogTibble |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) + # only warning: UNITfi has the ADD_INFO:sourceName_fi column but all values are empty + notSuccess <- validationsLogTibble |> dplyr::filter(type != "SUCCESS") + notSuccess |> nrow() |> expect_equal(1) + notSuccess |> dplyr::pull(context) |> as.character() |> expect_equal("UNITfi") + notSuccess |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + notSuccess |> dplyr::pull(step) |> expect_equal("SourceNameFi is empty") # Check that validated files were created expect_true(dir.exists(pathToValidatedVocabularyFolder))