Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
# ROMOPMappingTools (development version)

- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms.
- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows.
- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`.
- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`.

# ROMOPMappingTools 2.1.3

- Added rule to validate usagi file: sourceConceptId must be unique
Expand Down
12 changes: 10 additions & 2 deletions R/appendUsagiFileToSTCMtable.R
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@
#' - source_concept_class
#' - source_domain
#' - source_parents_concept_ids
#' - source_code_description_fi
#'
#' Extended Usagi columns:
#' - ADD_INFO:sourceConceptId
#' - ADD_INFO:sourceConceptClass
#' - ADD_INFO:sourceDomain
#' - ADD_INFO:sourceName_fi (optional, NA if missing)
#'
#' If formats don't match (e.g., extended Usagi with default STCM), the extended information
#' will be ignored with a warning.
Expand Down Expand Up @@ -83,7 +85,8 @@ appendUsagiFileToSTCMtable <- function(
stcmExtendedColumns <- c(
"source_code", "source_concept_id", "source_vocabulary_id", "source_code_description",
"target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date",
"invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids"
"invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids",
"source_code_description_fi"
)

if (all(stcmExtendedColumns %in% stcmTableColumns)) {
Expand Down Expand Up @@ -150,6 +153,10 @@ appendUsagiFileToSTCMtable <- function(
if (!"ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) {
usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceValidEndDate` = lubridate::as_date(ymd("2099-12-31")))
}
# if colum `ADD_INFO:sourceName_fi` is not present, set it to NA
if (!"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) {
usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceName_fi` = NA_character_)
}

# if colums `ADD_INFO:sourceParents` and `ADD_INFO:sourceParentVocabulary` are present and any of the `ADD_INFO:sourceParents` is not NA
if (all(c("ADD_INFO:sourceParents", "ADD_INFO:sourceParentVocabulary") %in% usagiTibbleColumns) &&
Expand Down Expand Up @@ -227,7 +234,8 @@ appendUsagiFileToSTCMtable <- function(
invalid_reason = NA_character_,
source_concept_class = `ADD_INFO:sourceConceptClass`,
source_domain = `ADD_INFO:sourceDomain`,
source_parents_concept_ids = sourceParentConceptIds
source_parents_concept_ids = sourceParentConceptIds,
source_code_description_fi = `ADD_INFO:sourceName_fi`
)
}

Expand Down
42 changes: 42 additions & 0 deletions R/buildVocabuliesAll.R
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,48 @@ buildVocabulariesAll <- function(
return(validationLogTibble)
}

# check all the Finnish names in the SourceToConceptMap table are in the CONCEPT_SYNONYM table
nFinnishNamesInSTCM <- DatabaseConnector::renderTranslateQuerySql(
connection,
"SELECT COUNT(*) AS n FROM (
SELECT DISTINCT source_concept_id, source_code_description_fi
FROM @vocabulary_database_schema.@source_to_concept_map_table
WHERE source_concept_id != 0
AND source_code_description_fi IS NOT NULL
AND LTRIM(RTRIM(source_code_description_fi)) != ''
AND (source_code_description IS NULL OR source_code_description_fi != source_code_description)
) AS t",
vocabulary_database_schema = vocabularyDatabaseSchema,
source_to_concept_map_table = sourceToConceptMapTable
)[[1]]
nFinnishSynonymsInCDM <- DatabaseConnector::renderTranslateQuerySql(
connection,
"SELECT COUNT(*) AS n FROM @vocabulary_database_schema.CONCEPT_SYNONYM
WHERE language_concept_id = 4181730
AND concept_id IN (SELECT DISTINCT source_concept_id FROM @vocabulary_database_schema.@source_to_concept_map_table)",
vocabulary_database_schema = vocabularyDatabaseSchema,
source_to_concept_map_table = sourceToConceptMapTable
)[[1]]

if (nFinnishNamesInSTCM != nFinnishSynonymsInCDM) {
validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble(
context = "STCMToCDMTables",
type = "ERROR",
step = "Finnish names in CONCEPT_SYNONYM",
message = paste0(
"Found ", nFinnishSynonymsInCDM, " Finnish synonyms in CONCEPT_SYNONYM but ",
nFinnishNamesInSTCM, " Finnish names in the SourceToConceptMap table"
)
))
} else {
validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble(
context = "STCMToCDMTables",
type = "SUCCESS",
step = "Finnish names in CONCEPT_SYNONYM",
message = ""
))
}

# create the ancestor tables
message("Creating the ancestor tables")
errorMessage <- ""
Expand Down
2 changes: 1 addition & 1 deletion R/calculateMappingStatus.R
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ calculateMappingStatus <- function(
ON c.concept_id = cr.concept_id_1
LEFT JOIN (
SELECT concept_id, concept_synonym_name FROM @vocabulary_database_schema.CONCEPT_SYNONYM
WHERE language_concept_id == 4181730
WHERE language_concept_id = 4181730
) AS cs
ON c.concept_id = cs.concept_id
"
Expand Down
2 changes: 1 addition & 1 deletion R/createSourceToConceptMapExtended.R
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#' Create Extended Source to Concept Map Table
#'
#' Creates an extended version of the Source to Concept Map table that includes additional columns
#' for source concept class, domain, and parent relationships.
#' for source concept class, domain, parent relationships, and the Finnish source code description.
#'
#' @param connection A DatabaseConnector connection object
#' @param vocabularyDatabaseSchema Schema name where the vocabulary tables are stored
Expand Down
14 changes: 12 additions & 2 deletions R/usagiFile.R
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#' - sourceValidEndDate (date): End date of validity
#' - sourceParents (character): Parent concepts
#' - sourceParentVocabulary (character): Vocabulary of parent concepts
#' - sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM
#' - validationMessagess (character): Validation messages
#'
#' @param pathToUsagiFile Path to the Usagi mapping file to read
Expand Down Expand Up @@ -69,6 +70,7 @@ readUsagiFile <- function(pathToUsagiFile) {
`ADD_INFO:sourceValidEndDate` = readr::col_date(),
`ADD_INFO:sourceParents` = readr::col_character(),
`ADD_INFO:sourceParentVocabulary` = readr::col_character(),
`ADD_INFO:sourceName_fi` = readr::col_character(),
`ADD_INFO:validationMessages` = readr::col_character(),
#
.default = readr::col_character()
Expand Down Expand Up @@ -110,6 +112,7 @@ readUsagiFile <- function(pathToUsagiFile) {
#' - ADD_INFO:sourceValidEndDate (date): Source valid end date
#' - ADD_INFO:sourceParents (character): Source parents
#' - ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary
#' - ADD_INFO:sourceName_fi (character): Source name in Finnish
#' - ADD_INFO:validationMessages (character): Validation messages
#' - ADD_INFO:autoUpdatingInfo (character): Auto updating info
#'
Expand Down Expand Up @@ -147,7 +150,8 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) {
"ADD_INFO:sourceValidStartDate",
"ADD_INFO:sourceValidEndDate",
"ADD_INFO:sourceParents",
"ADD_INFO:sourceParentVocabulary"
"ADD_INFO:sourceParentVocabulary",
"ADD_INFO:sourceName_fi"
)

lastColNames <- c(
Expand All @@ -164,7 +168,7 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) {
dplyr::any_of(c(firstColNames, midColNames, lastColNames))
)

# force "ADD_INFO:validationMessages" and "ADD_INFO:autoUpdatingInfo" to be NA if they are empty, if they exist
# force "ADD_INFO:validationMessages", "ADD_INFO:autoUpdatingInfo" and "ADD_INFO:sourceName_fi" to be NA if they are empty, if they exist
if ("ADD_INFO:validationMessages" %in% colNames) {
usagiTibble <- usagiTibble |>
dplyr::mutate(
Expand All @@ -177,6 +181,12 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) {
`ADD_INFO:autoUpdatingInfo` = ifelse(`ADD_INFO:autoUpdatingInfo` == "", NA, `ADD_INFO:autoUpdatingInfo`)
)
}
if ("ADD_INFO:sourceName_fi" %in% colNames) {
usagiTibble <- usagiTibble |>
dplyr::mutate(
`ADD_INFO:sourceName_fi` = ifelse(`ADD_INFO:sourceName_fi` == "", NA, `ADD_INFO:sourceName_fi`)
)
}

readr::write_csv(usagiTibble, pathToUsagiFile, na = "")
}
62 changes: 61 additions & 1 deletion R/validateUsagiFile.R
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
#' - Check codes with mapping to more than one domain are mapped to compatible domains
#' - Check if sourceValidStartDate is before sourceValidEndDate
#' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary
#' If usagi file has the ADD_INFO:sourceName_fi column:
#' - Check if sourceName_fi is less than 1000 characters
#' - Check if sourceName_fi is the same for all the rows of a sourceCode
#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus
#'
#' @param pathToUsagiFile Path to the Usagi mapping file to validate
#' @param connection A DatabaseConnector connection object
Expand Down Expand Up @@ -202,7 +206,8 @@ validateUsagiFile <- function(
# if it has any of the 3 columns it must have the other 2
if ("ADD_INFO:sourceConceptId" %in% usagiTibbleColumns || "ADD_INFO:sourceConceptClass" %in% usagiTibbleColumns || "ADD_INFO:sourceDomain" %in% usagiTibbleColumns ||
"ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns ||
"ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns) {
"ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns ||
"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) {
missingColumns <-
c(
"ADD_INFO:sourceConceptId",
Expand Down Expand Up @@ -347,6 +352,61 @@ validateUsagiFile <- function(
}
}

# - Check ADD_INFO:sourceName_fi, it will be used as Finnish CONCEPT_SYNONYM
# (whitespace-only names are read as NA by readUsagiFile, so they are reported as empty)
if ("ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) {
validationRules <- validate::validator(
SourceNameFi.is.more.than.1000.characters = field_length(`ADD_INFO:sourceName_fi`, min = 0, max = 1000)
)
validations <- validate::confront(usagiTibble, validationRules)

result <- .applyValidationRules(usagiTibble, validations, validationLogR6)
usagiTibble <- result$fileTibble
validationLogR6 <- result$validationLogR6

# check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row)
sourceCodesWithDifferentNameFi <- usagiTibble |>
dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |>
dplyr::count(sourceCode) |>
dplyr::filter(n > 1) |>
dplyr::pull(sourceCode)

if (length(sourceCodesWithDifferentNameFi) > 0) {
validationLogR6$ERROR(
"SourceNameFi is not the same for all rows of the sourceCode",
paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi")
)

usagiTibble <- usagiTibble |>
dplyr::mutate(tmpvalidationMessages = dplyr::if_else(
sourceCode %in% sourceCodesWithDifferentNameFi,
paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"),
tmpvalidationMessages
))
} else {
validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "")
}

# warn if sourceName_fi is empty, the column is optional so this is not an error
isEmptyNameFi <- usagiTibble$mappingStatus %in% c("APPROVED", "UNCHECKED") & is.na(usagiTibble$`ADD_INFO:sourceName_fi`)

if (any(isEmptyNameFi)) {
validationLogR6$WARNING(
"SourceNameFi is empty",
paste0("Found ", sum(isEmptyNameFi), " APPROVED or UNCHECKED rows with empty sourceName_fi")
)

usagiTibble <- usagiTibble |>
dplyr::mutate(tmpvalidationMessages = dplyr::if_else(
isEmptyNameFi,
paste0(tmpvalidationMessages, " | ", "WARNING: SourceNameFi is empty"),
tmpvalidationMessages
))
} else {
validationLogR6$SUCCESS("SourceNameFi is empty", "")
}
}

# - Check if ADD_INFO:sourceValidStartDate and ADD_INFO:sourceValidEndDate are used
if ("ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) {
missingColumns <-
Expand Down
3 changes: 2 additions & 1 deletion inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ CREATE TABLE @vocabularyDatabaseSchema.@sourceToConceptMapTable (
invalid_reason varchar(1) NULL ,
source_concept_class varchar(20) NULL,
source_domain varchar(20) NULL,
source_parents_concept_ids varchar(255) NULL
source_parents_concept_ids varchar(255) NULL,
source_code_description_fi varchar(1000) NULL
);
40 changes: 39 additions & 1 deletion inst/sql/sql_server/STCMExtendedToCDM.sql
Original file line number Diff line number Diff line change
Expand Up @@ -300,4 +300,42 @@ SELECT DISTINCT
) AS valid_end_date,
NULL AS invalid_reason
FROM split_parents AS sp
WHERE sp.source_parents_concept_ids IS NOT NULL
WHERE sp.source_parents_concept_ids IS NOT NULL;


-- 5. CONCEPT_SYNONYM table
-- Only Finnish synonyms (language_concept_id = 4181730) are managed here, synonyms in other languages are not touched
-- Delete previous Finnish rows with the same concept_id as in the source_to_concept_map.source_concept_id
-- Insert one row for each **unique** source_concept_id with a non-empty source_code_description_fi
-- that is different from source_code_description (same name as concept_name adds nothing as synonym)
-- - concept_id = source_to_concept_map.source_concept_id
-- - concept_synonym_name = source_to_concept_map.source_code_description_fi
-- - language_concept_id = 4181730 (Finnish)
DELETE FROM
@vocabularyDatabaseSchema.CONCEPT_SYNONYM
WHERE
language_concept_id = 4181730
AND concept_id IN (
SELECT
DISTINCT stcm.source_concept_id
FROM
@vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm
);

INSERT INTO
@vocabularyDatabaseSchema.CONCEPT_SYNONYM (
concept_id,
concept_synonym_name,
language_concept_id
)
SELECT DISTINCT
stcm.source_concept_id AS concept_id,
stcm.source_code_description_fi AS concept_synonym_name,
4181730 AS language_concept_id
FROM
@vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm
WHERE
stcm.source_concept_id != 0
AND stcm.source_code_description_fi IS NOT NULL
AND LTRIM(RTRIM(stcm.source_code_description_fi)) != ''
AND (stcm.source_code_description IS NULL OR stcm.source_code_description_fi != stcm.source_code_description);
Binary file modified inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb
Binary file not shown.
3 changes: 2 additions & 1 deletion inst/testdata/createTestData.R
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ allUsagiFiles <- dplyr::bind_rows(ICD10fiUsagiFile, UNITfiUsagiFile, ICD10fiUsag

conceptIdsToMap <- allUsagiFiles |> dplyr::pull(conceptId) |> unique()
# add extra conceptIds to map
conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219)
# 4181730 is the Finnish language concept, used as language_concept_id in CONCEPT_SYNONYM
conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219, 4181730)


# For each table we have to add also the concepts to the concept table
Expand Down
2 changes: 2 additions & 0 deletions man/appendUsagiFileToSTCMtable.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion man/createSourceToConceptMapExtended.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions man/readUsagiFile.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions man/validateUsagiFile.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions man/writeUsagiFile.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading