Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion NEWS.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# ROMOPMappingTools (development version)

- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms.
- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows.
- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceConceptId; a warning is given if it is empty for APPROVED or UNCHECKED rows.
- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`.
- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`.

Expand Down
27 changes: 15 additions & 12 deletions R/validateUsagiFile.R
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary
#' If usagi file has the ADD_INFO:sourceName_fi column:
#' - Check if sourceName_fi is less than 1000 characters
#' - Check if sourceName_fi is the same for all the rows of a sourceCode
#' - Check if sourceName_fi is the same for all the rows of a sourceConceptId
#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus
#'
#' @param pathToUsagiFile Path to the Usagi mapping file to validate
Expand Down Expand Up @@ -364,27 +364,30 @@ validateUsagiFile <- function(
usagiTibble <- result$fileTibble
validationLogR6 <- result$validationLogR6

# check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row)
sourceCodesWithDifferentNameFi <- usagiTibble |>
dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |>
dplyr::count(sourceCode) |>
# check all rows of a sourceConceptId have the same sourceName_fi (multi-mapped codes repeat the source row),
# each sourceConceptId gets one Finnish synonym. A sourceCode can have several sourceConceptIds, valid for
# different periods, each with its own name, so the check is not done per sourceCode
sourceConceptIdsWithDifferentNameFi <- usagiTibble |>
dplyr::filter(!is.na(`ADD_INFO:sourceConceptId`)) |>
dplyr::distinct(`ADD_INFO:sourceConceptId`, `ADD_INFO:sourceName_fi`) |>
dplyr::count(`ADD_INFO:sourceConceptId`) |>
dplyr::filter(n > 1) |>
dplyr::pull(sourceCode)
dplyr::pull(`ADD_INFO:sourceConceptId`)

if (length(sourceCodesWithDifferentNameFi) > 0) {
if (length(sourceConceptIdsWithDifferentNameFi) > 0) {
validationLogR6$ERROR(
"SourceNameFi is not the same for all rows of the sourceCode",
paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi")
"SourceNameFi is not the same for all rows of the sourceConceptId",
paste0("Found ", length(sourceConceptIdsWithDifferentNameFi), " sourceConceptIds with different sourceName_fi")
)

usagiTibble <- usagiTibble |>
dplyr::mutate(tmpvalidationMessages = dplyr::if_else(
sourceCode %in% sourceCodesWithDifferentNameFi,
paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"),
`ADD_INFO:sourceConceptId` %in% sourceConceptIdsWithDifferentNameFi,
paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceConceptId"),
tmpvalidationMessages
))
} else {
validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "")
validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceConceptId", "")
}

# warn if sourceName_fi is empty, the column is optional so this is not an error
Expand Down
2 changes: 1 addition & 1 deletion man/validateUsagiFile.Rd

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

104 changes: 0 additions & 104 deletions plan_add_finnish_names.md

This file was deleted.

42 changes: 35 additions & 7 deletions tests/testthat/test-validateUsagiFile.R
Original file line number Diff line number Diff line change
Expand Up @@ -193,8 +193,9 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", {
# SourceConceptId is not unique
validationsSummary |> dplyr::filter(step == "SourceConceptId is not unique") |> nrow() |> expect_equal(1)
validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> nrow() |> expect_equal(1)
validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |>
expect_equal("ERROR: SourceConceptId is not unique")
# the row shares sourceConceptId with another sourceCode that has a different sourceName_fi
validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |>
expect_equal("ERROR: SourceConceptId is not unique | ERROR: SourceNameFi is not the same for all rows of the sourceConceptId")
validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(mappingStatus) |>
expect_equal("FLAGGED")

Expand Down Expand Up @@ -324,7 +325,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", {
sourceCode == "A01.0+G01" ~ strrep("a", 1001),
# empty
sourceCode == "A01.0+I39.8" ~ NA_character_,
# different within a multi-mapped sourceCode
# different within a multi-mapped sourceCode, all rows share the same sourceConceptId
sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi",
TRUE ~ `ADD_INFO:sourceName_fi`
)
Expand Down Expand Up @@ -352,7 +353,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", {
validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |>
expect_setequal(c(
"SourceNameFi is more than 1000 characters",
"SourceNameFi is not the same for all rows of the sourceCode",
"SourceNameFi is not the same for all rows of the sourceConceptId",
"SourceNameFi is empty"
))

Expand All @@ -362,10 +363,10 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", {
expect_equal("ERROR: SourceNameFi is more than 1000 characters")
validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED")

# SourceNameFi is not the same for all rows of the sourceCode
validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR")
# SourceNameFi is not the same for all rows of the sourceConceptId
validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR")
validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |>
expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2))
expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceConceptId", 2))
validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2))

# SourceNameFi is empty, only a warning, mappingStatus is not changed
Expand Down Expand Up @@ -395,3 +396,30 @@ test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName

validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR")
})

test_that("test validateUsagiFile allows different ADD_INFO:sourceName_fi for different sourceConceptIds of the same sourceCode", {

pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB()
withr::defer(unlink(pathToOMOPVocabularyDuckDBfile))

# a sourceCode with two sourceConceptIds valid in different periods, each with its own names
usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools"))
secondRow <- which(usagiTibble$sourceCode == "A02.2+H22.0")[2]
usagiTibble$`ADD_INFO:sourceConceptId`[secondRow] <- 2000599999
usagiTibble$sourceName[secondRow] <- "Other name for a later period"
usagiTibble$`ADD_INFO:sourceName_fi`[secondRow] <- "Toinen nimi myöhemmälle jaksolle"
pathToUsagiFile <- tempfile(fileext = ".csv")
writeUsagiFile(usagiTibble, pathToUsagiFile)

connection <- DatabaseConnector::connect(
dbms = "duckdb",
server = pathToOMOPVocabularyDuckDBfile
)
on.exit(DatabaseConnector::disconnect(connection))

validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000)

validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |>
dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS")
validationsSummary |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0)
})
Loading