From 9ac4180986c62466bd0cf42a6e5a26c2ec9fd75a Mon Sep 17 00:00:00 2001 From: javier-gracia-tabuenca-tuni Date: Tue, 10 Nov 2026 10:59:58 +0200 Subject: [PATCH] Add Finnish names from Usagi files to CONCEPT_SYNONYM Carry the optional ADD_INFO:sourceName_fi Usagi column through the new source_code_description_fi column of the extended STCM table into CONCEPT_SYNONYM with language_concept_id = 4181730 (Finnish). - Validate sourceName_fi: max 1000 characters and the same for all rows of a sourceCode (ERROR), warn if empty for APPROVED/UNCHECKED rows - Skip empty names and names equal to sourceName when inserting synonyms - Only Finnish synonyms are deleted and reinserted; other languages kept - buildVocabulariesAll checks all Finnish names reach CONCEPT_SYNONYM - Use standard SQL '=' in calculateMappingStatus synonym query - Add Finnish language concept 4181730 to the test vocabulary Co-Authored-By: Claude Opus 5.5 --- NEWS.md | 7 ++ R/appendUsagiFileToSTCMtable.R | 12 +- R/buildVocabuliesAll.R | 42 +++++++ R/calculateMappingStatus.R | 2 +- R/createSourceToConceptMapExtended.R | 2 +- R/usagiFile.R | 14 ++- R/validateUsagiFile.R | 62 ++++++++++- .../createSourceToConceptMapExtended.sql | 3 +- inst/sql/sql_server/STCMExtendedToCDM.sql | 40 ++++++- .../OMOPVocabulary/OMOPVocabulary.duckdb | Bin 6303744 -> 6565888 bytes inst/testdata/createTestData.R | 3 +- man/appendUsagiFileToSTCMtable.Rd | 2 + man/createSourceToConceptMapExtended.Rd | 2 +- man/readUsagiFile.Rd | 1 + man/validateUsagiFile.Rd | 4 + man/writeUsagiFile.Rd | 1 + plan_add_finnish_names.md | 104 ++++++++++++++++++ tests/testthat/test-STCMToCDMTable.R | 78 +++++++++++++ .../test-appendUsagiFileToSTCMTable.R | 39 ++++++- tests/testthat/test-buildVocabulariesAll.R | 3 + tests/testthat/test-validateUsagiFile.R | 89 +++++++++++++++ .../testthat/test-validateVocabularyFolder.R | 7 +- 22 files changed, 504 insertions(+), 13 deletions(-) create mode 100644 plan_add_finnish_names.md diff --git a/NEWS.md b/NEWS.md index 4b57762..a8d1eed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# ROMOPMappingTools (development version) + +- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. +- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. + # ROMOPMappingTools 2.1.3 - Added rule to validate usagi file: sourceConceptId must be unique diff --git a/R/appendUsagiFileToSTCMtable.R b/R/appendUsagiFileToSTCMtable.R index 8b2aacb..3072e82 100644 --- a/R/appendUsagiFileToSTCMtable.R +++ b/R/appendUsagiFileToSTCMtable.R @@ -12,11 +12,13 @@ #' - source_concept_class #' - source_domain #' - source_parents_concept_ids +#' - source_code_description_fi #' #' Extended Usagi columns: #' - ADD_INFO:sourceConceptId #' - ADD_INFO:sourceConceptClass #' - ADD_INFO:sourceDomain +#' - ADD_INFO:sourceName_fi (optional, NA if missing) #' #' If formats don't match (e.g., extended Usagi with default STCM), the extended information #' will be ignored with a warning. @@ -83,7 +85,8 @@ appendUsagiFileToSTCMtable <- function( stcmExtendedColumns <- c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", - "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids" + "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids", + "source_code_description_fi" ) if (all(stcmExtendedColumns %in% stcmTableColumns)) { @@ -150,6 +153,10 @@ appendUsagiFileToSTCMtable <- function( if (!"ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceValidEndDate` = lubridate::as_date(ymd("2099-12-31"))) } + # if colum `ADD_INFO:sourceName_fi` is not present, set it to NA + if (!"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceName_fi` = NA_character_) + } # if colums `ADD_INFO:sourceParents` and `ADD_INFO:sourceParentVocabulary` are present and any of the `ADD_INFO:sourceParents` is not NA if (all(c("ADD_INFO:sourceParents", "ADD_INFO:sourceParentVocabulary") %in% usagiTibbleColumns) && @@ -227,7 +234,8 @@ appendUsagiFileToSTCMtable <- function( invalid_reason = NA_character_, source_concept_class = `ADD_INFO:sourceConceptClass`, source_domain = `ADD_INFO:sourceDomain`, - source_parents_concept_ids = sourceParentConceptIds + source_parents_concept_ids = sourceParentConceptIds, + source_code_description_fi = `ADD_INFO:sourceName_fi` ) } diff --git a/R/buildVocabuliesAll.R b/R/buildVocabuliesAll.R index d5b1344..f566fd3 100644 --- a/R/buildVocabuliesAll.R +++ b/R/buildVocabuliesAll.R @@ -111,6 +111,48 @@ buildVocabulariesAll <- function( return(validationLogTibble) } + # check all the Finnish names in the SourceToConceptMap table are in the CONCEPT_SYNONYM table + nFinnishNamesInSTCM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM ( + SELECT DISTINCT source_concept_id, source_code_description_fi + FROM @vocabulary_database_schema.@source_to_concept_map_table + WHERE source_concept_id != 0 + AND source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(source_code_description_fi)) != '' + AND (source_code_description IS NULL OR source_code_description_fi != source_code_description) + ) AS t", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + nFinnishSynonymsInCDM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM @vocabulary_database_schema.CONCEPT_SYNONYM + WHERE language_concept_id = 4181730 + AND concept_id IN (SELECT DISTINCT source_concept_id FROM @vocabulary_database_schema.@source_to_concept_map_table)", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + + if (nFinnishNamesInSTCM != nFinnishSynonymsInCDM) { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "ERROR", + step = "Finnish names in CONCEPT_SYNONYM", + message = paste0( + "Found ", nFinnishSynonymsInCDM, " Finnish synonyms in CONCEPT_SYNONYM but ", + nFinnishNamesInSTCM, " Finnish names in the SourceToConceptMap table" + ) + )) + } else { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "SUCCESS", + step = "Finnish names in CONCEPT_SYNONYM", + message = "" + )) + } + # create the ancestor tables message("Creating the ancestor tables") errorMessage <- "" diff --git a/R/calculateMappingStatus.R b/R/calculateMappingStatus.R index acee366..8294e81 100644 --- a/R/calculateMappingStatus.R +++ b/R/calculateMappingStatus.R @@ -90,7 +90,7 @@ calculateMappingStatus <- function( ON c.concept_id = cr.concept_id_1 LEFT JOIN ( SELECT concept_id, concept_synonym_name FROM @vocabulary_database_schema.CONCEPT_SYNONYM - WHERE language_concept_id == 4181730 + WHERE language_concept_id = 4181730 ) AS cs ON c.concept_id = cs.concept_id " diff --git a/R/createSourceToConceptMapExtended.R b/R/createSourceToConceptMapExtended.R index d20a236..4ba52dc 100644 --- a/R/createSourceToConceptMapExtended.R +++ b/R/createSourceToConceptMapExtended.R @@ -1,7 +1,7 @@ #' Create Extended Source to Concept Map Table #' #' Creates an extended version of the Source to Concept Map table that includes additional columns -#' for source concept class, domain, and parent relationships. +#' for source concept class, domain, parent relationships, and the Finnish source code description. #' #' @param connection A DatabaseConnector connection object #' @param vocabularyDatabaseSchema Schema name where the vocabulary tables are stored diff --git a/R/usagiFile.R b/R/usagiFile.R index 5c2c0ff..c5cb850 100644 --- a/R/usagiFile.R +++ b/R/usagiFile.R @@ -30,6 +30,7 @@ #' - sourceValidEndDate (date): End date of validity #' - sourceParents (character): Parent concepts #' - sourceParentVocabulary (character): Vocabulary of parent concepts +#' - sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM #' - validationMessagess (character): Validation messages #' #' @param pathToUsagiFile Path to the Usagi mapping file to read @@ -69,6 +70,7 @@ readUsagiFile <- function(pathToUsagiFile) { `ADD_INFO:sourceValidEndDate` = readr::col_date(), `ADD_INFO:sourceParents` = readr::col_character(), `ADD_INFO:sourceParentVocabulary` = readr::col_character(), + `ADD_INFO:sourceName_fi` = readr::col_character(), `ADD_INFO:validationMessages` = readr::col_character(), # .default = readr::col_character() @@ -110,6 +112,7 @@ readUsagiFile <- function(pathToUsagiFile) { #' - ADD_INFO:sourceValidEndDate (date): Source valid end date #' - ADD_INFO:sourceParents (character): Source parents #' - ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +#' - ADD_INFO:sourceName_fi (character): Source name in Finnish #' - ADD_INFO:validationMessages (character): Validation messages #' - ADD_INFO:autoUpdatingInfo (character): Auto updating info #' @@ -147,7 +150,8 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { "ADD_INFO:sourceValidStartDate", "ADD_INFO:sourceValidEndDate", "ADD_INFO:sourceParents", - "ADD_INFO:sourceParentVocabulary" + "ADD_INFO:sourceParentVocabulary", + "ADD_INFO:sourceName_fi" ) lastColNames <- c( @@ -164,7 +168,7 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { dplyr::any_of(c(firstColNames, midColNames, lastColNames)) ) - # force "ADD_INFO:validationMessages" and "ADD_INFO:autoUpdatingInfo" to be NA if they are empty, if they exist + # force "ADD_INFO:validationMessages", "ADD_INFO:autoUpdatingInfo" and "ADD_INFO:sourceName_fi" to be NA if they are empty, if they exist if ("ADD_INFO:validationMessages" %in% colNames) { usagiTibble <- usagiTibble |> dplyr::mutate( @@ -177,6 +181,12 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { `ADD_INFO:autoUpdatingInfo` = ifelse(`ADD_INFO:autoUpdatingInfo` == "", NA, `ADD_INFO:autoUpdatingInfo`) ) } + if ("ADD_INFO:sourceName_fi" %in% colNames) { + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = ifelse(`ADD_INFO:sourceName_fi` == "", NA, `ADD_INFO:sourceName_fi`) + ) + } readr::write_csv(usagiTibble, pathToUsagiFile, na = "") } diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 13ba240..60d49fc 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -14,6 +14,10 @@ #' - Check codes with mapping to more than one domain are mapped to compatible domains #' - Check if sourceValidStartDate is before sourceValidEndDate #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +#' If usagi file has the ADD_INFO:sourceName_fi column: +#' - Check if sourceName_fi is less than 1000 characters +#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate #' @param connection A DatabaseConnector connection object @@ -202,7 +206,8 @@ validateUsagiFile <- function( # if it has any of the 3 columns it must have the other 2 if ("ADD_INFO:sourceConceptId" %in% usagiTibbleColumns || "ADD_INFO:sourceConceptClass" %in% usagiTibbleColumns || "ADD_INFO:sourceDomain" %in% usagiTibbleColumns || "ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns || - "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns) { + "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns || + "ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { missingColumns <- c( "ADD_INFO:sourceConceptId", @@ -347,6 +352,61 @@ validateUsagiFile <- function( } } + # - Check ADD_INFO:sourceName_fi, it will be used as Finnish CONCEPT_SYNONYM + # (whitespace-only names are read as NA by readUsagiFile, so they are reported as empty) + if ("ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + validationRules <- validate::validator( + SourceNameFi.is.more.than.1000.characters = field_length(`ADD_INFO:sourceName_fi`, min = 0, max = 1000) + ) + validations <- validate::confront(usagiTibble, validationRules) + + result <- .applyValidationRules(usagiTibble, validations, validationLogR6) + usagiTibble <- result$fileTibble + validationLogR6 <- result$validationLogR6 + + # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) + sourceCodesWithDifferentNameFi <- usagiTibble |> + dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> + dplyr::count(sourceCode) |> + dplyr::filter(n > 1) |> + dplyr::pull(sourceCode) + + if (length(sourceCodesWithDifferentNameFi) > 0) { + validationLogR6$ERROR( + "SourceNameFi is not the same for all rows of the sourceCode", + paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + sourceCode %in% sourceCodesWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + } + + # warn if sourceName_fi is empty, the column is optional so this is not an error + isEmptyNameFi <- usagiTibble$mappingStatus %in% c("APPROVED", "UNCHECKED") & is.na(usagiTibble$`ADD_INFO:sourceName_fi`) + + if (any(isEmptyNameFi)) { + validationLogR6$WARNING( + "SourceNameFi is empty", + paste0("Found ", sum(isEmptyNameFi), " APPROVED or UNCHECKED rows with empty sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + isEmptyNameFi, + paste0(tmpvalidationMessages, " | ", "WARNING: SourceNameFi is empty"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is empty", "") + } + } + # - Check if ADD_INFO:sourceValidStartDate and ADD_INFO:sourceValidEndDate are used if ("ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { missingColumns <- diff --git a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql index 1693a3a..e964d5a 100644 --- a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql +++ b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql @@ -18,5 +18,6 @@ CREATE TABLE @vocabularyDatabaseSchema.@sourceToConceptMapTable ( invalid_reason varchar(1) NULL , source_concept_class varchar(20) NULL, source_domain varchar(20) NULL, - source_parents_concept_ids varchar(255) NULL + source_parents_concept_ids varchar(255) NULL, + source_code_description_fi varchar(1000) NULL ); \ No newline at end of file diff --git a/inst/sql/sql_server/STCMExtendedToCDM.sql b/inst/sql/sql_server/STCMExtendedToCDM.sql index c2328bf..f011e88 100644 --- a/inst/sql/sql_server/STCMExtendedToCDM.sql +++ b/inst/sql/sql_server/STCMExtendedToCDM.sql @@ -300,4 +300,42 @@ SELECT DISTINCT ) AS valid_end_date, NULL AS invalid_reason FROM split_parents AS sp -WHERE sp.source_parents_concept_ids IS NOT NULL +WHERE sp.source_parents_concept_ids IS NOT NULL; + + +-- 5. CONCEPT_SYNONYM table +-- Only Finnish synonyms (language_concept_id = 4181730) are managed here, synonyms in other languages are not touched +-- Delete previous Finnish rows with the same concept_id as in the source_to_concept_map.source_concept_id +-- Insert one row for each **unique** source_concept_id with a non-empty source_code_description_fi +-- that is different from source_code_description (same name as concept_name adds nothing as synonym) +-- - concept_id = source_to_concept_map.source_concept_id +-- - concept_synonym_name = source_to_concept_map.source_code_description_fi +-- - language_concept_id = 4181730 (Finnish) +DELETE FROM + @vocabularyDatabaseSchema.CONCEPT_SYNONYM +WHERE + language_concept_id = 4181730 + AND concept_id IN ( + SELECT + DISTINCT stcm.source_concept_id + FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm + ); + +INSERT INTO + @vocabularyDatabaseSchema.CONCEPT_SYNONYM ( + concept_id, + concept_synonym_name, + language_concept_id + ) +SELECT DISTINCT + stcm.source_concept_id AS concept_id, + stcm.source_code_description_fi AS concept_synonym_name, + 4181730 AS language_concept_id +FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm +WHERE + stcm.source_concept_id != 0 + AND stcm.source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(stcm.source_code_description_fi)) != '' + AND (stcm.source_code_description IS NULL OR stcm.source_code_description_fi != stcm.source_code_description); diff --git a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb index 9e8753d179ed4a768824baf2b2d8da83e87043ed..61e9209c0924c2429c6800092de45514698581e7 100644 GIT binary patch delta 13254 zcmbVT3wTt;_1}AUvyW^@vOGd~u*IdkUBP5p^_bNz`%^S1cZu9`nR_KXfPYEU?nkYt64X zcBI4)N{+kchT9(+`JzJ|ld{w_j@CC92BZOwpvfK%FEg2zn@lU{(QSopSz19lrkgWK z%EN^R6Yj28KbD~BqrqZeQ|iQVPww>xCDCsSn(j*M)lb_WExcb!)Cmj-nm&nT1712N zv9%b9ri=+62?|4krsilPg*QmmuEHS_A4#|FDjX}R-^r=R3wMZVi9u8Q0OPbQdguv^ zB@1Vo?Xkmyruy^P%GzBxTqd4tnpN)ry;p4{zH534dtPbzA6$@l*G!F#DNa)J zbg9DFct@wUulDMzzy>|KK6ismjATsS86?X7ukD{0wZs<7v0qYM)}9GEcn zUfJ+Sr)8726e%{E9IJFJs8)&I%C=9mLWP-_bRT(NNp0F~VjV3~A~p^NkX{xzc+~MRvOQ*X(`Ct{V|L zA7hh-Q7KXK7+LFxrqkQg*GG^NF?N(p0-e&8ENXr4Fwpr!6wUAU?lppFqC*5cG%L@86q1JloN(3)zq#mJl<)SKw?HRT^h6HNIBQ}#ti zr`Skz0x}&ITDj4@p6>yFG2(oujn3O@englSs=CZc%lDYALM$Dd0RBdqPH*!Z6xal| zGpsal*qm;p`GSMycbV4-33a`IK)gh=Pnp+q6+9*805#1V+%XdMobznc4X2_8-n4`5 z0hSoSC0KgP$+iaa$%4XFItX<}0pr0@A;l6>@Y5PnY$3%SQXC;AI;1#5N=!%@5K>}8 zN?b^Z4=JvYGBBhJ3Mqp_%8-zf5K@MQl*EuSETjw%DI-G4$dEEBq$Guu|+gnQb&&o;K8or@u$j)0JuU z7gGFSC=*jiPfglN7GD0wO>og*utYcm4%bD|6Lo1X2=&9}>7d70>TXC&7BE!Jg4G_; zxhHVCRrE-6nq6jGysmQ5Pb_kkNX(6az)wJytmwu__EI{qT}cNH6w2l7Yp|BJVAHuGWja2=Er35Th!y|3z*a7Sd21v{B)Y;_PFl5lQmDSZNf3+gz zkjbupB`Kwg23vk&&bIZ5nX{^@m)}&iJcv2^*T0!ee>X|5vV6MFs`0&64Vosfg)^@# zD=)pQ*kl3`{i{yIdJ#k_IQO$}s=9vVvXzVz{VTcsyTSNVa`n zI4sjQF@K^rPfFj=3 zcezPgp&Hmkj=q*Xmc;*9XtQYp$=3O$X=LjoQj8gg4C2}@4Pfy}5~37ADO>occKn?Oxvl*wklaPk&^i33FPe`=rhDG(*8Fb>v(7!BQ!lQsoauL@K z>3mLZV+V@8I9B|pw1V^FBlUswuUosMeEw$%JyI5MnI(z5<_e^f_79|}2p5yFB{DCq zD^EYpz1zigIeB1jftfBz)}@mag0*niz(HkGN__7>aLzcdn?GTv41%)|W{Cle2;VSp zmdH4diblHE66y&@>*2Kc5B`|ZqT|1kyf&#!#X$YTlEY{%&is;TWeRDWTDF9lq>(6Z zO94__nVvz40%eWNidruS!=9IwHEwa@3&!HdWXxhVWnwH*i5z@yI==l9W&A;jq_;U5FmN9&E>{zt@8Y zoZ-uQKe-P_u3kgxapdpy;1nL9tiYZcwo~k#uW5j;Y)HSk&vu1Wdzefsn$w5!1Mvv4 z&tbo!*!-Eg%UjY{^(ZQuUFOqK3*JI~nW3GT7pdNqq7**)pY81q*dJI6X{N;rLtP)FZXg5uGQ z+D4un=UyGvn+DV~P}|PqvfVzuonPE};dc(Wy<)Rxnnqk(Q@wO0*)*%vL$*ya2g#o4 zDA*(iP5(OYe7$pT@jGTw??G;G_pn~aV0z%G;vo{bp%~K(gQnKrhh!SpT8!L%>tlX< ztcBQMy zyi;;eDIh<`oy~AWW_Es4wR5=4z0l0^v%wme=eR+Ds`BOi;LBR`iv)=b;I(ft9(_G*!l!L)Z;>5;AO zmqqdmtd7>asB@1y{e0OdWV=qH2ix2$ISy|&(sHO?MA{F#UtnBEIzbijX>XT%rw{~L zLMth zTuo~idp<|WB~gkO{dj%~fAFYZ?U}~CFj4e^?wvNeWR2%(9^n&=9mBOOy>g?+C&a)6 zy|OkJVQ;JF9Uk`5j1#}mPlyM;?v+h84oBgnVUmza!#gS<2w|-Q(UONe} z(xV-1u=ZO&7O>}5w1L`hJ=15;ki?KI)29jyu-D$&UX?j^ZVM}dYmF8` zrG@a7n&Z4O_vt-i7W#Y^+L-LE=H3wLfUmq}pov_*%~{4;I=a51CwEnoqim&8alcqueFO;+E)xgvh2`b!P5+a>6~@ zjy)!5dUu5AxB^qaYFM@CEvntG7$YIkFt z+Lnj?<0Z23ySYdi==SgC9+DD>wl)%%FOmD|H);DG^Z!#qX6fzFH*)46V^u7dw|GYi z3kv0V?4+pOdzWrU@jRkbV0xg6=|HE~CyFb8#$GG(K<`!~nrEtL&hGJk+Xu}Iff7jx zdn=e_mWzE1hWBwW)8shcKqKcz5Yv-=zZF;px|y4?yk?j^1`!FLR9f|Z?gZ_kiJDsj zJ|a>aP`*NiZ@A3&A0a%*oJP=C>|1Q)vE-un`GJvr9-O#TME6wta*QIiprW?M_n^Qs z3{mRfuGGEJ_l-cY?w%m@01_adn_t%5qf-)rs!F5ebmMyTjY)wxs4L4EVie&u z1dhcyYXtjH!*wbWYpQeJ5mxStZlT|Jb*n&So#?t}#}Ty#{U^1va%0Z7x;@p2!0{A2 za+;j$g>(jkX6hK7NRLiQcQcGaghOUzGdXt%E5ZWHRgeY_=Nva$GRlekC1;voUpFHh zX|@Ww>{B^k8};x-6iqpoQ>in_?m5@1pi7SRAK<3!FBTs7YCMaDx(Hb32_6#Z^170z zczO3z!8ZJVlGQQ0_*mD>+NoC!pqp^_&y$|`h!t+hd)?AnCwg6<2B))HzO7B|4vu{w5x0??Uo+@ zheqUtqfyxQ&l0>FeD`u$ozfha`-2fWA4eep$PF5yb0^n@jW%TGJ}<%mz|pU-s_8J3 z;!#>3$Q{Mw+1*BkOlA8lj8s>iD+ytMiT7>T{d0dIU1LpN>8yHwkDpF(M2ao!-FJgyU30Pp1*K6|S3&Kg%a8I8>`Z>hki z9@&7J9L;py8}rul$Pgh2z#qkHGF$U*6(|L4YBZE~?#cUyu$-m`2E)H#hs=xP`Tw1Q zH|Tq=1v_EhS^lU$cGBYLKuplIq89;j-Tft7#eCF*&dA38ab(&9 zC1;!NDdc6RmdGngboTE{{wj(0MXW*7K2$`FS!T(DB_GK+PVDQNJCGYP5}BanXhsFIaGN7#!;m5Z&j7~ZmheFv$Y zN!Lx9_c)LF>jZECn0il_{xN4k-A4SSp z7@loL6(pb4Solj^Vl%)yO=|4fn18#FTPWl?E7FDhlYNj6Klyr6OdSLzc&%EXZJvygJBqvn$7KURJ4_~V6CIIsNfZ0e%)Q6g;#OH_S;Cfyg*^0YbLKC_UE74TWC;Jz{2L*>DfxZMF@z+ zj@wWnD5#DI*q!!UC(iu^Z|fZCct%C3UFHnn+Ph%C#11NN3)o(yxd{imh3nuaE>sx}B07RaoPH*olsOj7LCW2IV!P~;dP?U?+AfTa^ zo`O8ae}B5^ozSri$%m?BxFDTN_LW!8pP^FRYFo7&Cp<$ot3Qc8BAS2m&Ryt ze$}k%p{AyAEwj^q1yP{!7dBOz<#kf8lsd-AE;s|+{r*3fDh9h|$m=Sa3(+Q_1<;Dn zCZkP3n~GM9b{X0)iwIV|vDRO?xl5wn~1soPaSPaBe zyhhi-QPZCzXfMEFQIH0(rKzc@eUC*g>xO$lz_MFJ`xgQ{*Ri3$F{m>s1jc64jC7z zP-v2|OejfvNhBO58~*5jK~$04LDLVoQZ;4+_w4e|Hj4qipy~PZF1Uv8_V1BN(GPR2 zO^Eg+TvISG;eIWx!HX^%U4kCL``wwOytd$D^6gs-hD5Dk7pAo8-bLT=G~+6kjshbY zv?^83(R}+^%?2;GW%WAmEKsqF_XQW66v!2NoUirLF1I-PTp&^*$p1`*GF}y&i}%YA zJgmafEEK9N;qAbz8-+b!=W3R@fYoeB^65<6zbER-sRliVD?}IOsZ@sFJdiAwP&?BG?8&} z(ir!@(c@5?xsQWl3!AEfL%MV)6e;9lkxdUnUFKy*sa9)YsF1DnvhYxY&%dIFp>x;N z3I9X@x>f^cfX)2Sh=p@@1lDxCTcZUeofyE^d79E+;EI%|7uzr=b+i02cN?o%8NmD@T1tHKOl4Ug(O`+oWlTF$S#Z+MA}Ib% zKxJ6$$bl%#eZQ_+2q&!fE_%qmuD~)j@fSKA`Z^Q~Eze?+qs_Cvho%#yCS57;r3Dt= zQ(OpZCrVH1Jz)AQlc%FS?F!+@K=u|V)=X+FOkkt8>!S_I?PlWa>-Jw~H2iH=yY`PL z{5s^2(fEnb4c#72^XUSxq8Pud0ozp)3J8lXuZ@*_&oMu zvYMFidp7o!U6ZkolWb~ORF~{j|1#O|J6ij$af-jfq^F^92wcb#j~^4FUG;GX+r!dO yZD2S&d6SEEUv1C8g;Vlb{BUV~=hPdh_XmlwvZ)@=V%EVE(T1T7hgXisjQc+a-p@Gz delta 10766 zcmeG?X?Rpcwtf5dzNC{Ngf$(Y7hsUw&I*{$Z6FH>S#Tg|Ad8L-qCn#442no1YO4`R zhkT|Kk${2`okw7Jq`*(RXyy{c|)S5U0&suj1y8pl{xUY_xZ`$K`ut??!ZaeGE5g>A1S#SU7lyJxBS`@lyk z{^z+{HICLcdu72=tNaOmm4PJf&yncWE#5PNF#^|;bFX-R>_dz3t6m_UDc(Kea<9-1dD__=&mmBLoqOn1lP*#Y}5lrvTJp2 zJbmzr_fJ~Z5ql~>iFF)8(ksQLpS?*W`?~iVMskx#s`eL1v@7ID)Uc`h_roBMo>bSD z8A!M}XE*Df%vdC?dkZ8s{X8m-%|32skB_s*Vd9=p=W}W{+b>!IT(*;*?A#G_+05*9 zI<~gh#Btdol`a)jtNZmZM!X_PlAMd1d}(_6RFPm-_an_wYnEp(6@bW|mflZ<7N8-v zz&ttv5=bDSU%W}wnU(z&3Tl{|#3oLj*%GXgMYgmak@T!98(bqSR+;ZsMbxkLtNwCR zZ|@C&y@Lk;UL7(3(2y_y&^r)te&_&~53ucX*@tDOeZW>%vz4)OP0e$QSt$b4q6o`Q;=5=iy>S!bDc z>a+l!a{`4mQe;LXn&`?3p;XPD@3YX@vYb=_aEjy1401*T&$mCMW17Cs=3Q$5z@p>g zfQcJOY+~h9Zh+@vP zS!N6EWY^V#p(LiD%ua8aBIlwp%bmh++ftle0$WhAK&5o&{UX`YDaVn*L-KI)V(nuo5M~hQiYfcx^wKU&(}CNsh|!3Qny^1V%{w z6;V)BwF&`P2p~7H@yGw!3i!@2V9aMm zdNGiFM$bI03O(%^f!LDOKQQkJ8<`8ZwNx*wQ}yimeFN<+R6j3r%fE0?zSTmcnruC} zT&|W#&&m9DQax9`kHnt$T1a`EY$kzs<-zoWPWfI8-rXzINnB&h0+_~pq*E0rpRi-n1bD@t zC=8!E#sd{Zj*h^m?1_;Md;%&GDW4#>_ai?5g^{A8FU!5U2+wZfF*%2kYZmbRupQ`y z5`kU#sbnyTKtPhn%8~H1&I+%s>=j`a){DqB$}owK0GKiY@G2QmWtf}W`wB;rSetC5%VK@otgNGRr`n;%9L03jPL9rnv0FYbaFSKAzA<}> zf7hm=hYz@J)1%@R?`~BuR_UUubNE$fZ%AJN`1FSK1%S{E=?efmZ^%{vc;SX@<)9}s za~uZ7UZ_LuR#V_C7L&q3Goh!{o0zrVZXxaK@+`D>QO-p*Lv6jF&IdoJ6Nf-XnV$EO zj<#HPg|K{?Vj|5kqM1HhBn(jx$QIEwwOF7earQA28$DJ_pUTe6)N@}-YRP%s(ByOI z1yAENc__(FcKQ1XILWW-h~WiX%dVUgVqb^XuR6}lE&9%X<_s3B&nUhQrQr7?_y}8D zjz%;_I&LB(_vajpkGzictDaZ)_AXYXkkOY*Cejga<;;juar#wtR$dR9LOE^ZgHsvR zjm=(zc7k8^AIiMe;H)mSo|kv2w4qMiF0$3#W+9tmeFnO(c-C?Ytv2L8F4C$Mq6RCz zeOd~MJWEfPtP#g)`=qtoL@U;dbpq?$XJB&PhP2|5(MU%RDSQ+v%?)!_S=)S$w^1yA1oI2$exPCYLVJrG39!^U zaScKj8E8+1qgG&$078B+Gd$8=b?daLmN;1RH&N~7SL zL)AE**zcHb;${T3U{K3|q(0f%z?3*h$L)y)pXW)f?EDv_I~ryBcMFcMDRO>|7IntR zXy2?M)fLY1Y_J}dJ3a41J)%`rIhW$H#_$A4-dR{-q-Ce`W}9hcjZ**;8|Lh?vRINY z;E0NP=Sn1;Y3O6m0sJ&O6BMo(@_3G1)aqRQZ=r+nK31TzeC4agwkat!DV!YpSj9=3tfxQS(Ttk^d0%0h}s?0*OA|2L$!3Ta7`mU zkn>sf#u|m1m}mEPp53wVl1}CaNo}gLoma7J08^mEU$Lx%(x8U`9%Pi`uMEUBmrsnJOh<+-#$($`(Shs5)~ARSo=kuc;O(WVBru6kWeSQZ)ZhQ1OO4! zcf&hvxs?B~5!D@D@A?Mq%4VfUH@gmNnYcek@h;>V>)h#b0>*ULX8;Z7UeVL`1FoeS zZXCF;K_P`YT<`URv6Ua$*CjnwVnGdmg_<%!z`X7%{x_5A zv&2kpcg->G8OZ5dNbHh^1{2+_NQfe2gMrp&yFXNn{^~fvNUvX-H3s(QjG4wzr(TfB zKR3(@HeSdyXdzi0*Ynh~zj0Q)+WM@LlSIF&LEs&RrdiHthsCejYT{|;t1}tReFKL0 zRrMAQf9q_e@=b}D*feXB9g*y*Z8DuN2n94Qo{lY-a zQSjmON3B(8A*X?#GM5RiUj3>}!iA?933wV5`!OLdr73(+WE)iA~H(ff8`1e#`G zuVl&d1(htk?nUlTni;V1a>XiM&!Xl@UVnyf-YzlmSE(n zv*K!o;gn{+{Fqlt-a}4keki>TMZ5$>ER3L^h8*WG?rSY}I9Y!%~f^uL7 zM({Of3ij~#V+mKwRlUt;zQP3hHA`b7VsIe|_Kcv3z2C{@nP7LN;PK2M2E2xBuw+z< z21oi-J(f-@%4@YTa?_B|6ta^Tr{#uUJ#N61}2`K zca3Wo_8$vSyBm=)*pHO#?=${`$P8>e^8SDdUX8MZXa&r%R1?k}j?#pUhByt2GW3}# zJWhKU+BICmCC&ouK`KzC8@NibhHpn{N?fHY%S9J6tq*ODZ+Du6bVlMNTH{TeU#Q|` z)Z?FVW+9W6X|MM4ke6}sP{ZGD;uaNptLQP2ty{-MJB88w07W?gMafF~@o<6)vmz71 zEes=!vWw%4Ls_4TT>ObE)XQw~HB{)WT%l0l$Ng$BUe8Vo$6eO4swT|S8ifs5ECj|h zy`ZN%Gx{fqY`^L|<@kKk%!2O)>pCT&nW4kZ^UeCH!d;>k64n7_TlCL@f*}IQmIZ^> z4cSM{lZ;$cFoo1)yGGM}^`0lyY#;SpBJ1+dbr(x^tI0LuF=!{l$qgSsS6wO@FEDZb zBpBf^G2XlIl~M~2#muy`nbqtuw)v!%U$wekIH%OGErPTcGyD#Wtc@DjB1nmtPdLP5 z;E3H-(ilu4oo3QiBdupX28Ced?A%^<)C{D;_0o%2dGH{;M@Nn}OD6hdv*Tr?J=etX zI$I?*1{5A6W!Nc5u;Ya!v=L3^7A?mMbxLb6A@bp$gNufxCvK=D=hTTt(t2^8-eBQj z+)5jQa>&Z^j7QzFrB_0RTKN2M5y1Ftwt3VzQ8$z+cFzi|Y1s`zJnyeh_N>A^9v^m8 z3p)ePX~rK1tT1xCnj+7)xG8~5T)jSLE3K~ZjK(y$7Bl~R4acjh@~lRIuESaXp|G)} zCVe>y^a(Smtnr*+0#){#q+*t2q&tdpS{2bQXm&KO^eSdnlNqr(*=@r683 za=@?E7;wmpYnMAcDMp6&9gaJpfz5#jw!;89jR9_@qVD8w&lQ1z&!Kp%P;B*jilgEC zCxxKhmJ*NI0`S1#F6m%Xkn#vRFo$eVv<}hvU_ZaFZIe?7CeCkhg4K%Bdka&3z=fSf zdI?6_Q=U?eoN&awQ1&n_cCScD#WM2{j=u`G9BWcGVZ{6Zhc+p44zEwSjE6WJoMwjl zJH;jR<*&D3yUC&rC)>F*B>~y`NC^+HYy1H=vjPY#Dk@yIwjcjtx zty=UCD8^GjZ_xKS_N`MaqV+<`MU?sH(Mik@u&$?+A*Y`W3{6vzIXFg2eU=5j#WtwJ zQy)gTA4SToI?jImq}0Em@Hx1^L~65BeT;)oP)vO5R;wtlrZ9CdvIl!PP+poAYT48^ z$QeMu9{;LXL)D7Z6R3kXv2I*$iTGOv-CbQ$jA3Al$XTdZpLzw=unkGf(?bcHx(u0m z8K>#y79mVw!RvQi5#*C)V3!{sT(AsIgmOU}P}Zix4GS>-1`hgIu@AW;^=niF2j6Ri z>NYhGc>@R(Udt_{_U8PTI4{W;QvZbu+>JzT1-HJQT7leg$XUfm>@jKYpy+oZVgL%| zG(CY^xK9+5hk&w7H0IvivTVZNF4set!IBrCkgPyZ)w(uc+IjSEGwhb(2Cug$Z8qHC z$tAB@Nl!)EJuH4~#q8P*6*?evSEb#J_3?8CICV)oih*sM1%hT;3O7zX_+z1rpPMa# zE4+#<18K-B7)IazvE;XcV@hx2&v#miZGT~o6GC_rWo&6 zH7hsJC!5RXihUhve$^3WXSMv*@||j^n?G&N@uJc0MJa1xiyKO-LeuyzYVuE@0O5KW z%menC{Z9P)_C?^zyTWNtqkWu5l*i%h0fgGxopu_{<1)?w;bTj0+8i_wfPi`Ig<~DS zYqq6dL&La)cmj;Er~A<`IHVcQbpWqEIsFjY#YM#PnrW?^KB}K9Rl_k8;MEkR3&_pK z*artW)fMUYA}5_nsKK2A0fRomMv$ti^!J&0LjVW+DYn8fQTo%!(w7Kn;#N{opMDx0 z5>BDOMLh;MAxekiJQxYS2#`*0szt5oPeh~hJM35}eFw_A3CZop0*1<+>3ZafL+6=@ z6i$DDQ3C)NYd4(c!E%zj(_P3*BhE1yvU6|x2IM2CET_~^dI05RWXCDL;4r%_<0iOa zg)10lwUuJuNlFj9$Q<5eE*(lANhp0@&%}^J;fUp3h(z`bs3Gw_90ZX-T~aBeLHO-K znzrPOZ`J&+8)!yu&LiYVk*QQmPLC~}$FAnP1i0?>sGH=B1h{>x6#4Bpbig4J-ol@q zSsJUOH4di^)h3K(Re8F|?h-4T0%bwGW+1-#rMkQNdCH$-*r`rGz{|D)fOWA009OYB zzKRo?+b)|$W_tK%#B#9n4|grW(}mcOc9wiqlCWa zAxG}WT#O7`agH_IzcWvX$D-<3#s!#sA+rXfjtQ4>59s}R<~j6lfPj&RJQvwxCi4^t zP!Ql&D}`@E1mp-N0k8W8ZYw4Sq$vSuYCyWh=$)57rQv~w4K^jsfuNQly(bNAzN`;X z7o5AXtQwoFC%r{kQk1PtDF1gUTKf%G0cx0L0GLtUJ_f{OYK)`~q$f%7Sln`X?TU z9_4&Us57ey@5BQ*CD_)kaMp4B#)d;SvXTNs(%PMs!AL)j&NdlNZF;ku$Q8c_7|V(< z;Gx;(eGpkIM?^aWRl66WJq*1Qr+ACinSfZI?0qrf>TvRR9#WB4kE|3c;_aSO_Kcdv z)-e>m`{{y6J${JBVB^j*h~?AOzyMsE!>&kj!bK3=kwa*NXFUFHXHUwP5G)jv)dgiq zbjQN7!w?~hgi+Yd;w&5E9AXq~XuT}#TIM&q6=urDfrwsS}f zxu?oo7oot;=-GuYEqbsFzOjL!sXR=Rw%W3NkPpkiT&vfXy=0mjF%B?1xtjF#bW)+HIyobOG~AL|0d;Gg#VG7boLyp zz!{EKdDg&x<*rkuhG*8;9~;N;*q=zY)|Vxi6@>txmJ}u3IT+MCE(y*-V*4%=ll81z z{8s-|akCd%HZ&QU585`# dplyr::pull(conceptId) |> unique() # add extra conceptIds to map -conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219) +# 4181730 is the Finnish language concept, used as language_concept_id in CONCEPT_SYNONYM +conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219, 4181730) # For each table we have to add also the concepts to the concept table diff --git a/man/appendUsagiFileToSTCMtable.Rd b/man/appendUsagiFileToSTCMtable.Rd index c2503ea..1281765 100644 --- a/man/appendUsagiFileToSTCMtable.Rd +++ b/man/appendUsagiFileToSTCMtable.Rd @@ -57,6 +57,7 @@ Extended STCM columns (includes all default columns plus): \item source_concept_class \item source_domain \item source_parents_concept_ids +\item source_code_description_fi } Extended Usagi columns: @@ -64,6 +65,7 @@ Extended Usagi columns: \item ADD_INFO:sourceConceptId \item ADD_INFO:sourceConceptClass \item ADD_INFO:sourceDomain +\item ADD_INFO:sourceName_fi (optional, NA if missing) } If formats don't match (e.g., extended Usagi with default STCM), the extended information diff --git a/man/createSourceToConceptMapExtended.Rd b/man/createSourceToConceptMapExtended.Rd index 14ae818..2e4b035 100644 --- a/man/createSourceToConceptMapExtended.Rd +++ b/man/createSourceToConceptMapExtended.Rd @@ -22,5 +22,5 @@ createSourceToConceptMapExtended( } \description{ Creates an extended version of the Source to Concept Map table that includes additional columns -for source concept class, domain, and parent relationships. +for source concept class, domain, parent relationships, and the Finnish source code description. } diff --git a/man/readUsagiFile.Rd b/man/readUsagiFile.Rd index cc63ab9..796130a 100644 --- a/man/readUsagiFile.Rd +++ b/man/readUsagiFile.Rd @@ -47,6 +47,7 @@ Optional ADD_INFO columns: \item sourceValidEndDate (date): End date of validity \item sourceParents (character): Parent concepts \item sourceParentVocabulary (character): Vocabulary of parent concepts +\item sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM \item validationMessagess (character): Validation messages } } diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index 06f74cc..a1f9363 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -37,5 +37,9 @@ If usagi file has C&CR columns: \item Check codes with mapping to more than one domain are mapped to compatible domains \item Check if sourceValidStartDate is before sourceValidEndDate \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +If usagi file has the ADD_INFO:sourceName_fi column: +\item Check if sourceName_fi is less than 1000 characters +\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/man/writeUsagiFile.Rd b/man/writeUsagiFile.Rd index 9441d51..7a941dd 100644 --- a/man/writeUsagiFile.Rd +++ b/man/writeUsagiFile.Rd @@ -44,6 +44,7 @@ The function writes a Usagi mapping file with the following columns: \item ADD_INFO:sourceValidEndDate (date): Source valid end date \item ADD_INFO:sourceParents (character): Source parents \item ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +\item ADD_INFO:sourceName_fi (character): Source name in Finnish \item ADD_INFO:validationMessages (character): Validation messages \item ADD_INFO:autoUpdatingInfo (character): Auto updating info } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md new file mode 100644 index 0000000..0291d04 --- /dev/null +++ b/plan_add_finnish_names.md @@ -0,0 +1,104 @@ +# Finnish names from Usagi files to `CONCEPT_SYNONYM` + +The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. + +## Decisions +- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. +- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. +- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. +- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. +- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. +- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. +- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. + +## Data flow + +``` +Usagi CSV STCM (extended) CDM +ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM + concept_id = source_concept_id + concept_synonym_name = source_code_description_fi + language_concept_id = 4181730 (Finnish) +``` + +- The language is hardcoded to Finnish (`4181730`). +- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. +- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). + +## Changes by step + +### 1. Reading and writing Usagi files (`R/usagiFile.R`) +- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. +- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. + +### 2. Validation (`R/validateUsagiFile.R`) +- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. +- New checks: + +| Step in the log | Level | Rule | +|---|---|---| +| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | +| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | +| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | + +- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. + +### 3. STCM table +- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. +- `R/appendUsagiFileToSTCMtable.R`: + - The column is part of the extended STCM columns. + - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. +- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. + +### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) +- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. +- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: + - `source_concept_id = 0` + - the name is NULL or blank + - the name equals `source_code_description` +- Re-running it gives the same result. + +### 5. Post-load check (`R/buildVocabuliesAll.R`) +- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: + - the expected names in the STCM table, using the same filters as the SQL; + - the Finnish rows in `CONCEPT_SYNONYM`. +- It logs SUCCESS if the counts match and ERROR otherwise. + +### 6. Other +- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. +- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. + +## Tests +- `test-validateUsagiFile.R`: + - The too-long, differing-within-sourceCode and empty checks each fire. + - The C&CR columns are required when `sourceName_fi` is present. +- `test-appendUsagiFileToSTCMTable.R`: + - The new column appears in the extended STCM table and gets its value. + - It is NA when the Usagi file has no Finnish column. +- `test-STCMToCDMTable.R`: + - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. + - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. + - Running it twice gives the same result. + - An existing English synonym is preserved. +- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. +- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. +- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. + +Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. + +## Files changed +- `R/`: + - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` + - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` +- `inst/`: + - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` + - `sql/sql_server/STCMExtendedToCDM.sql` + - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` + - `testdata/createTestData.R` +- `man/`: + - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` + - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` +- `tests/testthat/`: + - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` + - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` +- `NEWS.md` diff --git a/tests/testthat/test-STCMToCDMTable.R b/tests/testthat/test-STCMToCDMTable.R index 6584227..4042d07 100644 --- a/tests/testthat/test-STCMToCDMTable.R +++ b/tests/testthat/test-STCMToCDMTable.R @@ -25,6 +25,7 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre "code9", 2000000009, "TestVocab", "Test Code 9 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002", "code10", 2000000010, "TestVocab", "Test Code 10 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002|2000000003" ) + testData <- testData |> dplyr::mutate(source_code_description_fi = NA_character_) # Create connection to test database connection <- DatabaseConnector::connect( @@ -113,3 +114,80 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre res |> dplyr::pull(concept_id_2) |> expect_equal(c(2000000001, 2000000001, 2000000002, 2000000001, 2000000002, 2000000003)) }) + +test_that("STCMToCDMTables creates Finnish CONCEPT_SYNONYM entries from source_code_description_fi", { + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + sourceToConceptMapTable <- "source_to_concept_map_extended" + + testData <- tibble::tribble( + ~source_code, ~source_concept_id, ~source_vocabulary_id, ~source_code_description, ~target_concept_id, ~target_vocabulary_id, ~valid_start_date, ~valid_end_date, ~invalid_reason, ~source_concept_class, ~source_domain, ~source_parents_concept_ids, ~source_code_description_fi, + # multi-mapped code, one synonym + "code1", 2000000001, "TestVocab", "Test Code 1", 141797, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + "code1", 2000000001, "TestVocab", "Test Code 1", 36713461, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + # unmapped code also gets synonym + "code2", 2000000002, "TestVocab", "Test Code 2", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 2", + # same name as concept_name is not added as synonym + "code3", 2000000003, "TestVocab", "Test Code 3", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Test Code 3", + # no synonym + "code4", 2000000004, "TestVocab", "Test Code 4 no fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, NA_character_, + "code5", 2000000005, "TestVocab", "Test Code 5 blank fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, " ", + # source_concept_id 0 is skipped + "code6", 0, "TestVocab", "Test Code 6 no source concept id", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 6" + ) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + createSourceToConceptMapExtended( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable, + overwrite = TRUE + ) + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = sourceToConceptMapTable, + data = testData + ) + + # existing synonyms: an old Finnish one to be replaced and an English one to be kept + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = "CONCEPT_SYNONYM", + data = tibble::tribble( + ~concept_id, ~concept_synonym_name, ~language_concept_id, + 2000000001, "Vanha nimi", 4181730, + 2000000001, "Old English synonym", 4180186 + ), + dropTableIfExists = FALSE, + createTable = FALSE + ) + + # run twice to check it is idempotent + for (i in 1:2) { + STCMToCDMTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + } + + res <- dplyr::tbl(connection, "CONCEPT_SYNONYM") |> + dplyr::filter(concept_id >= 2000000000 | concept_id == 0) |> + dplyr::arrange(concept_id, language_concept_id) |> + dplyr::collect() + + finnish <- res |> dplyr::filter(language_concept_id == 4181730) + finnish |> dplyr::pull(concept_id) |> expect_equal(c(2000000001, 2000000002)) + finnish |> dplyr::pull(concept_synonym_name) |> expect_equal(c("Testikoodi 1", "Testikoodi 2")) + + # other languages are not touched + res |> dplyr::filter(language_concept_id != 4181730) |> dplyr::pull(concept_synonym_name) |> expect_equal("Old English synonym") +}) diff --git a/tests/testthat/test-appendUsagiFileToSTCMTable.R b/tests/testthat/test-appendUsagiFileToSTCMTable.R index 6749c9c..2fb6846 100644 --- a/tests/testthat/test-appendUsagiFileToSTCMTable.R +++ b/tests/testthat/test-appendUsagiFileToSTCMTable.R @@ -81,13 +81,19 @@ test_that("test appendUsagiFileToSTCMTable appends the usagi file to the sourceT expect_equal(c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", "invalid_reason", "source_concept_class", - "source_domain", "source_parents_concept_ids")) + "source_domain", "source_parents_concept_ids", "source_code_description_fi")) stcmTable |> dplyr::filter(is.na(source_parents_concept_ids)) |> nrow() |> expect_equal(0) + # source_code_description_fi comes from ADD_INFO:sourceName_fi + stcmTable |> + dplyr::filter(source_code == "A01.0+G01") |> + dplyr::pull(source_code_description_fi) |> + expect_equal("Lavantautiin liittyvä aivokalvotulehdus") + }) test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the sourceToConceptMapTable with ICD10 parent information", { @@ -130,3 +136,34 @@ test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the expect_equal("2000503725") }) + +test_that("test appendUsagiFileToSTCMTable sets source_code_description_fi to NA when the usagi file has no ADD_INFO:sourceName_fi", { + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-`ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + sourceToConceptMapTable <- "source_to_concept_map_extended" + createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) + + appendUsagiFileToSTCMtable( + vocabularyId = "ICD10fi", + pathToUsagiFile = pathToUsagiFile, + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + + stcmTable <- DBI::dbReadTable(connection, sourceToConceptMapTable) |> tibble::as_tibble() + stcmTable |> nrow() |> expect_equal(nrow(usagiTibble)) + stcmTable |> dplyr::pull(source_code_description_fi) |> is.na() |> all() |> expect_true() +}) diff --git a/tests/testthat/test-buildVocabulariesAll.R b/tests/testthat/test-buildVocabulariesAll.R index c0fcf28..8e0416a 100644 --- a/tests/testthat/test-buildVocabulariesAll.R +++ b/tests/testthat/test-buildVocabulariesAll.R @@ -28,6 +28,9 @@ test_that("buildVocabulariesAll works", { # Check results validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) + # Finnish names were added to CONCEPT_SYNONYM + validationLogTibble |> dplyr::filter(step == "Finnish names in CONCEPT_SYNONYM") |> dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + # check the validation results folder expect_true(!file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) }) diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index 8a5d44b..fcb4111 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -306,3 +306,92 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { + +test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + pathToValidatedUsagiFile <- tempfile(fileext = ".csv") + vocabularyDatabaseSchema = "main" + sourceConceptIdOffset = 2000500000 + + # make a copy of a valid usagi file with errors in ADD_INFO:sourceName_fi + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = dplyr::case_when( + # too long + sourceCode == "A01.0+G01" ~ strrep("a", 1001), + # empty + sourceCode == "A01.0+I39.8" ~ NA_character_, + # different within a multi-mapped sourceCode + sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", + TRUE ~ `ADD_INFO:sourceName_fi` + ) + ) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile( + pathToUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile, + sourceConceptIdOffset + ) + + validatedUsagiFile <- readUsagiFile(pathToValidatedUsagiFile) + + # only the sourceName_fi validations fail + validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> + expect_setequal(c( + "SourceNameFi is more than 1000 characters", + "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is empty" + )) + + # SourceNameFi is more than 1000 characters + validationsSummary |> dplyr::filter(step == "SourceNameFi is more than 1000 characters") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceNameFi is more than 1000 characters") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") + + # SourceNameFi is not the same for all rows of the sourceCode + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) + + # SourceNameFi is empty, only a warning, mappingStatus is not changed + validationsSummary |> dplyr::filter(step == "SourceNameFi is empty") |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("WARNING: SourceNameFi is empty") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(mappingStatus) |> expect_equal("APPROVED") +}) + +test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName_fi is present", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-dplyr::starts_with("ADD_INFO:"), `ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") +}) diff --git a/tests/testthat/test-validateVocabularyFolder.R b/tests/testthat/test-validateVocabularyFolder.R index 597b028..95aa62b 100644 --- a/tests/testthat/test-validateVocabularyFolder.R +++ b/tests/testthat/test-validateVocabularyFolder.R @@ -23,7 +23,12 @@ test_that("validateVocabularyFolder returns all SUCCESS for valid vocabulary fol ) # Check validation results - validationsLogTibble |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) + # only warning: UNITfi has the ADD_INFO:sourceName_fi column but all values are empty + notSuccess <- validationsLogTibble |> dplyr::filter(type != "SUCCESS") + notSuccess |> nrow() |> expect_equal(1) + notSuccess |> dplyr::pull(context) |> as.character() |> expect_equal("UNITfi") + notSuccess |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + notSuccess |> dplyr::pull(step) |> expect_equal("SourceNameFi is empty") # Check that validated files were created expect_true(dir.exists(pathToValidatedVocabularyFolder))