From 9ac4180986c62466bd0cf42a6e5a26c2ec9fd75a Mon Sep 17 00:00:00 2001 From: javier-gracia-tabuenca-tuni Date: Tue, 10 Nov 2026 10:59:58 +0200 Subject: [PATCH 1/3] Add Finnish names from Usagi files to CONCEPT_SYNONYM Carry the optional ADD_INFO:sourceName_fi Usagi column through the new source_code_description_fi column of the extended STCM table into CONCEPT_SYNONYM with language_concept_id = 4181730 (Finnish). - Validate sourceName_fi: max 1000 characters and the same for all rows of a sourceCode (ERROR), warn if empty for APPROVED/UNCHECKED rows - Skip empty names and names equal to sourceName when inserting synonyms - Only Finnish synonyms are deleted and reinserted; other languages kept - buildVocabulariesAll checks all Finnish names reach CONCEPT_SYNONYM - Use standard SQL '=' in calculateMappingStatus synonym query - Add Finnish language concept 4181730 to the test vocabulary Co-Authored-By: Claude Opus 5.5 --- NEWS.md | 7 ++ R/appendUsagiFileToSTCMtable.R | 12 +- R/buildVocabuliesAll.R | 42 +++++++ R/calculateMappingStatus.R | 2 +- R/createSourceToConceptMapExtended.R | 2 +- R/usagiFile.R | 14 ++- R/validateUsagiFile.R | 62 ++++++++++- .../createSourceToConceptMapExtended.sql | 3 +- inst/sql/sql_server/STCMExtendedToCDM.sql | 40 ++++++- .../OMOPVocabulary/OMOPVocabulary.duckdb | Bin 6303744 -> 6565888 bytes inst/testdata/createTestData.R | 3 +- man/appendUsagiFileToSTCMtable.Rd | 2 + man/createSourceToConceptMapExtended.Rd | 2 +- man/readUsagiFile.Rd | 1 + man/validateUsagiFile.Rd | 4 + man/writeUsagiFile.Rd | 1 + plan_add_finnish_names.md | 104 ++++++++++++++++++ tests/testthat/test-STCMToCDMTable.R | 78 +++++++++++++ .../test-appendUsagiFileToSTCMTable.R | 39 ++++++- tests/testthat/test-buildVocabulariesAll.R | 3 + tests/testthat/test-validateUsagiFile.R | 89 +++++++++++++++ .../testthat/test-validateVocabularyFolder.R | 7 +- 22 files changed, 504 insertions(+), 13 deletions(-) create mode 100644 plan_add_finnish_names.md diff --git a/NEWS.md b/NEWS.md index 4b57762..a8d1eed 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +# ROMOPMappingTools (development version) + +- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. +- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. + # ROMOPMappingTools 2.1.3 - Added rule to validate usagi file: sourceConceptId must be unique diff --git a/R/appendUsagiFileToSTCMtable.R b/R/appendUsagiFileToSTCMtable.R index 8b2aacb..3072e82 100644 --- a/R/appendUsagiFileToSTCMtable.R +++ b/R/appendUsagiFileToSTCMtable.R @@ -12,11 +12,13 @@ #' - source_concept_class #' - source_domain #' - source_parents_concept_ids +#' - source_code_description_fi #' #' Extended Usagi columns: #' - ADD_INFO:sourceConceptId #' - ADD_INFO:sourceConceptClass #' - ADD_INFO:sourceDomain +#' - ADD_INFO:sourceName_fi (optional, NA if missing) #' #' If formats don't match (e.g., extended Usagi with default STCM), the extended information #' will be ignored with a warning. @@ -83,7 +85,8 @@ appendUsagiFileToSTCMtable <- function( stcmExtendedColumns <- c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", - "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids" + "invalid_reason", "source_concept_class", "source_domain", "source_parents_concept_ids", + "source_code_description_fi" ) if (all(stcmExtendedColumns %in% stcmTableColumns)) { @@ -150,6 +153,10 @@ appendUsagiFileToSTCMtable <- function( if (!"ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceValidEndDate` = lubridate::as_date(ymd("2099-12-31"))) } + # if colum `ADD_INFO:sourceName_fi` is not present, set it to NA + if (!"ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + usagiTibble <- usagiTibble |> dplyr::mutate(`ADD_INFO:sourceName_fi` = NA_character_) + } # if colums `ADD_INFO:sourceParents` and `ADD_INFO:sourceParentVocabulary` are present and any of the `ADD_INFO:sourceParents` is not NA if (all(c("ADD_INFO:sourceParents", "ADD_INFO:sourceParentVocabulary") %in% usagiTibbleColumns) && @@ -227,7 +234,8 @@ appendUsagiFileToSTCMtable <- function( invalid_reason = NA_character_, source_concept_class = `ADD_INFO:sourceConceptClass`, source_domain = `ADD_INFO:sourceDomain`, - source_parents_concept_ids = sourceParentConceptIds + source_parents_concept_ids = sourceParentConceptIds, + source_code_description_fi = `ADD_INFO:sourceName_fi` ) } diff --git a/R/buildVocabuliesAll.R b/R/buildVocabuliesAll.R index d5b1344..f566fd3 100644 --- a/R/buildVocabuliesAll.R +++ b/R/buildVocabuliesAll.R @@ -111,6 +111,48 @@ buildVocabulariesAll <- function( return(validationLogTibble) } + # check all the Finnish names in the SourceToConceptMap table are in the CONCEPT_SYNONYM table + nFinnishNamesInSTCM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM ( + SELECT DISTINCT source_concept_id, source_code_description_fi + FROM @vocabulary_database_schema.@source_to_concept_map_table + WHERE source_concept_id != 0 + AND source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(source_code_description_fi)) != '' + AND (source_code_description IS NULL OR source_code_description_fi != source_code_description) + ) AS t", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + nFinnishSynonymsInCDM <- DatabaseConnector::renderTranslateQuerySql( + connection, + "SELECT COUNT(*) AS n FROM @vocabulary_database_schema.CONCEPT_SYNONYM + WHERE language_concept_id = 4181730 + AND concept_id IN (SELECT DISTINCT source_concept_id FROM @vocabulary_database_schema.@source_to_concept_map_table)", + vocabulary_database_schema = vocabularyDatabaseSchema, + source_to_concept_map_table = sourceToConceptMapTable + )[[1]] + + if (nFinnishNamesInSTCM != nFinnishSynonymsInCDM) { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "ERROR", + step = "Finnish names in CONCEPT_SYNONYM", + message = paste0( + "Found ", nFinnishSynonymsInCDM, " Finnish synonyms in CONCEPT_SYNONYM but ", + nFinnishNamesInSTCM, " Finnish names in the SourceToConceptMap table" + ) + )) + } else { + validationLogTibble <- dplyr::bind_rows(validationLogTibble, dplyr::tibble( + context = "STCMToCDMTables", + type = "SUCCESS", + step = "Finnish names in CONCEPT_SYNONYM", + message = "" + )) + } + # create the ancestor tables message("Creating the ancestor tables") errorMessage <- "" diff --git a/R/calculateMappingStatus.R b/R/calculateMappingStatus.R index acee366..8294e81 100644 --- a/R/calculateMappingStatus.R +++ b/R/calculateMappingStatus.R @@ -90,7 +90,7 @@ calculateMappingStatus <- function( ON c.concept_id = cr.concept_id_1 LEFT JOIN ( SELECT concept_id, concept_synonym_name FROM @vocabulary_database_schema.CONCEPT_SYNONYM - WHERE language_concept_id == 4181730 + WHERE language_concept_id = 4181730 ) AS cs ON c.concept_id = cs.concept_id " diff --git a/R/createSourceToConceptMapExtended.R b/R/createSourceToConceptMapExtended.R index d20a236..4ba52dc 100644 --- a/R/createSourceToConceptMapExtended.R +++ b/R/createSourceToConceptMapExtended.R @@ -1,7 +1,7 @@ #' Create Extended Source to Concept Map Table #' #' Creates an extended version of the Source to Concept Map table that includes additional columns -#' for source concept class, domain, and parent relationships. +#' for source concept class, domain, parent relationships, and the Finnish source code description. #' #' @param connection A DatabaseConnector connection object #' @param vocabularyDatabaseSchema Schema name where the vocabulary tables are stored diff --git a/R/usagiFile.R b/R/usagiFile.R index 5c2c0ff..c5cb850 100644 --- a/R/usagiFile.R +++ b/R/usagiFile.R @@ -30,6 +30,7 @@ #' - sourceValidEndDate (date): End date of validity #' - sourceParents (character): Parent concepts #' - sourceParentVocabulary (character): Vocabulary of parent concepts +#' - sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM #' - validationMessagess (character): Validation messages #' #' @param pathToUsagiFile Path to the Usagi mapping file to read @@ -69,6 +70,7 @@ readUsagiFile <- function(pathToUsagiFile) { `ADD_INFO:sourceValidEndDate` = readr::col_date(), `ADD_INFO:sourceParents` = readr::col_character(), `ADD_INFO:sourceParentVocabulary` = readr::col_character(), + `ADD_INFO:sourceName_fi` = readr::col_character(), `ADD_INFO:validationMessages` = readr::col_character(), # .default = readr::col_character() @@ -110,6 +112,7 @@ readUsagiFile <- function(pathToUsagiFile) { #' - ADD_INFO:sourceValidEndDate (date): Source valid end date #' - ADD_INFO:sourceParents (character): Source parents #' - ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +#' - ADD_INFO:sourceName_fi (character): Source name in Finnish #' - ADD_INFO:validationMessages (character): Validation messages #' - ADD_INFO:autoUpdatingInfo (character): Auto updating info #' @@ -147,7 +150,8 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { "ADD_INFO:sourceValidStartDate", "ADD_INFO:sourceValidEndDate", "ADD_INFO:sourceParents", - "ADD_INFO:sourceParentVocabulary" + "ADD_INFO:sourceParentVocabulary", + "ADD_INFO:sourceName_fi" ) lastColNames <- c( @@ -164,7 +168,7 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { dplyr::any_of(c(firstColNames, midColNames, lastColNames)) ) - # force "ADD_INFO:validationMessages" and "ADD_INFO:autoUpdatingInfo" to be NA if they are empty, if they exist + # force "ADD_INFO:validationMessages", "ADD_INFO:autoUpdatingInfo" and "ADD_INFO:sourceName_fi" to be NA if they are empty, if they exist if ("ADD_INFO:validationMessages" %in% colNames) { usagiTibble <- usagiTibble |> dplyr::mutate( @@ -177,6 +181,12 @@ writeUsagiFile <- function(usagiTibble, pathToUsagiFile) { `ADD_INFO:autoUpdatingInfo` = ifelse(`ADD_INFO:autoUpdatingInfo` == "", NA, `ADD_INFO:autoUpdatingInfo`) ) } + if ("ADD_INFO:sourceName_fi" %in% colNames) { + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = ifelse(`ADD_INFO:sourceName_fi` == "", NA, `ADD_INFO:sourceName_fi`) + ) + } readr::write_csv(usagiTibble, pathToUsagiFile, na = "") } diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 13ba240..60d49fc 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -14,6 +14,10 @@ #' - Check codes with mapping to more than one domain are mapped to compatible domains #' - Check if sourceValidStartDate is before sourceValidEndDate #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +#' If usagi file has the ADD_INFO:sourceName_fi column: +#' - Check if sourceName_fi is less than 1000 characters +#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate #' @param connection A DatabaseConnector connection object @@ -202,7 +206,8 @@ validateUsagiFile <- function( # if it has any of the 3 columns it must have the other 2 if ("ADD_INFO:sourceConceptId" %in% usagiTibbleColumns || "ADD_INFO:sourceConceptClass" %in% usagiTibbleColumns || "ADD_INFO:sourceDomain" %in% usagiTibbleColumns || "ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns || - "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns) { + "ADD_INFO:sourceParents" %in% usagiTibbleColumns || "ADD_INFO:sourceParentVocabulary" %in% usagiTibbleColumns || + "ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { missingColumns <- c( "ADD_INFO:sourceConceptId", @@ -347,6 +352,61 @@ validateUsagiFile <- function( } } + # - Check ADD_INFO:sourceName_fi, it will be used as Finnish CONCEPT_SYNONYM + # (whitespace-only names are read as NA by readUsagiFile, so they are reported as empty) + if ("ADD_INFO:sourceName_fi" %in% usagiTibbleColumns) { + validationRules <- validate::validator( + SourceNameFi.is.more.than.1000.characters = field_length(`ADD_INFO:sourceName_fi`, min = 0, max = 1000) + ) + validations <- validate::confront(usagiTibble, validationRules) + + result <- .applyValidationRules(usagiTibble, validations, validationLogR6) + usagiTibble <- result$fileTibble + validationLogR6 <- result$validationLogR6 + + # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) + sourceCodesWithDifferentNameFi <- usagiTibble |> + dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> + dplyr::count(sourceCode) |> + dplyr::filter(n > 1) |> + dplyr::pull(sourceCode) + + if (length(sourceCodesWithDifferentNameFi) > 0) { + validationLogR6$ERROR( + "SourceNameFi is not the same for all rows of the sourceCode", + paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + sourceCode %in% sourceCodesWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + } + + # warn if sourceName_fi is empty, the column is optional so this is not an error + isEmptyNameFi <- usagiTibble$mappingStatus %in% c("APPROVED", "UNCHECKED") & is.na(usagiTibble$`ADD_INFO:sourceName_fi`) + + if (any(isEmptyNameFi)) { + validationLogR6$WARNING( + "SourceNameFi is empty", + paste0("Found ", sum(isEmptyNameFi), " APPROVED or UNCHECKED rows with empty sourceName_fi") + ) + + usagiTibble <- usagiTibble |> + dplyr::mutate(tmpvalidationMessages = dplyr::if_else( + isEmptyNameFi, + paste0(tmpvalidationMessages, " | ", "WARNING: SourceNameFi is empty"), + tmpvalidationMessages + )) + } else { + validationLogR6$SUCCESS("SourceNameFi is empty", "") + } + } + # - Check if ADD_INFO:sourceValidStartDate and ADD_INFO:sourceValidEndDate are used if ("ADD_INFO:sourceValidStartDate" %in% usagiTibbleColumns || "ADD_INFO:sourceValidEndDate" %in% usagiTibbleColumns) { missingColumns <- diff --git a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql index 1693a3a..e964d5a 100644 --- a/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql +++ b/inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql @@ -18,5 +18,6 @@ CREATE TABLE @vocabularyDatabaseSchema.@sourceToConceptMapTable ( invalid_reason varchar(1) NULL , source_concept_class varchar(20) NULL, source_domain varchar(20) NULL, - source_parents_concept_ids varchar(255) NULL + source_parents_concept_ids varchar(255) NULL, + source_code_description_fi varchar(1000) NULL ); \ No newline at end of file diff --git a/inst/sql/sql_server/STCMExtendedToCDM.sql b/inst/sql/sql_server/STCMExtendedToCDM.sql index c2328bf..f011e88 100644 --- a/inst/sql/sql_server/STCMExtendedToCDM.sql +++ b/inst/sql/sql_server/STCMExtendedToCDM.sql @@ -300,4 +300,42 @@ SELECT DISTINCT ) AS valid_end_date, NULL AS invalid_reason FROM split_parents AS sp -WHERE sp.source_parents_concept_ids IS NOT NULL +WHERE sp.source_parents_concept_ids IS NOT NULL; + + +-- 5. CONCEPT_SYNONYM table +-- Only Finnish synonyms (language_concept_id = 4181730) are managed here, synonyms in other languages are not touched +-- Delete previous Finnish rows with the same concept_id as in the source_to_concept_map.source_concept_id +-- Insert one row for each **unique** source_concept_id with a non-empty source_code_description_fi +-- that is different from source_code_description (same name as concept_name adds nothing as synonym) +-- - concept_id = source_to_concept_map.source_concept_id +-- - concept_synonym_name = source_to_concept_map.source_code_description_fi +-- - language_concept_id = 4181730 (Finnish) +DELETE FROM + @vocabularyDatabaseSchema.CONCEPT_SYNONYM +WHERE + language_concept_id = 4181730 + AND concept_id IN ( + SELECT + DISTINCT stcm.source_concept_id + FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm + ); + +INSERT INTO + @vocabularyDatabaseSchema.CONCEPT_SYNONYM ( + concept_id, + concept_synonym_name, + language_concept_id + ) +SELECT DISTINCT + stcm.source_concept_id AS concept_id, + stcm.source_code_description_fi AS concept_synonym_name, + 4181730 AS language_concept_id +FROM + @vocabularyDatabaseSchema.@sourceToConceptMapTable AS stcm +WHERE + stcm.source_concept_id != 0 + AND stcm.source_code_description_fi IS NOT NULL + AND LTRIM(RTRIM(stcm.source_code_description_fi)) != '' + AND (stcm.source_code_description IS NULL OR stcm.source_code_description_fi != stcm.source_code_description); diff --git a/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb b/inst/testdata/OMOPVocabulary/OMOPVocabulary.duckdb index 9e8753d179ed4a768824baf2b2d8da83e87043ed..61e9209c0924c2429c6800092de45514698581e7 100644 GIT binary patch delta 13254 zcmbVT3wTt;_1}AUvyW^@vOGd~u*IdkUBP5p^_bNz`%^S1cZu9`nR_KXfPYEU?nkYt64X zcBI4)N{+kchT9(+`JzJ|ld{w_j@CC92BZOwpvfK%FEg2zn@lU{(QSopSz19lrkgWK z%EN^R6Yj28KbD~BqrqZeQ|iQVPww>xCDCsSn(j*M)lb_WExcb!)Cmj-nm&nT1712N zv9%b9ri=+62?|4krsilPg*QmmuEHS_A4#|FDjX}R-^r=R3wMZVi9u8Q0OPbQdguv^ zB@1Vo?Xkmyruy^P%GzBxTqd4tnpN)ry;p4{zH534dtPbzA6$@l*G!F#DNa)J zbg9DFct@wUulDMzzy>|KK6ismjATsS86?X7ukD{0wZs<7v0qYM)}9GEcn zUfJ+Sr)8726e%{E9IJFJs8)&I%C=9mLWP-_bRT(NNp0F~VjV3~A~p^NkX{xzc+~MRvOQ*X(`Ct{V|L zA7hh-Q7KXK7+LFxrqkQg*GG^NF?N(p0-e&8ENXr4Fwpr!6wUAU?lppFqC*5cG%L@86q1JloN(3)zq#mJl<)SKw?HRT^h6HNIBQ}#ti zr`Skz0x}&ITDj4@p6>yFG2(oujn3O@englSs=CZc%lDYALM$Dd0RBdqPH*!Z6xal| zGpsal*qm;p`GSMycbV4-33a`IK)gh=Pnp+q6+9*805#1V+%XdMobznc4X2_8-n4`5 z0hSoSC0KgP$+iaa$%4XFItX<}0pr0@A;l6>@Y5PnY$3%SQXC;AI;1#5N=!%@5K>}8 zN?b^Z4=JvYGBBhJ3Mqp_%8-zf5K@MQl*EuSETjw%DI-G4$dEEBq$Guu|+gnQb&&o;K8or@u$j)0JuU z7gGFSC=*jiPfglN7GD0wO>og*utYcm4%bD|6Lo1X2=&9}>7d70>TXC&7BE!Jg4G_; zxhHVCRrE-6nq6jGysmQ5Pb_kkNX(6az)wJytmwu__EI{qT}cNH6w2l7Yp|BJVAHuGWja2=Er35Th!y|3z*a7Sd21v{B)Y;_PFl5lQmDSZNf3+gz zkjbupB`Kwg23vk&&bIZ5nX{^@m)}&iJcv2^*T0!ee>X|5vV6MFs`0&64Vosfg)^@# zD=)pQ*kl3`{i{yIdJ#k_IQO$}s=9vVvXzVz{VTcsyTSNVa`n zI4sjQF@K^rPfFj=3 zcezPgp&Hmkj=q*Xmc;*9XtQYp$=3O$X=LjoQj8gg4C2}@4Pfy}5~37ADO>occKn?Oxvl*wklaPk&^i33FPe`=rhDG(*8Fb>v(7!BQ!lQsoauL@K z>3mLZV+V@8I9B|pw1V^FBlUswuUosMeEw$%JyI5MnI(z5<_e^f_79|}2p5yFB{DCq zD^EYpz1zigIeB1jftfBz)}@mag0*niz(HkGN__7>aLzcdn?GTv41%)|W{Cle2;VSp zmdH4diblHE66y&@>*2Kc5B`|ZqT|1kyf&#!#X$YTlEY{%&is;TWeRDWTDF9lq>(6Z zO94__nVvz40%eWNidruS!=9IwHEwa@3&!HdWXxhVWnwH*i5z@yI==l9W&A;jq_;U5FmN9&E>{zt@8Y zoZ-uQKe-P_u3kgxapdpy;1nL9tiYZcwo~k#uW5j;Y)HSk&vu1Wdzefsn$w5!1Mvv4 z&tbo!*!-Eg%UjY{^(ZQuUFOqK3*JI~nW3GT7pdNqq7**)pY81q*dJI6X{N;rLtP)FZXg5uGQ z+D4un=UyGvn+DV~P}|PqvfVzuonPE};dc(Wy<)Rxnnqk(Q@wO0*)*%vL$*ya2g#o4 zDA*(iP5(OYe7$pT@jGTw??G;G_pn~aV0z%G;vo{bp%~K(gQnKrhh!SpT8!L%>tlX< ztcBQMy zyi;;eDIh<`oy~AWW_Es4wR5=4z0l0^v%wme=eR+Ds`BOi;LBR`iv)=b;I(ft9(_G*!l!L)Z;>5;AO zmqqdmtd7>asB@1y{e0OdWV=qH2ix2$ISy|&(sHO?MA{F#UtnBEIzbijX>XT%rw{~L zLMth zTuo~idp<|WB~gkO{dj%~fAFYZ?U}~CFj4e^?wvNeWR2%(9^n&=9mBOOy>g?+C&a)6 zy|OkJVQ;JF9Uk`5j1#}mPlyM;?v+h84oBgnVUmza!#gS<2w|-Q(UONe} z(xV-1u=ZO&7O>}5w1L`hJ=15;ki?KI)29jyu-D$&UX?j^ZVM}dYmF8` zrG@a7n&Z4O_vt-i7W#Y^+L-LE=H3wLfUmq}pov_*%~{4;I=a51CwEnoqim&8alcqueFO;+E)xgvh2`b!P5+a>6~@ zjy)!5dUu5AxB^qaYFM@CEvntG7$YIkFt z+Lnj?<0Z23ySYdi==SgC9+DD>wl)%%FOmD|H);DG^Z!#qX6fzFH*)46V^u7dw|GYi z3kv0V?4+pOdzWrU@jRkbV0xg6=|HE~CyFb8#$GG(K<`!~nrEtL&hGJk+Xu}Iff7jx zdn=e_mWzE1hWBwW)8shcKqKcz5Yv-=zZF;px|y4?yk?j^1`!FLR9f|Z?gZ_kiJDsj zJ|a>aP`*NiZ@A3&A0a%*oJP=C>|1Q)vE-un`GJvr9-O#TME6wta*QIiprW?M_n^Qs z3{mRfuGGEJ_l-cY?w%m@01_adn_t%5qf-)rs!F5ebmMyTjY)wxs4L4EVie&u z1dhcyYXtjH!*wbWYpQeJ5mxStZlT|Jb*n&So#?t}#}Ty#{U^1va%0Z7x;@p2!0{A2 za+;j$g>(jkX6hK7NRLiQcQcGaghOUzGdXt%E5ZWHRgeY_=Nva$GRlekC1;voUpFHh zX|@Ww>{B^k8};x-6iqpoQ>in_?m5@1pi7SRAK<3!FBTs7YCMaDx(Hb32_6#Z^170z zczO3z!8ZJVlGQQ0_*mD>+NoC!pqp^_&y$|`h!t+hd)?AnCwg6<2B))HzO7B|4vu{w5x0??Uo+@ zheqUtqfyxQ&l0>FeD`u$ozfha`-2fWA4eep$PF5yb0^n@jW%TGJ}<%mz|pU-s_8J3 z;!#>3$Q{Mw+1*BkOlA8lj8s>iD+ytMiT7>T{d0dIU1LpN>8yHwkDpF(M2ao!-FJgyU30Pp1*K6|S3&Kg%a8I8>`Z>hki z9@&7J9L;py8}rul$Pgh2z#qkHGF$U*6(|L4YBZE~?#cUyu$-m`2E)H#hs=xP`Tw1Q zH|Tq=1v_EhS^lU$cGBYLKuplIq89;j-Tft7#eCF*&dA38ab(&9 zC1;!NDdc6RmdGngboTE{{wj(0MXW*7K2$`FS!T(DB_GK+PVDQNJCGYP5}BanXhsFIaGN7#!;m5Z&j7~ZmheFv$Y zN!Lx9_c)LF>jZECn0il_{xN4k-A4SSp z7@loL6(pb4Solj^Vl%)yO=|4fn18#FTPWl?E7FDhlYNj6Klyr6OdSLzc&%EXZJvygJBqvn$7KURJ4_~V6CIIsNfZ0e%)Q6g;#OH_S;Cfyg*^0YbLKC_UE74TWC;Jz{2L*>DfxZMF@z+ zj@wWnD5#DI*q!!UC(iu^Z|fZCct%C3UFHnn+Ph%C#11NN3)o(yxd{imh3nuaE>sx}B07RaoPH*olsOj7LCW2IV!P~;dP?U?+AfTa^ zo`O8ae}B5^ozSri$%m?BxFDTN_LW!8pP^FRYFo7&Cp<$ot3Qc8BAS2m&Ryt ze$}k%p{AyAEwj^q1yP{!7dBOz<#kf8lsd-AE;s|+{r*3fDh9h|$m=Sa3(+Q_1<;Dn zCZkP3n~GM9b{X0)iwIV|vDRO?xl5wn~1soPaSPaBe zyhhi-QPZCzXfMEFQIH0(rKzc@eUC*g>xO$lz_MFJ`xgQ{*Ri3$F{m>s1jc64jC7z zP-v2|OejfvNhBO58~*5jK~$04LDLVoQZ;4+_w4e|Hj4qipy~PZF1Uv8_V1BN(GPR2 zO^Eg+TvISG;eIWx!HX^%U4kCL``wwOytd$D^6gs-hD5Dk7pAo8-bLT=G~+6kjshbY zv?^83(R}+^%?2;GW%WAmEKsqF_XQW66v!2NoUirLF1I-PTp&^*$p1`*GF}y&i}%YA zJgmafEEK9N;qAbz8-+b!=W3R@fYoeB^65<6zbER-sRliVD?}IOsZ@sFJdiAwP&?BG?8&} z(ir!@(c@5?xsQWl3!AEfL%MV)6e;9lkxdUnUFKy*sa9)YsF1DnvhYxY&%dIFp>x;N z3I9X@x>f^cfX)2Sh=p@@1lDxCTcZUeofyE^d79E+;EI%|7uzr=b+i02cN?o%8NmD@T1tHKOl4Ug(O`+oWlTF$S#Z+MA}Ib% zKxJ6$$bl%#eZQ_+2q&!fE_%qmuD~)j@fSKA`Z^Q~Eze?+qs_Cvho%#yCS57;r3Dt= zQ(OpZCrVH1Jz)AQlc%FS?F!+@K=u|V)=X+FOkkt8>!S_I?PlWa>-Jw~H2iH=yY`PL z{5s^2(fEnb4c#72^XUSxq8Pud0ozp)3J8lXuZ@*_&oMu zvYMFidp7o!U6ZkolWb~ORF~{j|1#O|J6ij$af-jfq^F^92wcb#j~^4FUG;GX+r!dO yZD2S&d6SEEUv1C8g;Vlb{BUV~=hPdh_XmlwvZ)@=V%EVE(T1T7hgXisjQc+a-p@Gz delta 10766 zcmeG?X?Rpcwtf5dzNC{Ngf$(Y7hsUw&I*{$Z6FH>S#Tg|Ad8L-qCn#442no1YO4`R zhkT|Kk${2`okw7Jq`*(RXyy{c|)S5U0&suj1y8pl{xUY_xZ`$K`ut??!ZaeGE5g>A1S#SU7lyJxBS`@lyk z{^z+{HICLcdu72=tNaOmm4PJf&yncWE#5PNF#^|;bFX-R>_dz3t6m_UDc(Kea<9-1dD__=&mmBLoqOn1lP*#Y}5lrvTJp2 zJbmzr_fJ~Z5ql~>iFF)8(ksQLpS?*W`?~iVMskx#s`eL1v@7ID)Uc`h_roBMo>bSD z8A!M}XE*Df%vdC?dkZ8s{X8m-%|32skB_s*Vd9=p=W}W{+b>!IT(*;*?A#G_+05*9 zI<~gh#Btdol`a)jtNZmZM!X_PlAMd1d}(_6RFPm-_an_wYnEp(6@bW|mflZ<7N8-v zz&ttv5=bDSU%W}wnU(z&3Tl{|#3oLj*%GXgMYgmak@T!98(bqSR+;ZsMbxkLtNwCR zZ|@C&y@Lk;UL7(3(2y_y&^r)te&_&~53ucX*@tDOeZW>%vz4)OP0e$QSt$b4q6o`Q;=5=iy>S!bDc z>a+l!a{`4mQe;LXn&`?3p;XPD@3YX@vYb=_aEjy1401*T&$mCMW17Cs=3Q$5z@p>g zfQcJOY+~h9Zh+@vP zS!N6EWY^V#p(LiD%ua8aBIlwp%bmh++ftle0$WhAK&5o&{UX`YDaVn*L-KI)V(nuo5M~hQiYfcx^wKU&(}CNsh|!3Qny^1V%{w z6;V)BwF&`P2p~7H@yGw!3i!@2V9aMm zdNGiFM$bI03O(%^f!LDOKQQkJ8<`8ZwNx*wQ}yimeFN<+R6j3r%fE0?zSTmcnruC} zT&|W#&&m9DQax9`kHnt$T1a`EY$kzs<-zoWPWfI8-rXzINnB&h0+_~pq*E0rpRi-n1bD@t zC=8!E#sd{Zj*h^m?1_;Md;%&GDW4#>_ai?5g^{A8FU!5U2+wZfF*%2kYZmbRupQ`y z5`kU#sbnyTKtPhn%8~H1&I+%s>=j`a){DqB$}owK0GKiY@G2QmWtf}W`wB;rSetC5%VK@otgNGRr`n;%9L03jPL9rnv0FYbaFSKAzA<}> zf7hm=hYz@J)1%@R?`~BuR_UUubNE$fZ%AJN`1FSK1%S{E=?efmZ^%{vc;SX@<)9}s za~uZ7UZ_LuR#V_C7L&q3Goh!{o0zrVZXxaK@+`D>QO-p*Lv6jF&IdoJ6Nf-XnV$EO zj<#HPg|K{?Vj|5kqM1HhBn(jx$QIEwwOF7earQA28$DJ_pUTe6)N@}-YRP%s(ByOI z1yAENc__(FcKQ1XILWW-h~WiX%dVUgVqb^XuR6}lE&9%X<_s3B&nUhQrQr7?_y}8D zjz%;_I&LB(_vajpkGzictDaZ)_AXYXkkOY*Cejga<;;juar#wtR$dR9LOE^ZgHsvR zjm=(zc7k8^AIiMe;H)mSo|kv2w4qMiF0$3#W+9tmeFnO(c-C?Ytv2L8F4C$Mq6RCz zeOd~MJWEfPtP#g)`=qtoL@U;dbpq?$XJB&PhP2|5(MU%RDSQ+v%?)!_S=)S$w^1yA1oI2$exPCYLVJrG39!^U zaScKj8E8+1qgG&$078B+Gd$8=b?daLmN;1RH&N~7SL zL)AE**zcHb;${T3U{K3|q(0f%z?3*h$L)y)pXW)f?EDv_I~ryBcMFcMDRO>|7IntR zXy2?M)fLY1Y_J}dJ3a41J)%`rIhW$H#_$A4-dR{-q-Ce`W}9hcjZ**;8|Lh?vRINY z;E0NP=Sn1;Y3O6m0sJ&O6BMo(@_3G1)aqRQZ=r+nK31TzeC4agwkat!DV!YpSj9=3tfxQS(Ttk^d0%0h}s?0*OA|2L$!3Ta7`mU zkn>sf#u|m1m}mEPp53wVl1}CaNo}gLoma7J08^mEU$Lx%(x8U`9%Pi`uMEUBmrsnJOh<+-#$($`(Shs5)~ARSo=kuc;O(WVBru6kWeSQZ)ZhQ1OO4! zcf&hvxs?B~5!D@D@A?Mq%4VfUH@gmNnYcek@h;>V>)h#b0>*ULX8;Z7UeVL`1FoeS zZXCF;K_P`YT<`URv6Ua$*CjnwVnGdmg_<%!z`X7%{x_5A zv&2kpcg->G8OZ5dNbHh^1{2+_NQfe2gMrp&yFXNn{^~fvNUvX-H3s(QjG4wzr(TfB zKR3(@HeSdyXdzi0*Ynh~zj0Q)+WM@LlSIF&LEs&RrdiHthsCejYT{|;t1}tReFKL0 zRrMAQf9q_e@=b}D*feXB9g*y*Z8DuN2n94Qo{lY-a zQSjmON3B(8A*X?#GM5RiUj3>}!iA?933wV5`!OLdr73(+WE)iA~H(ff8`1e#`G zuVl&d1(htk?nUlTni;V1a>XiM&!Xl@UVnyf-YzlmSE(n zv*K!o;gn{+{Fqlt-a}4keki>TMZ5$>ER3L^h8*WG?rSY}I9Y!%~f^uL7 zM({Of3ij~#V+mKwRlUt;zQP3hHA`b7VsIe|_Kcv3z2C{@nP7LN;PK2M2E2xBuw+z< z21oi-J(f-@%4@YTa?_B|6ta^Tr{#uUJ#N61}2`K zca3Wo_8$vSyBm=)*pHO#?=${`$P8>e^8SDdUX8MZXa&r%R1?k}j?#pUhByt2GW3}# zJWhKU+BICmCC&ouK`KzC8@NibhHpn{N?fHY%S9J6tq*ODZ+Du6bVlMNTH{TeU#Q|` z)Z?FVW+9W6X|MM4ke6}sP{ZGD;uaNptLQP2ty{-MJB88w07W?gMafF~@o<6)vmz71 zEes=!vWw%4Ls_4TT>ObE)XQw~HB{)WT%l0l$Ng$BUe8Vo$6eO4swT|S8ifs5ECj|h zy`ZN%Gx{fqY`^L|<@kKk%!2O)>pCT&nW4kZ^UeCH!d;>k64n7_TlCL@f*}IQmIZ^> z4cSM{lZ;$cFoo1)yGGM}^`0lyY#;SpBJ1+dbr(x^tI0LuF=!{l$qgSsS6wO@FEDZb zBpBf^G2XlIl~M~2#muy`nbqtuw)v!%U$wekIH%OGErPTcGyD#Wtc@DjB1nmtPdLP5 z;E3H-(ilu4oo3QiBdupX28Ced?A%^<)C{D;_0o%2dGH{;M@Nn}OD6hdv*Tr?J=etX zI$I?*1{5A6W!Nc5u;Ya!v=L3^7A?mMbxLb6A@bp$gNufxCvK=D=hTTt(t2^8-eBQj z+)5jQa>&Z^j7QzFrB_0RTKN2M5y1Ftwt3VzQ8$z+cFzi|Y1s`zJnyeh_N>A^9v^m8 z3p)ePX~rK1tT1xCnj+7)xG8~5T)jSLE3K~ZjK(y$7Bl~R4acjh@~lRIuESaXp|G)} zCVe>y^a(Smtnr*+0#){#q+*t2q&tdpS{2bQXm&KO^eSdnlNqr(*=@r683 za=@?E7;wmpYnMAcDMp6&9gaJpfz5#jw!;89jR9_@qVD8w&lQ1z&!Kp%P;B*jilgEC zCxxKhmJ*NI0`S1#F6m%Xkn#vRFo$eVv<}hvU_ZaFZIe?7CeCkhg4K%Bdka&3z=fSf zdI?6_Q=U?eoN&awQ1&n_cCScD#WM2{j=u`G9BWcGVZ{6Zhc+p44zEwSjE6WJoMwjl zJH;jR<*&D3yUC&rC)>F*B>~y`NC^+HYy1H=vjPY#Dk@yIwjcjtx zty=UCD8^GjZ_xKS_N`MaqV+<`MU?sH(Mik@u&$?+A*Y`W3{6vzIXFg2eU=5j#WtwJ zQy)gTA4SToI?jImq}0Em@Hx1^L~65BeT;)oP)vO5R;wtlrZ9CdvIl!PP+poAYT48^ z$QeMu9{;LXL)D7Z6R3kXv2I*$iTGOv-CbQ$jA3Al$XTdZpLzw=unkGf(?bcHx(u0m z8K>#y79mVw!RvQi5#*C)V3!{sT(AsIgmOU}P}Zix4GS>-1`hgIu@AW;^=niF2j6Ri z>NYhGc>@R(Udt_{_U8PTI4{W;QvZbu+>JzT1-HJQT7leg$XUfm>@jKYpy+oZVgL%| zG(CY^xK9+5hk&w7H0IvivTVZNF4set!IBrCkgPyZ)w(uc+IjSEGwhb(2Cug$Z8qHC z$tAB@Nl!)EJuH4~#q8P*6*?evSEb#J_3?8CICV)oih*sM1%hT;3O7zX_+z1rpPMa# zE4+#<18K-B7)IazvE;XcV@hx2&v#miZGT~o6GC_rWo&6 zH7hsJC!5RXihUhve$^3WXSMv*@||j^n?G&N@uJc0MJa1xiyKO-LeuyzYVuE@0O5KW z%menC{Z9P)_C?^zyTWNtqkWu5l*i%h0fgGxopu_{<1)?w;bTj0+8i_wfPi`Ig<~DS zYqq6dL&La)cmj;Er~A<`IHVcQbpWqEIsFjY#YM#PnrW?^KB}K9Rl_k8;MEkR3&_pK z*artW)fMUYA}5_nsKK2A0fRomMv$ti^!J&0LjVW+DYn8fQTo%!(w7Kn;#N{opMDx0 z5>BDOMLh;MAxekiJQxYS2#`*0szt5oPeh~hJM35}eFw_A3CZop0*1<+>3ZafL+6=@ z6i$DDQ3C)NYd4(c!E%zj(_P3*BhE1yvU6|x2IM2CET_~^dI05RWXCDL;4r%_<0iOa zg)10lwUuJuNlFj9$Q<5eE*(lANhp0@&%}^J;fUp3h(z`bs3Gw_90ZX-T~aBeLHO-K znzrPOZ`J&+8)!yu&LiYVk*QQmPLC~}$FAnP1i0?>sGH=B1h{>x6#4Bpbig4J-ol@q zSsJUOH4di^)h3K(Re8F|?h-4T0%bwGW+1-#rMkQNdCH$-*r`rGz{|D)fOWA009OYB zzKRo?+b)|$W_tK%#B#9n4|grW(}mcOc9wiqlCWa zAxG}WT#O7`agH_IzcWvX$D-<3#s!#sA+rXfjtQ4>59s}R<~j6lfPj&RJQvwxCi4^t zP!Ql&D}`@E1mp-N0k8W8ZYw4Sq$vSuYCyWh=$)57rQv~w4K^jsfuNQly(bNAzN`;X z7o5AXtQwoFC%r{kQk1PtDF1gUTKf%G0cx0L0GLtUJ_f{OYK)`~q$f%7Sln`X?TU z9_4&Us57ey@5BQ*CD_)kaMp4B#)d;SvXTNs(%PMs!AL)j&NdlNZF;ku$Q8c_7|V(< z;Gx;(eGpkIM?^aWRl66WJq*1Qr+ACinSfZI?0qrf>TvRR9#WB4kE|3c;_aSO_Kcdv z)-e>m`{{y6J${JBVB^j*h~?AOzyMsE!>&kj!bK3=kwa*NXFUFHXHUwP5G)jv)dgiq zbjQN7!w?~hgi+Yd;w&5E9AXq~XuT}#TIM&q6=urDfrwsS}f zxu?oo7oot;=-GuYEqbsFzOjL!sXR=Rw%W3NkPpkiT&vfXy=0mjF%B?1xtjF#bW)+HIyobOG~AL|0d;Gg#VG7boLyp zz!{EKdDg&x<*rkuhG*8;9~;N;*q=zY)|Vxi6@>txmJ}u3IT+MCE(y*-V*4%=ll81z z{8s-|akCd%HZ&QU585`# dplyr::pull(conceptId) |> unique() # add extra conceptIds to map -conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219) +# 4181730 is the Finnish language concept, used as language_concept_id in CONCEPT_SYNONYM +conceptIdsToMap <- c(conceptIdsToMap, 138604, 313219, 4181730) # For each table we have to add also the concepts to the concept table diff --git a/man/appendUsagiFileToSTCMtable.Rd b/man/appendUsagiFileToSTCMtable.Rd index c2503ea..1281765 100644 --- a/man/appendUsagiFileToSTCMtable.Rd +++ b/man/appendUsagiFileToSTCMtable.Rd @@ -57,6 +57,7 @@ Extended STCM columns (includes all default columns plus): \item source_concept_class \item source_domain \item source_parents_concept_ids +\item source_code_description_fi } Extended Usagi columns: @@ -64,6 +65,7 @@ Extended Usagi columns: \item ADD_INFO:sourceConceptId \item ADD_INFO:sourceConceptClass \item ADD_INFO:sourceDomain +\item ADD_INFO:sourceName_fi (optional, NA if missing) } If formats don't match (e.g., extended Usagi with default STCM), the extended information diff --git a/man/createSourceToConceptMapExtended.Rd b/man/createSourceToConceptMapExtended.Rd index 14ae818..2e4b035 100644 --- a/man/createSourceToConceptMapExtended.Rd +++ b/man/createSourceToConceptMapExtended.Rd @@ -22,5 +22,5 @@ createSourceToConceptMapExtended( } \description{ Creates an extended version of the Source to Concept Map table that includes additional columns -for source concept class, domain, and parent relationships. +for source concept class, domain, parent relationships, and the Finnish source code description. } diff --git a/man/readUsagiFile.Rd b/man/readUsagiFile.Rd index cc63ab9..796130a 100644 --- a/man/readUsagiFile.Rd +++ b/man/readUsagiFile.Rd @@ -47,6 +47,7 @@ Optional ADD_INFO columns: \item sourceValidEndDate (date): End date of validity \item sourceParents (character): Parent concepts \item sourceParentVocabulary (character): Vocabulary of parent concepts +\item sourceName_fi (character): Finnish name of the source code, added to CONCEPT_SYNONYM \item validationMessagess (character): Validation messages } } diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index 06f74cc..a1f9363 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -37,5 +37,9 @@ If usagi file has C&CR columns: \item Check codes with mapping to more than one domain are mapped to compatible domains \item Check if sourceValidStartDate is before sourceValidEndDate \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary +If usagi file has the ADD_INFO:sourceName_fi column: +\item Check if sourceName_fi is less than 1000 characters +\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/man/writeUsagiFile.Rd b/man/writeUsagiFile.Rd index 9441d51..7a941dd 100644 --- a/man/writeUsagiFile.Rd +++ b/man/writeUsagiFile.Rd @@ -44,6 +44,7 @@ The function writes a Usagi mapping file with the following columns: \item ADD_INFO:sourceValidEndDate (date): Source valid end date \item ADD_INFO:sourceParents (character): Source parents \item ADD_INFO:sourceParentVocabulary (character): Source parent vocabulary +\item ADD_INFO:sourceName_fi (character): Source name in Finnish \item ADD_INFO:validationMessages (character): Validation messages \item ADD_INFO:autoUpdatingInfo (character): Auto updating info } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md new file mode 100644 index 0000000..0291d04 --- /dev/null +++ b/plan_add_finnish_names.md @@ -0,0 +1,104 @@ +# Finnish names from Usagi files to `CONCEPT_SYNONYM` + +The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. + +## Decisions +- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. +- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. +- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. +- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. +- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. +- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. +- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. + +## Data flow + +``` +Usagi CSV STCM (extended) CDM +ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM + concept_id = source_concept_id + concept_synonym_name = source_code_description_fi + language_concept_id = 4181730 (Finnish) +``` + +- The language is hardcoded to Finnish (`4181730`). +- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. +- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). + +## Changes by step + +### 1. Reading and writing Usagi files (`R/usagiFile.R`) +- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. +- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. + +### 2. Validation (`R/validateUsagiFile.R`) +- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. +- New checks: + +| Step in the log | Level | Rule | +|---|---|---| +| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | +| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | +| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | + +- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. + +### 3. STCM table +- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. +- `R/appendUsagiFileToSTCMtable.R`: + - The column is part of the extended STCM columns. + - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. +- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. + +### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) +- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. +- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: + - `source_concept_id = 0` + - the name is NULL or blank + - the name equals `source_code_description` +- Re-running it gives the same result. + +### 5. Post-load check (`R/buildVocabuliesAll.R`) +- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: + - the expected names in the STCM table, using the same filters as the SQL; + - the Finnish rows in `CONCEPT_SYNONYM`. +- It logs SUCCESS if the counts match and ERROR otherwise. + +### 6. Other +- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. +- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. + +## Tests +- `test-validateUsagiFile.R`: + - The too-long, differing-within-sourceCode and empty checks each fire. + - The C&CR columns are required when `sourceName_fi` is present. +- `test-appendUsagiFileToSTCMTable.R`: + - The new column appears in the extended STCM table and gets its value. + - It is NA when the Usagi file has no Finnish column. +- `test-STCMToCDMTable.R`: + - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. + - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. + - Running it twice gives the same result. + - An existing English synonym is preserved. +- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. +- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. +- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. + +Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. + +## Files changed +- `R/`: + - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` + - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` +- `inst/`: + - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` + - `sql/sql_server/STCMExtendedToCDM.sql` + - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` + - `testdata/createTestData.R` +- `man/`: + - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` + - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` +- `tests/testthat/`: + - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` + - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` +- `NEWS.md` diff --git a/tests/testthat/test-STCMToCDMTable.R b/tests/testthat/test-STCMToCDMTable.R index 6584227..4042d07 100644 --- a/tests/testthat/test-STCMToCDMTable.R +++ b/tests/testthat/test-STCMToCDMTable.R @@ -25,6 +25,7 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre "code9", 2000000009, "TestVocab", "Test Code 9 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002", "code10", 2000000010, "TestVocab", "Test Code 10 parent concept ids", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class 3", "Condition", "2000000001|2000000002|2000000003" ) + testData <- testData |> dplyr::mutate(source_code_description_fi = NA_character_) # Create connection to test database connection <- DatabaseConnector::connect( @@ -113,3 +114,80 @@ test_that("STCMToCDMTables creates CONCEPT entries from STCM Extended with corre res |> dplyr::pull(concept_id_2) |> expect_equal(c(2000000001, 2000000001, 2000000002, 2000000001, 2000000002, 2000000003)) }) + +test_that("STCMToCDMTables creates Finnish CONCEPT_SYNONYM entries from source_code_description_fi", { + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + sourceToConceptMapTable <- "source_to_concept_map_extended" + + testData <- tibble::tribble( + ~source_code, ~source_concept_id, ~source_vocabulary_id, ~source_code_description, ~target_concept_id, ~target_vocabulary_id, ~valid_start_date, ~valid_end_date, ~invalid_reason, ~source_concept_class, ~source_domain, ~source_parents_concept_ids, ~source_code_description_fi, + # multi-mapped code, one synonym + "code1", 2000000001, "TestVocab", "Test Code 1", 141797, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + "code1", 2000000001, "TestVocab", "Test Code 1", 36713461, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 1", + # unmapped code also gets synonym + "code2", 2000000002, "TestVocab", "Test Code 2", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 2", + # same name as concept_name is not added as synonym + "code3", 2000000003, "TestVocab", "Test Code 3", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Test Code 3", + # no synonym + "code4", 2000000004, "TestVocab", "Test Code 4 no fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, NA_character_, + "code5", 2000000005, "TestVocab", "Test Code 5 blank fi", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, " ", + # source_concept_id 0 is skipped + "code6", 0, "TestVocab", "Test Code 6 no source concept id", 0, "SNOMED", as.Date("2023-01-01"), as.Date("2099-12-31"), NA_character_, "Test concept class", "Condition", NA_character_, "Testikoodi 6" + ) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + createSourceToConceptMapExtended( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable, + overwrite = TRUE + ) + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = sourceToConceptMapTable, + data = testData + ) + + # existing synonyms: an old Finnish one to be replaced and an English one to be kept + DatabaseConnector::insertTable( + connection = connection, + databaseSchema = vocabularyDatabaseSchema, + tableName = "CONCEPT_SYNONYM", + data = tibble::tribble( + ~concept_id, ~concept_synonym_name, ~language_concept_id, + 2000000001, "Vanha nimi", 4181730, + 2000000001, "Old English synonym", 4180186 + ), + dropTableIfExists = FALSE, + createTable = FALSE + ) + + # run twice to check it is idempotent + for (i in 1:2) { + STCMToCDMTables( + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + } + + res <- dplyr::tbl(connection, "CONCEPT_SYNONYM") |> + dplyr::filter(concept_id >= 2000000000 | concept_id == 0) |> + dplyr::arrange(concept_id, language_concept_id) |> + dplyr::collect() + + finnish <- res |> dplyr::filter(language_concept_id == 4181730) + finnish |> dplyr::pull(concept_id) |> expect_equal(c(2000000001, 2000000002)) + finnish |> dplyr::pull(concept_synonym_name) |> expect_equal(c("Testikoodi 1", "Testikoodi 2")) + + # other languages are not touched + res |> dplyr::filter(language_concept_id != 4181730) |> dplyr::pull(concept_synonym_name) |> expect_equal("Old English synonym") +}) diff --git a/tests/testthat/test-appendUsagiFileToSTCMTable.R b/tests/testthat/test-appendUsagiFileToSTCMTable.R index 6749c9c..2fb6846 100644 --- a/tests/testthat/test-appendUsagiFileToSTCMTable.R +++ b/tests/testthat/test-appendUsagiFileToSTCMTable.R @@ -81,13 +81,19 @@ test_that("test appendUsagiFileToSTCMTable appends the usagi file to the sourceT expect_equal(c( "source_code", "source_concept_id", "source_vocabulary_id", "source_code_description", "target_concept_id", "target_vocabulary_id", "valid_start_date", "valid_end_date", "invalid_reason", "source_concept_class", - "source_domain", "source_parents_concept_ids")) + "source_domain", "source_parents_concept_ids", "source_code_description_fi")) stcmTable |> dplyr::filter(is.na(source_parents_concept_ids)) |> nrow() |> expect_equal(0) + # source_code_description_fi comes from ADD_INFO:sourceName_fi + stcmTable |> + dplyr::filter(source_code == "A01.0+G01") |> + dplyr::pull(source_code_description_fi) |> + expect_equal("Lavantautiin liittyvä aivokalvotulehdus") + }) test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the sourceToConceptMapTable with ICD10 parent information", { @@ -130,3 +136,34 @@ test_that("test appendUsagiFileToSTCMTable appends the ICD10fi usagi file to the expect_equal("2000503725") }) + +test_that("test appendUsagiFileToSTCMTable sets source_code_description_fi to NA when the usagi file has no ADD_INFO:sourceName_fi", { + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-`ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + vocabularyDatabaseSchema <- "main" + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + sourceToConceptMapTable <- "source_to_concept_map_extended" + createSourceToConceptMapExtended(connection, vocabularyDatabaseSchema, sourceToConceptMapTable) + + appendUsagiFileToSTCMtable( + vocabularyId = "ICD10fi", + pathToUsagiFile = pathToUsagiFile, + connection = connection, + vocabularyDatabaseSchema = vocabularyDatabaseSchema, + sourceToConceptMapTable = sourceToConceptMapTable + ) + + stcmTable <- DBI::dbReadTable(connection, sourceToConceptMapTable) |> tibble::as_tibble() + stcmTable |> nrow() |> expect_equal(nrow(usagiTibble)) + stcmTable |> dplyr::pull(source_code_description_fi) |> is.na() |> all() |> expect_true() +}) diff --git a/tests/testthat/test-buildVocabulariesAll.R b/tests/testthat/test-buildVocabulariesAll.R index c0fcf28..8e0416a 100644 --- a/tests/testthat/test-buildVocabulariesAll.R +++ b/tests/testthat/test-buildVocabulariesAll.R @@ -28,6 +28,9 @@ test_that("buildVocabulariesAll works", { # Check results validationLogTibble |> dplyr::filter(type == "ERROR") |> nrow() |> expect_equal(0) + # Finnish names were added to CONCEPT_SYNONYM + validationLogTibble |> dplyr::filter(step == "Finnish names in CONCEPT_SYNONYM") |> dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + # check the validation results folder expect_true(!file.exists(file.path(validationResultsFolder, "resultsDQD.json"))) }) diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index 8a5d44b..fcb4111 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -306,3 +306,92 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { + +test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + pathToValidatedUsagiFile <- tempfile(fileext = ".csv") + vocabularyDatabaseSchema = "main" + sourceConceptIdOffset = 2000500000 + + # make a copy of a valid usagi file with errors in ADD_INFO:sourceName_fi + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + usagiTibble <- usagiTibble |> + dplyr::mutate( + `ADD_INFO:sourceName_fi` = dplyr::case_when( + # too long + sourceCode == "A01.0+G01" ~ strrep("a", 1001), + # empty + sourceCode == "A01.0+I39.8" ~ NA_character_, + # different within a multi-mapped sourceCode + sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", + TRUE ~ `ADD_INFO:sourceName_fi` + ) + ) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile( + pathToUsagiFile, + connection, + vocabularyDatabaseSchema, + pathToValidatedUsagiFile, + sourceConceptIdOffset + ) + + validatedUsagiFile <- readUsagiFile(pathToValidatedUsagiFile) + + # only the sourceName_fi validations fail + validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> + expect_setequal(c( + "SourceNameFi is more than 1000 characters", + "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is empty" + )) + + # SourceNameFi is more than 1000 characters + validationsSummary |> dplyr::filter(step == "SourceNameFi is more than 1000 characters") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceNameFi is more than 1000 characters") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") + + # SourceNameFi is not the same for all rows of the sourceCode + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) + + # SourceNameFi is empty, only a warning, mappingStatus is not changed + validationsSummary |> dplyr::filter(step == "SourceNameFi is empty") |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("WARNING: SourceNameFi is empty") + validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+I39.8") |> dplyr::pull(mappingStatus) |> expect_equal("APPROVED") +}) + +test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName_fi is present", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) |> + dplyr::select(-dplyr::starts_with("ADD_INFO:"), `ADD_INFO:sourceName_fi`) + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") +}) diff --git a/tests/testthat/test-validateVocabularyFolder.R b/tests/testthat/test-validateVocabularyFolder.R index 597b028..95aa62b 100644 --- a/tests/testthat/test-validateVocabularyFolder.R +++ b/tests/testthat/test-validateVocabularyFolder.R @@ -23,7 +23,12 @@ test_that("validateVocabularyFolder returns all SUCCESS for valid vocabulary fol ) # Check validation results - validationsLogTibble |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) + # only warning: UNITfi has the ADD_INFO:sourceName_fi column but all values are empty + notSuccess <- validationsLogTibble |> dplyr::filter(type != "SUCCESS") + notSuccess |> nrow() |> expect_equal(1) + notSuccess |> dplyr::pull(context) |> as.character() |> expect_equal("UNITfi") + notSuccess |> dplyr::pull(type) |> as.character() |> expect_equal("WARNING") + notSuccess |> dplyr::pull(step) |> expect_equal("SourceNameFi is empty") # Check that validated files were created expect_true(dir.exists(pathToValidatedVocabularyFolder)) From f50a629bcd57696653ec9134675d4e4a2d8e001a Mon Sep 17 00:00:00 2001 From: javier-gracia-tabuenca-tuni Date: Tue, 10 Nov 2026 13:59:43 +0200 Subject: [PATCH 2/3] Check sourceName_fi uniqueness per sourceConceptId, not per sourceCode A sourceCode can have several sourceConceptIds, each valid for a different period and with its own English and Finnish name (e.g. REIMB 307). The previous check flagged these valid files as ERROR, which stopped buildVocabulariesAll. Each sourceConceptId must now have one sourceName_fi, so each concept gets exactly one Finnish synonym. Co-Authored-By: Claude Opus 5.5 --- NEWS.md | 2 +- R/validateUsagiFile.R | 27 +++--- man/validateUsagiFile.Rd | 2 +- plan_add_finnish_names.md | 104 ------------------------ tests/testthat/test-validateUsagiFile.R | 42 ++++++++-- 5 files changed, 52 insertions(+), 125 deletions(-) delete mode 100644 plan_add_finnish_names.md diff --git a/NEWS.md b/NEWS.md index a8d1eed..e26e8e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # ROMOPMappingTools (development version) - Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. -- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceCode; a warning is given if it is empty for APPROVED or UNCHECKED rows. +- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceConceptId; a warning is given if it is empty for APPROVED or UNCHECKED rows. - `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. - Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. diff --git a/R/validateUsagiFile.R b/R/validateUsagiFile.R index 60d49fc..bf2e2d4 100644 --- a/R/validateUsagiFile.R +++ b/R/validateUsagiFile.R @@ -16,7 +16,7 @@ #' - Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary #' If usagi file has the ADD_INFO:sourceName_fi column: #' - Check if sourceName_fi is less than 1000 characters -#' - Check if sourceName_fi is the same for all the rows of a sourceCode +#' - Check if sourceName_fi is the same for all the rows of a sourceConceptId #' - Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus #' #' @param pathToUsagiFile Path to the Usagi mapping file to validate @@ -364,27 +364,30 @@ validateUsagiFile <- function( usagiTibble <- result$fileTibble validationLogR6 <- result$validationLogR6 - # check all rows of a sourceCode have the same sourceName_fi (multi-mapped codes repeat the source row) - sourceCodesWithDifferentNameFi <- usagiTibble |> - dplyr::distinct(sourceCode, `ADD_INFO:sourceName_fi`) |> - dplyr::count(sourceCode) |> + # check all rows of a sourceConceptId have the same sourceName_fi (multi-mapped codes repeat the source row), + # each sourceConceptId gets one Finnish synonym. A sourceCode can have several sourceConceptIds, valid for + # different periods, each with its own name, so the check is not done per sourceCode + sourceConceptIdsWithDifferentNameFi <- usagiTibble |> + dplyr::filter(!is.na(`ADD_INFO:sourceConceptId`)) |> + dplyr::distinct(`ADD_INFO:sourceConceptId`, `ADD_INFO:sourceName_fi`) |> + dplyr::count(`ADD_INFO:sourceConceptId`) |> dplyr::filter(n > 1) |> - dplyr::pull(sourceCode) + dplyr::pull(`ADD_INFO:sourceConceptId`) - if (length(sourceCodesWithDifferentNameFi) > 0) { + if (length(sourceConceptIdsWithDifferentNameFi) > 0) { validationLogR6$ERROR( - "SourceNameFi is not the same for all rows of the sourceCode", - paste0("Found ", length(sourceCodesWithDifferentNameFi), " sourceCodes with different sourceName_fi") + "SourceNameFi is not the same for all rows of the sourceConceptId", + paste0("Found ", length(sourceConceptIdsWithDifferentNameFi), " sourceConceptIds with different sourceName_fi") ) usagiTibble <- usagiTibble |> dplyr::mutate(tmpvalidationMessages = dplyr::if_else( - sourceCode %in% sourceCodesWithDifferentNameFi, - paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceCode"), + `ADD_INFO:sourceConceptId` %in% sourceConceptIdsWithDifferentNameFi, + paste0(tmpvalidationMessages, " | ", "ERROR: SourceNameFi is not the same for all rows of the sourceConceptId"), tmpvalidationMessages )) } else { - validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceCode", "") + validationLogR6$SUCCESS("SourceNameFi is not the same for all rows of the sourceConceptId", "") } # warn if sourceName_fi is empty, the column is optional so this is not an error diff --git a/man/validateUsagiFile.Rd b/man/validateUsagiFile.Rd index a1f9363..a8b5ad7 100644 --- a/man/validateUsagiFile.Rd +++ b/man/validateUsagiFile.Rd @@ -39,7 +39,7 @@ If usagi file has C&CR columns: \item Check if ADD_INFO:sourceParents is a valid concept code in the ADD_INFO:sourceParentVocabulary If usagi file has the ADD_INFO:sourceName_fi column: \item Check if sourceName_fi is less than 1000 characters -\item Check if sourceName_fi is the same for all the rows of a sourceCode +\item Check if sourceName_fi is the same for all the rows of a sourceConceptId \item Warn if sourceName_fi is empty for APPROVED or UNCHECKED mappingStatus } } diff --git a/plan_add_finnish_names.md b/plan_add_finnish_names.md deleted file mode 100644 index 0291d04..0000000 --- a/plan_add_finnish_names.md +++ /dev/null @@ -1,104 +0,0 @@ -# Finnish names from Usagi files to `CONCEPT_SYNONYM` - -The optional Usagi column `ADD_INFO:sourceName_fi` is carried through the STCM table into the CDM `CONCEPT_SYNONYM` table as a Finnish synonym of the source concept. - -## Decisions -- **Finnish only.** `language_concept_id` is hardcoded to 4181730. There is no parameter yet. -- **New STCM column.** `source_code_description_fi` carries the name, as for the other extended columns. -- **Empty name is a warning.** The column is optional, so a missing name never blocks a mapping. -- **Same name as `sourceName` is skipped silently.** It is neither warned about nor added as a synonym. 11 ICD10fi Latin terms are affected. -- **No blank-name rule.** readr already turns whitespace-only names into NA, so they count as empty. -- **Post-load check against the STCM table.** The STCM table is already filtered by mappingStatus, so it is a better reference than the Usagi files. -- **Minimal fixture change.** Only concept 4181730 was added to the test vocabulary. The full vocabulary was not regenerated. - -## Data flow - -``` -Usagi CSV STCM (extended) CDM -ADD_INFO:sourceName_fi -> source_code_description_fi -> CONCEPT_SYNONYM - concept_id = source_concept_id - concept_synonym_name = source_code_description_fi - language_concept_id = 4181730 (Finnish) -``` - -- The language is hardcoded to Finnish (`4181730`). -- A synonym is added only when the name is not empty and differs from `sourceName`. Names equal to `sourceName` are skipped silently, with no warning. -- Only rows that reach the STCM table become synonyms: mappingStatus APPROVED, UNCHECKED or INVALID_TARGET (`includeConcepts`). - -## Changes by step - -### 1. Reading and writing Usagi files (`R/usagiFile.R`) -- `readUsagiFile()` reads `ADD_INFO:sourceName_fi` as character. readr trims whitespace and reads `""` as NA, so blank names become NA. -- `writeUsagiFile()` places the column after `ADD_INFO:sourceParentVocabulary` and writes empty strings as NA. - -### 2. Validation (`R/validateUsagiFile.R`) -- If `ADD_INFO:sourceName_fi` is present, the C&CR columns (`sourceConceptId`, `sourceConceptClass`, `sourceDomain`) are required, as for the other extended columns. -- New checks: - -| Step in the log | Level | Rule | -|---|---|---| -| `SourceNameFi is more than 1000 characters` | ERROR | Fits `concept_synonym_name varchar(1000)` | -| `SourceNameFi is not the same for all rows of the sourceCode` | ERROR | Multi-mapped codes must repeat the same Finnish name | -| `SourceNameFi is empty` | WARNING | Missing name for an APPROVED or UNCHECKED row | - -- ERRORs set the row to FLAGGED, like the other checks. The WARNING only adds a message to `ADD_INFO:validationMessages`. - -### 3. STCM table -- `inst/ddl/5.4/sql_server/createSourceToConceptMapExtended.sql`: new column `source_code_description_fi varchar(1000) NULL`. -- `R/appendUsagiFileToSTCMtable.R`: - - The column is part of the extended STCM columns. - - It is filled from `ADD_INFO:sourceName_fi`, or with NA when the Usagi file doesn't have that column. -- An extended STCM table created before this change lacks the column and must be recreated with `createSourceToConceptMapExtended()`. - -### 4. STCM to CDM (`inst/sql/sql_server/STCMExtendedToCDM.sql`, section 5) -- DELETE removes Finnish synonyms (`language_concept_id = 4181730`) for the `source_concept_id`s in the STCM table. Synonyms in other languages are not touched. -- INSERT adds one row per distinct `(source_concept_id, source_code_description_fi)`, skipping rows where: - - `source_concept_id = 0` - - the name is NULL or blank - - the name equals `source_code_description` -- Re-running it gives the same result. - -### 5. Post-load check (`R/buildVocabuliesAll.R`) -- After `STCMToCDMTables()`, a new log step, `Finnish names in CONCEPT_SYNONYM`, compares two counts: - - the expected names in the STCM table, using the same filters as the SQL; - - the Finnish rows in `CONCEPT_SYNONYM`. -- It logs SUCCESS if the counts match and ERROR otherwise. - -### 6. Other -- `R/calculateMappingStatus.R`: changed `language_concept_id == 4181730` to standard SQL `=`. This query fills `concept_name_fi` in the status reports, so it now also picks up synonyms for the custom vocabularies. -- Docs: roxygen comments, Rd files (edited by hand because roxygen2 isn't installed locally; run `devtools::document()` to confirm) and NEWS.md. - -## Tests -- `test-validateUsagiFile.R`: - - The too-long, differing-within-sourceCode and empty checks each fire. - - The C&CR columns are required when `sourceName_fi` is present. -- `test-appendUsagiFileToSTCMTable.R`: - - The new column appears in the extended STCM table and gets its value. - - It is NA when the Usagi file has no Finnish column. -- `test-STCMToCDMTable.R`: - - Synonyms are created with language 4181730, including for multi-mapped and unmapped codes. - - Rows with an empty name, a blank name, a name equal to the concept name, or `source_concept_id = 0` are skipped. - - Running it twice gives the same result. - - An existing English synonym is preserved. -- `test-validateVocabularyFolder.R`: the valid test folder gives one expected WARNING. UNITfi has the `sourceName_fi` column with all values empty. -- `test-buildVocabulariesAll.R`: the step `Finnish names in CONCEPT_SYNONYM` is SUCCESS. -- Test vocabulary: added concept 4181730 (Finnish language) to `OMOPVocabulary.duckdb`, `CONCEPT.csv` (gitignored) and `createTestData.R`. Otherwise DQD fails the `CONCEPT_SYNONYM.language_concept_id` foreign-key check. - -Full suite: 27 tests, 0 failures, 0 errors. Run with `Rscript --no-init-file`, because the renv library isn't installed for R 4.6. - -## Files changed -- `R/`: - - `usagiFile.R`, `validateUsagiFile.R`, `appendUsagiFileToSTCMtable.R` - - `createSourceToConceptMapExtended.R`, `buildVocabuliesAll.R`, `calculateMappingStatus.R` -- `inst/`: - - `ddl/5.4/sql_server/createSourceToConceptMapExtended.sql` - - `sql/sql_server/STCMExtendedToCDM.sql` - - `testdata/OMOPVocabulary/OMOPVocabulary.duckdb` - - `testdata/createTestData.R` -- `man/`: - - `readUsagiFile.Rd`, `writeUsagiFile.Rd`, `validateUsagiFile.Rd` - - `appendUsagiFileToSTCMtable.Rd`, `createSourceToConceptMapExtended.Rd` -- `tests/testthat/`: - - `test-validateUsagiFile.R`, `test-appendUsagiFileToSTCMTable.R`, `test-STCMToCDMTable.R` - - `test-validateVocabularyFolder.R`, `test-buildVocabulariesAll.R` -- `NEWS.md` diff --git a/tests/testthat/test-validateUsagiFile.R b/tests/testthat/test-validateUsagiFile.R index fcb4111..7127132 100644 --- a/tests/testthat/test-validateUsagiFile.R +++ b/tests/testthat/test-validateUsagiFile.R @@ -193,8 +193,9 @@ test_that("test validateUsagiFile returns errors with the errored usagi file", { # SourceConceptId is not unique validationsSummary |> dplyr::filter(step == "SourceConceptId is not unique") |> nrow() |> expect_equal(1) validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> nrow() |> expect_equal(1) - validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal("ERROR: SourceConceptId is not unique") + # the row shares sourceConceptId with another sourceCode that has a different sourceName_fi + validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(`ADD_INFO:validationMessages`) |> + expect_equal("ERROR: SourceConceptId is not unique | ERROR: SourceNameFi is not the same for all rows of the sourceConceptId") validatedUsagiFile |> dplyr::filter(stringr::str_detect(sourceName, "SourceConceptId is not unique")) |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") @@ -324,7 +325,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { sourceCode == "A01.0+G01" ~ strrep("a", 1001), # empty sourceCode == "A01.0+I39.8" ~ NA_character_, - # different within a multi-mapped sourceCode + # different within a multi-mapped sourceCode, all rows share the same sourceConceptId sourceCode == "A02.2+H22.0" & dplyr::row_number() == min(which(sourceCode == "A02.2+H22.0")) ~ "Eri nimi", TRUE ~ `ADD_INFO:sourceName_fi` ) @@ -352,7 +353,7 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { validationsSummary |> dplyr::filter(type != "SUCCESS") |> dplyr::pull(step) |> expect_setequal(c( "SourceNameFi is more than 1000 characters", - "SourceNameFi is not the same for all rows of the sourceCode", + "SourceNameFi is not the same for all rows of the sourceConceptId", "SourceNameFi is empty" )) @@ -362,10 +363,10 @@ test_that("test validateUsagiFile validates ADD_INFO:sourceName_fi", { expect_equal("ERROR: SourceNameFi is more than 1000 characters") validatedUsagiFile |> dplyr::filter(sourceCode == "A01.0+G01") |> dplyr::pull(mappingStatus) |> expect_equal("FLAGGED") - # SourceNameFi is not the same for all rows of the sourceCode - validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceCode") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") + # SourceNameFi is not the same for all rows of the sourceConceptId + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(`ADD_INFO:validationMessages`) |> - expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceCode", 2)) + expect_equal(rep("ERROR: SourceNameFi is not the same for all rows of the sourceConceptId", 2)) validatedUsagiFile |> dplyr::filter(sourceCode == "A02.2+H22.0") |> dplyr::pull(mappingStatus) |> expect_equal(rep("FLAGGED", 2)) # SourceNameFi is empty, only a warning, mappingStatus is not changed @@ -395,3 +396,30 @@ test_that("test validateUsagiFile requires C&CR columns when ADD_INFO:sourceName validationsSummary |> dplyr::filter(step == "Missing C&CR columns") |> dplyr::pull(type) |> as.character() |> expect_equal("ERROR") }) + +test_that("test validateUsagiFile allows different ADD_INFO:sourceName_fi for different sourceConceptIds of the same sourceCode", { + + pathToOMOPVocabularyDuckDBfile <- helper_createATemporaryCopyOfTheOMOPVocabularyDuckDB() + withr::defer(unlink(pathToOMOPVocabularyDuckDBfile)) + + # a sourceCode with two sourceConceptIds valid in different periods, each with its own names + usagiTibble <- readUsagiFile(system.file("testdata/VOCABULARIES/ICD10fi/ICD10fi.usagi.csv", package = "ROMOPMappingTools")) + secondRow <- which(usagiTibble$sourceCode == "A02.2+H22.0")[2] + usagiTibble$`ADD_INFO:sourceConceptId`[secondRow] <- 2000599999 + usagiTibble$sourceName[secondRow] <- "Other name for a later period" + usagiTibble$`ADD_INFO:sourceName_fi`[secondRow] <- "Toinen nimi myöhemmälle jaksolle" + pathToUsagiFile <- tempfile(fileext = ".csv") + writeUsagiFile(usagiTibble, pathToUsagiFile) + + connection <- DatabaseConnector::connect( + dbms = "duckdb", + server = pathToOMOPVocabularyDuckDBfile + ) + on.exit(DatabaseConnector::disconnect(connection)) + + validationsSummary <- validateUsagiFile(pathToUsagiFile, connection, "main", tempfile(fileext = ".csv"), 2000500000) + + validationsSummary |> dplyr::filter(step == "SourceNameFi is not the same for all rows of the sourceConceptId") |> + dplyr::pull(type) |> as.character() |> expect_equal("SUCCESS") + validationsSummary |> dplyr::filter(type != "SUCCESS") |> nrow() |> expect_equal(0) +}) From b9afca7f8d5f6b84706f96007b82278bb249370f Mon Sep 17 00:00:00 2001 From: javier-gracia-tabuenca-tuni Date: Tue, 10 Nov 2026 21:15:10 +0200 Subject: [PATCH 3/3] Bump version to 2.2.0 and update NEWS.md Co-Authored-By: Claude Opus 5.5 --- DESCRIPTION | 2 +- NEWS.md | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3dced7b..ff07bde 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: ROMOPMappingTools Title: Tools for Working with OMOP CDM Mappings -Version: 2.1.3 +Version: 2.2.0 Authors@R: person("Javier", "Gracia-Tabuenca", , "javier.graciatabuenca@tuni.fi", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-2455-0598")) diff --git a/NEWS.md b/NEWS.md index e26e8e5..fdcf4a1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,6 @@ -# ROMOPMappingTools (development version) +# ROMOPMappingTools 2.2.0 -- Added `ADD_INFO:sourceName_fi` Usagi column. It is carried to the new `source_code_description_fi` column of the extended STCM table and inserted into `CONCEPT_SYNONYM` with `language_concept_id = 4181730` (Finnish). Names equal to `sourceName` are not added as synonyms. -- Added rules to validate usagi file: sourceName_fi must be less than 1000 characters and the same for all rows of a sourceConceptId; a warning is given if it is empty for APPROVED or UNCHECKED rows. -- `buildVocabulariesAll` checks that all Finnish names in the STCM table end up in `CONCEPT_SYNONYM`. -- Fixed non-standard `==` in the `CONCEPT_SYNONYM` query of `calculateMappingStatus`. +- Added `ADD_INFO:sourceName_fi` Usagi column, validated and loaded into `CONCEPT_SYNONYM` as Finnish synonyms. # ROMOPMappingTools 2.1.3