From d4795b9b67bd75f1546fb4ff6d1da06d64a0f935 Mon Sep 17 00:00:00 2001 From: dc Date: Fri, 16 Oct 2026 02:16:04 +0000 Subject: [PATCH 1/4] feat(mp_filtercheck): accept SAS temporal literals in RAW_VALUE Date, datetime and time literals (eg '20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't) are valid values for a numeric column, but the check rejected them: it strips quoted strings and then flags any non-digit remainder, so the trailing d/dt/t failed the test. The macro header carried this as a TODO. Temporal literals are now accepted for numeric columns with the comparison operators and inside IN / NOT IN / BETWEEN lists. They remain invalid for character columns, and other literal suffixes (hex, name) are still rejected. While adding the list cases, the value loop was corrected: the return sat inside the per-value loop, so only the first value of a numeric IN / NOT IN / BETWEEN list was ever checked. Every value is now validated, and a bad one produces a bad record rather than passing silently. --- base/mp_filtercheck.sas | 49 +++++++++----- tests/base/mp_filtercheck.test.sas | 101 +++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 16 deletions(-) diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 4de711c3..2e89d3ce 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -29,7 +29,9 @@ @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS - @li RAW_VALUE - no unquoted values except integers, commas and spaces. + @li RAW_VALUE - no unquoted values except integers, commas and spaces. For + numeric columns, SAS date, datetime and time literals are also accepted, eg + '20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -59,7 +61,7 @@ @version 9.3 @author Allan Bowe - @todo Support date / hex / name literals and exponents in RAW_VALUE field + @todo Support hex / name literals and exponents in RAW_VALUE field **/ %macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); @@ -93,8 +95,11 @@ data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; - length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; drop tmp; + retain re_temporal; + /* SAS temporal literals - date, datetime and time */ + if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); /* quick check to ensure column exists */ if upcase(VARIABLE_NM) not in @@ -203,6 +208,9 @@ data &outds; end; end; + /* a SAS temporal literal is a valid value for a numeric column */ + if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return; + /* special logic */ if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do; if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',','); @@ -222,7 +230,10 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + if prxmatch(re_temporal,cats(tmp))>0 then do; + /* valid SAS temporal literal (eg '20APR2042'd) */ + end; + else if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' ) @@ -234,28 +245,34 @@ data &outds; /* now we can get the type */ vtype2=vartype(dsid,vnum); /* check type matches */ - if vtype2=vtype then do; - /* valid target var - exit loop */ - return; - end; - else do; + if vtype2 ne vtype then do; REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); putlog REASON_CD= dsid=; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - goto endstep; end; end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + end; + end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; end; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; end; - return; end; + /* the values of a numeric IN / BETWEEN list are checked one by one above, + so skip the generic check below - it would flag the list brackets */ + return; end; else raw_value1=raw_value; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index c895f337..f15d0c00 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -230,3 +230,104 @@ run; test=HASOBS, outds=work.test_results ) + +/* temporal literals are valid values for numeric columns */ +data work.dates; + length dt dtm tm 8; + format dt date9. dtm datetime20. tm time8.; + dt='20APR2042'd; + dtm='01JAN1960:00:00:42'dt; + tm='00:00:42't; +run; + +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,=,"'20APR2042'd" +AND,AND,1,dt,NE,"'20APR2042'd" +AND,AND,1,dtm,>=,"'01JAN1960:00:00:42'dt" +AND,AND,1,tm,<,"'00:00:42't" +AND,AND,1,dt,IN,"('20APR2042'd,'21APR2042'd)" +AND,AND,1,dt,BETWEEN,"'20APR2042'd AND '21APR2042'd" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Temporal literals in RAW_VALUE, + test=EMPTY, + outds=work.test_results +) + +/* a temporal literal is not valid for a character column */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,name,=,"'20APR2042'd" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.class, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Temporal literal against a character column, + test=HASOBS, + outds=work.test_results +) + +/* every value of an IN list is checked, not just the first */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,IN,"('20APR2042'd,notanumber)" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Invalid value after the first in an IN list, + test=HASOBS, + outds=work.test_results +) + +/* a suffix that is not temporal is still rejected */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,=,"'20APR2042'x" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Non temporal literal suffix, + test=HASOBS, + outds=work.test_results +) From 6314dd1894a91aca0e48f7cc6e38fef7fee3e499 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 16 Oct 2026 09:35:06 +0000 Subject: [PATCH 2/4] chore: updating all.sas --- all.sas | 49 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/all.sas b/all.sas index 7894300f..2d66fab2 100644 --- a/all.sas +++ b/all.sas @@ -7097,7 +7097,9 @@ drop table &ds1, &ds2; @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS - @li RAW_VALUE - no unquoted values except integers, commas and spaces. + @li RAW_VALUE - no unquoted values except integers, commas and spaces. For + numeric columns, SAS date, datetime and time literals are also accepted, eg + '20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -7127,7 +7129,7 @@ drop table &ds1, &ds2; @version 9.3 @author Allan Bowe - @todo Support date / hex / name literals and exponents in RAW_VALUE field + @todo Support hex / name literals and exponents in RAW_VALUE field **/ %macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); @@ -7161,8 +7163,11 @@ data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; - length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; drop tmp; + retain re_temporal; + /* SAS temporal literals - date, datetime and time */ + if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); /* quick check to ensure column exists */ if upcase(VARIABLE_NM) not in @@ -7271,6 +7276,9 @@ data &outds; end; end; + /* a SAS temporal literal is a valid value for a numeric column */ + if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return; + /* special logic */ if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do; if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',','); @@ -7290,7 +7298,10 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + if prxmatch(re_temporal,cats(tmp))>0 then do; + /* valid SAS temporal literal (eg '20APR2042'd) */ + end; + else if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' ) @@ -7302,28 +7313,34 @@ data &outds; /* now we can get the type */ vtype2=vartype(dsid,vnum); /* check type matches */ - if vtype2=vtype then do; - /* valid target var - exit loop */ - return; - end; - else do; + if vtype2 ne vtype then do; REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); putlog REASON_CD= dsid=; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - goto endstep; end; end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + end; + end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; end; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; end; - return; end; + /* the values of a numeric IN / BETWEEN list are checked one by one above, + so skip the generic check below - it would flag the list brackets */ + return; end; else raw_value1=raw_value; From 072fdc930fdbb5f6f8419f3112064c2dddefcd22 Mon Sep 17 00:00:00 2001 From: dc Date: Fri, 16 Oct 2026 10:11:03 +0000 Subject: [PATCH 3/4] fix(mp_filtercheck): address review on the temporal literal change - gate the early return on vtype='N'. The value-by-value path is numeric only, and a character IN / BETWEEN list fell through to the generic check before - it still does. The unconditional return silently changed that. - restore `goto endstep` on a compared-type mismatch inside a list, so the remaining values of that record are not re-checked after it is already flagged. - drop re_temporal from the output, so the working regex id does not become a column of the bad records table. - remove the empty `then do; end;` block - the temporal test is inverted. Two tests added: a character IN list is still validated, and re_temporal is not in the bad records table. --- base/mp_filtercheck.sas | 60 +++++++++++++++--------------- tests/base/mp_filtercheck.test.sas | 38 +++++++++++++++++++ 2 files changed, 69 insertions(+), 29 deletions(-) diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 2e89d3ce..26c1a2bb 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -96,7 +96,7 @@ data &outds; OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; - drop tmp; + drop tmp re_temporal; retain re_temporal; /* SAS temporal literals - date, datetime and time */ if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); @@ -230,24 +230,32 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if prxmatch(re_temporal,cats(tmp))>0 then do; - /* valid SAS temporal literal (eg '20APR2042'd) */ - end; - else if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( - 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' - 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' - ) - then do; - /* check if the raw_value contains a valid variable NAME */ - /* is not valid syntax for IN or NOT IN */ - vnum=varnum(dsid,subpad(tmp,1,32)); - if vnum>0 then do; - /* now we can get the type */ - vtype2=vartype(dsid,vnum); - /* check type matches */ - if vtype2 ne vtype then do; - REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); - putlog REASON_CD= dsid=; + /* a SAS temporal literal is a valid value for a numeric column */ + if prxmatch(re_temporal,cats(tmp))=0 then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2 ne vtype then do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; @@ -261,18 +269,12 @@ data &outds; output; end; end; - else do; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; end; end; - /* the values of a numeric IN / BETWEEN list are checked one by one above, - so skip the generic check below - it would flag the list brackets */ - return; + /* every value of a numeric IN / BETWEEN list is checked one by one above, + so a numeric list skips the generic check below - that would only flag + the list syntax itself. A character list is unchanged. */ + if vtype='N' then return; end; else raw_value1=raw_value; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index f15d0c00..51043bbb 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -331,3 +331,41 @@ run; test=HASOBS, outds=work.test_results ) + +/* a character IN list is still validated - only a numeric list takes the + value-by-value path and skips the generic check */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,name,IN,"(Alfred,Alice)" +;;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.class, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Character IN list is still validated, + test=HASOBS, + outds=work.test_results +) + +/* the regex id is a working variable - it must not leak into the output */ +%let re_temporal_col=1; +data _null_; + dsid=open('work.badrecords','i'); + if dsid>0 then do; + call symputx('re_temporal_col',(varnum(dsid,'RE_TEMPORAL')>0)); + rc=close(dsid); + end; +run; +%mp_assert(iftrue=(&re_temporal_col=0), + desc=re_temporal is not in the bad records table, + outds=work.test_results +) + From 093258f96584e4f6f3470fa7b6eb394c1cc02ae7 Mon Sep 17 00:00:00 2001 From: github-actions Date: Fri, 16 Oct 2026 10:11:32 +0000 Subject: [PATCH 4/4] chore: updating all.sas --- all.sas | 60 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/all.sas b/all.sas index 2d66fab2..783da7c3 100644 --- a/all.sas +++ b/all.sas @@ -7164,7 +7164,7 @@ data &outds; OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; - drop tmp; + drop tmp re_temporal; retain re_temporal; /* SAS temporal literals - date, datetime and time */ if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); @@ -7298,24 +7298,32 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if prxmatch(re_temporal,cats(tmp))>0 then do; - /* valid SAS temporal literal (eg '20APR2042'd) */ - end; - else if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( - 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' - 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' - ) - then do; - /* check if the raw_value contains a valid variable NAME */ - /* is not valid syntax for IN or NOT IN */ - vnum=varnum(dsid,subpad(tmp,1,32)); - if vnum>0 then do; - /* now we can get the type */ - vtype2=vartype(dsid,vnum); - /* check type matches */ - if vtype2 ne vtype then do; - REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); - putlog REASON_CD= dsid=; + /* a SAS temporal literal is a valid value for a numeric column */ + if prxmatch(re_temporal,cats(tmp))=0 then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2 ne vtype then do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; + end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; @@ -7329,18 +7337,12 @@ data &outds; output; end; end; - else do; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; - end; end; end; - /* the values of a numeric IN / BETWEEN list are checked one by one above, - so skip the generic check below - it would flag the list brackets */ - return; + /* every value of a numeric IN / BETWEEN list is checked one by one above, + so a numeric list skips the generic check below - that would only flag + the list syntax itself. A character list is unchanged. */ + if vtype='N' then return; end; else raw_value1=raw_value;