diff --git a/all.sas b/all.sas index 7894300..783da7c 100644 --- a/all.sas +++ b/all.sas @@ -7097,7 +7097,9 @@ drop table &ds1, &ds2; @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS - @li RAW_VALUE - no unquoted values except integers, commas and spaces. + @li RAW_VALUE - no unquoted values except integers, commas and spaces. For + numeric columns, SAS date, datetime and time literals are also accepted, eg + '20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -7127,7 +7129,7 @@ drop table &ds1, &ds2; @version 9.3 @author Allan Bowe - @todo Support date / hex / name literals and exponents in RAW_VALUE field + @todo Support hex / name literals and exponents in RAW_VALUE field **/ %macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); @@ -7161,8 +7163,11 @@ data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; - length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; - drop tmp; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; + drop tmp re_temporal; + retain re_temporal; + /* SAS temporal literals - date, datetime and time */ + if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); /* quick check to ensure column exists */ if upcase(VARIABLE_NM) not in @@ -7271,6 +7276,9 @@ data &outds; end; end; + /* a SAS temporal literal is a valid value for a numeric column */ + if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return; + /* special logic */ if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do; if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',','); @@ -7290,40 +7298,51 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( - 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' - 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' - ) - then do; - /* check if the raw_value contains a valid variable NAME */ - /* is not valid syntax for IN or NOT IN */ - vnum=varnum(dsid,subpad(tmp,1,32)); - if vnum>0 then do; - /* now we can get the type */ - vtype2=vartype(dsid,vnum); - /* check type matches */ - if vtype2=vtype then do; - /* valid target var - exit loop */ - return; + /* a SAS temporal literal is a valid value for a numeric column */ + if prxmatch(re_temporal,cats(tmp))=0 then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2 ne vtype then do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; end; else do; - REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); - putlog REASON_CD= dsid=; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - goto endstep; end; end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + end; end; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; end; - return; end; + /* every value of a numeric IN / BETWEEN list is checked one by one above, + so a numeric list skips the generic check below - that would only flag + the list syntax itself. A character list is unchanged. */ + if vtype='N' then return; end; else raw_value1=raw_value; diff --git a/base/mp_filtercheck.sas b/base/mp_filtercheck.sas index 4de711c..26c1a2b 100644 --- a/base/mp_filtercheck.sas +++ b/base/mp_filtercheck.sas @@ -29,7 +29,9 @@ @li SUBGROUP_ID - only integers @li VARIABLE_NM - must be in the target table @li OPERATOR_NM - only =/>/=/BETWEEN/IN/NOT IN/NE/CONTAINS - @li RAW_VALUE - no unquoted values except integers, commas and spaces. + @li RAW_VALUE - no unquoted values except integers, commas and spaces. For + numeric columns, SAS date, datetime and time literals are also accepted, eg + '20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't. @returns The &outds table containing any bad rows, plus a REASON_CD column. @@ -59,7 +61,7 @@ @version 9.3 @author Allan Bowe - @todo Support date / hex / name literals and exponents in RAW_VALUE field + @todo Support hex / name literals and exponents in RAW_VALUE field **/ %macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES); @@ -93,8 +95,11 @@ data &outds; /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32 OPERATOR_NM $10 RAW_VALUE $4000;*/ set &inds end=last; - length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000; - drop tmp; + length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8; + drop tmp re_temporal; + retain re_temporal; + /* SAS temporal literals - date, datetime and time */ + if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i"); /* quick check to ensure column exists */ if upcase(VARIABLE_NM) not in @@ -203,6 +208,9 @@ data &outds; end; end; + /* a SAS temporal literal is a valid value for a numeric column */ + if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return; + /* special logic */ if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do; if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',','); @@ -222,40 +230,51 @@ data &outds; if vtype='N' then do i=1 to countc(raw_value1, ',')+1; tmp=scan(raw_value1,i,','); if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do; - if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( - 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' - 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' - ) - then do; - /* check if the raw_value contains a valid variable NAME */ - /* is not valid syntax for IN or NOT IN */ - vnum=varnum(dsid,subpad(tmp,1,32)); - if vnum>0 then do; - /* now we can get the type */ - vtype2=vartype(dsid,vnum); - /* check type matches */ - if vtype2=vtype then do; - /* valid target var - exit loop */ - return; + /* a SAS temporal literal is a valid value for a numeric column */ + if prxmatch(re_temporal,cats(tmp))=0 then do; + if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in ( + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N' + 'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_' + ) + then do; + /* check if the raw_value contains a valid variable NAME */ + /* is not valid syntax for IN or NOT IN */ + vnum=varnum(dsid,subpad(tmp,1,32)); + if vnum>0 then do; + /* now we can get the type */ + vtype2=vartype(dsid,vnum); + /* check type matches */ + if vtype2 ne vtype then do; + REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); + putlog REASON_CD= dsid=; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + goto endstep; + end; end; else do; - REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")"); - putlog REASON_CD= dsid=; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; call symputx('reason_cd',reason_cd,'l'); call symputx('nobs',_n_,'l'); output; - goto endstep; end; end; + else do; + REASON_CD='Non Numeric value provided'; + putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; + call symputx('reason_cd',reason_cd,'l'); + call symputx('nobs',_n_,'l'); + output; + end; end; - REASON_CD='Non Numeric value provided'; - putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ; - call symputx('reason_cd',reason_cd,'l'); - call symputx('nobs',_n_,'l'); - output; end; - return; end; + /* every value of a numeric IN / BETWEEN list is checked one by one above, + so a numeric list skips the generic check below - that would only flag + the list syntax itself. A character list is unchanged. */ + if vtype='N' then return; end; else raw_value1=raw_value; diff --git a/tests/base/mp_filtercheck.test.sas b/tests/base/mp_filtercheck.test.sas index c895f33..51043bb 100644 --- a/tests/base/mp_filtercheck.test.sas +++ b/tests/base/mp_filtercheck.test.sas @@ -230,3 +230,142 @@ run; test=HASOBS, outds=work.test_results ) + +/* temporal literals are valid values for numeric columns */ +data work.dates; + length dt dtm tm 8; + format dt date9. dtm datetime20. tm time8.; + dt='20APR2042'd; + dtm='01JAN1960:00:00:42'dt; + tm='00:00:42't; +run; + +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,=,"'20APR2042'd" +AND,AND,1,dt,NE,"'20APR2042'd" +AND,AND,1,dtm,>=,"'01JAN1960:00:00:42'dt" +AND,AND,1,tm,<,"'00:00:42't" +AND,AND,1,dt,IN,"('20APR2042'd,'21APR2042'd)" +AND,AND,1,dt,BETWEEN,"'20APR2042'd AND '21APR2042'd" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Temporal literals in RAW_VALUE, + test=EMPTY, + outds=work.test_results +) + +/* a temporal literal is not valid for a character column */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,name,=,"'20APR2042'd" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.class, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Temporal literal against a character column, + test=HASOBS, + outds=work.test_results +) + +/* every value of an IN list is checked, not just the first */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,IN,"('20APR2042'd,notanumber)" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Invalid value after the first in an IN list, + test=HASOBS, + outds=work.test_results +) + +/* a suffix that is not temporal is still rejected */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,dt,=,"'20APR2042'x" +;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.dates, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Non temporal literal suffix, + test=HASOBS, + outds=work.test_results +) + +/* a character IN list is still validated - only a numeric list takes the + value-by-value path and skips the generic check */ +data work.inds; + infile datalines4 dsd; + input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32. + OPERATOR_NM:$10. RAW_VALUE:$4000.; +datalines4; +AND,AND,1,name,IN,"(Alfred,Alice)" +;;;;; +run; + +%mp_filtercheck(work.inds, + targetds=work.class, + outds=work.badrecords, + abort=NO +) +%let syscc=0; +%mp_assertdsobs(work.badrecords, + desc=Character IN list is still validated, + test=HASOBS, + outds=work.test_results +) + +/* the regex id is a working variable - it must not leak into the output */ +%let re_temporal_col=1; +data _null_; + dsid=open('work.badrecords','i'); + if dsid>0 then do; + call symputx('re_temporal_col',(varnum(dsid,'RE_TEMPORAL')>0)); + rc=close(dsid); + end; +run; +%mp_assert(iftrue=(&re_temporal_col=0), + desc=re_temporal is not in the bad records table, + outds=work.test_results +) +