Skip to content

feat(mp_filtercheck): accept SAS temporal literals in RAW_VALUE #460

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
allanbowe merged 4 commits into from
Oct 16, 2026
Merged

feat(mp_filtercheck): accept SAS temporal literals in RAW_VALUE #460

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d4795b9
feat(mp_filtercheck): accept SAS temporal literals in RAW_VALUE
Oct 16, 2026
6314dd1
chore: updating all.sas
invalid-email-address Oct 16, 2026
072fdc9
fix(mp_filtercheck): address review on the temporal literal change
Oct 16, 2026
093258f
chore: updating all.sas
invalid-email-address Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 47 additions & 28 deletions all.sas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7097,7 +7097,9 @@ drop table &ds1, &ds2;
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@li RAW_VALUE - no unquoted values except integers, commas and spaces. For
numeric columns, SAS date, datetime and time literals are also accepted, eg
'20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't.

@returns The &outds table containing any bad rows, plus a REASON_CD column.

Expand Down Expand Up @@ -7127,7 +7129,7 @@ drop table &ds1, &ds2;
@version 9.3
@author Allan Bowe

@todo Support date / hex / name literals and exponents in RAW_VALUE field
@todo Support hex / name literals and exponents in RAW_VALUE field
**/

%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
Expand Down Expand Up @@ -7161,8 +7163,11 @@ data &outds;
/*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
OPERATOR_NM $10 RAW_VALUE $4000;*/
set &inds end=last;
length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
drop tmp;
length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8;
drop tmp re_temporal;
retain re_temporal;
/* SAS temporal literals - date, datetime and time */
if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i");

/* quick check to ensure column exists */
if upcase(VARIABLE_NM) not in
Expand Down Expand Up @@ -7271,6 +7276,9 @@ data &outds;
end;
end;

/* a SAS temporal literal is a valid value for a numeric column */
if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return;

/* special logic */
if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
Expand All @@ -7290,40 +7298,51 @@ data &outds;
if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
tmp=scan(raw_value1,i,',');
if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
)
then do;
/* check if the raw_value contains a valid variable NAME */
/* is not valid syntax for IN or NOT IN */
vnum=varnum(dsid,subpad(tmp,1,32));
if vnum>0 then do;
/* now we can get the type */
vtype2=vartype(dsid,vnum);
/* check type matches */
if vtype2=vtype then do;
/* valid target var - exit loop */
return;
/* a SAS temporal literal is a valid value for a numeric column */
if prxmatch(re_temporal,cats(tmp))=0 then do;
if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
)
then do;
/* check if the raw_value contains a valid variable NAME */
/* is not valid syntax for IN or NOT IN */
vnum=varnum(dsid,subpad(tmp,1,32));
if vnum>0 then do;
/* now we can get the type */
vtype2=vartype(dsid,vnum);
/* check type matches */
if vtype2 ne vtype then do;
REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
putlog REASON_CD= dsid=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
goto endstep;
end;
end;
else do;
REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
putlog REASON_CD= dsid=;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
goto endstep;
end;
end;
else do;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
end;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
return;
end;
/* every value of a numeric IN / BETWEEN list is checked one by one above,
so a numeric list skips the generic check below - that would only flag
the list syntax itself. A character list is unchanged. */
if vtype='N' then return;
end;
else raw_value1=raw_value;

Expand Down
75 changes: 47 additions & 28 deletions base/mp_filtercheck.sas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
@li SUBGROUP_ID - only integers
@li VARIABLE_NM - must be in the target table
@li OPERATOR_NM - only =/>/</<=/>=/BETWEEN/IN/NOT IN/NE/CONTAINS
@li RAW_VALUE - no unquoted values except integers, commas and spaces.
@li RAW_VALUE - no unquoted values except integers, commas and spaces. For
numeric columns, SAS date, datetime and time literals are also accepted, eg
'20APR2042'd, '01JAN1960:00:00:42'dt, '00:00:42't.

@returns The &outds table containing any bad rows, plus a REASON_CD column.

Expand Down Expand Up @@ -59,7 +61,7 @@
@version 9.3
@author Allan Bowe

@todo Support date / hex / name literals and exponents in RAW_VALUE field
@todo Support hex / name literals and exponents in RAW_VALUE field
**/

%macro mp_filtercheck(inds,targetds=,outds=work.badrecords,abort=YES);
Expand Down Expand Up @@ -93,8 +95,11 @@ data &outds;
/*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
OPERATOR_NM $10 RAW_VALUE $4000;*/
set &inds end=last;
length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000;
drop tmp;
length reason_cd $4032 vtype vtype2 $1 vnum dsid 8 tmp $4000 re_temporal 8;
drop tmp re_temporal;
retain re_temporal;
/* SAS temporal literals - date, datetime and time */
if _n_=1 then re_temporal=prxparse("/^'[^']*'(dt|d|t)$/i");

/* quick check to ensure column exists */
if upcase(VARIABLE_NM) not in
Expand Down Expand Up @@ -203,6 +208,9 @@ data &outds;
end;
end;

/* a SAS temporal literal is a valid value for a numeric column */
if vtype='N' and prxmatch(re_temporal,cats(raw_value))>0 then return;

/* special logic */
if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
Expand All @@ -222,40 +230,51 @@ data &outds;
if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
tmp=scan(raw_value1,i,',');
if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
)
then do;
/* check if the raw_value contains a valid variable NAME */
/* is not valid syntax for IN or NOT IN */
vnum=varnum(dsid,subpad(tmp,1,32));
if vnum>0 then do;
/* now we can get the type */
vtype2=vartype(dsid,vnum);
/* check type matches */
if vtype2=vtype then do;
/* valid target var - exit loop */
return;
/* a SAS temporal literal is a valid value for a numeric column */
if prxmatch(re_temporal,cats(tmp))=0 then do;
if OPERATOR_NM ='BETWEEN' and subpad(upcase(tmp),1,1) in (
'A','B','C','D','E','F','G','H','I','J','K','L','M','N'
'N','O','P','Q','R','S','T','U','V','W','X','Y','Z','_'
)
then do;
/* check if the raw_value contains a valid variable NAME */
/* is not valid syntax for IN or NOT IN */
vnum=varnum(dsid,subpad(tmp,1,32));
if vnum>0 then do;
/* now we can get the type */
vtype2=vartype(dsid,vnum);
/* check type matches */
if vtype2 ne vtype then do;
REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
putlog REASON_CD= dsid=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
goto endstep;
end;
end;
else do;
REASON_CD=cats("Compared Type (",vtype2,") is not (",vtype,")");
putlog REASON_CD= dsid=;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
goto endstep;
end;
end;
else do;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
end;
REASON_CD='Non Numeric value provided';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
return;
end;
/* every value of a numeric IN / BETWEEN list is checked one by one above,
so a numeric list skips the generic check below - that would only flag
the list syntax itself. A character list is unchanged. */
if vtype='N' then return;
end;
else raw_value1=raw_value;

Expand Down
139 changes: 139 additions & 0 deletions tests/base/mp_filtercheck.test.sas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,3 +230,142 @@ run;
test=HASOBS,
outds=work.test_results
)

/* temporal literals are valid values for numeric columns */
data work.dates;
length dt dtm tm 8;
format dt date9. dtm datetime20. tm time8.;
dt='20APR2042'd;
dtm='01JAN1960:00:00:42'dt;
tm='00:00:42't;
run;

data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,dt,=,"'20APR2042'd"
AND,AND,1,dt,NE,"'20APR2042'd"
AND,AND,1,dtm,>=,"'01JAN1960:00:00:42'dt"
AND,AND,1,tm,<,"'00:00:42't"
AND,AND,1,dt,IN,"('20APR2042'd,'21APR2042'd)"
AND,AND,1,dt,BETWEEN,"'20APR2042'd AND '21APR2042'd"
;;;;
run;

%mp_filtercheck(work.inds,
targetds=work.dates,
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
desc=Temporal literals in RAW_VALUE,
test=EMPTY,
outds=work.test_results
)

/* a temporal literal is not valid for a character column */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,name,=,"'20APR2042'd"
;;;;
run;

%mp_filtercheck(work.inds,
targetds=work.class,
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
desc=Temporal literal against a character column,
test=HASOBS,
outds=work.test_results
)

/* every value of an IN list is checked, not just the first */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,dt,IN,"('20APR2042'd,notanumber)"
;;;;
run;

%mp_filtercheck(work.inds,
targetds=work.dates,
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
desc=Invalid value after the first in an IN list,
test=HASOBS,
outds=work.test_results
)

/* a suffix that is not temporal is still rejected */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,dt,=,"'20APR2042'x"
;;;;
run;

%mp_filtercheck(work.inds,
targetds=work.dates,
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
desc=Non temporal literal suffix,
test=HASOBS,
outds=work.test_results
)

/* a character IN list is still validated - only a numeric list takes the
value-by-value path and skips the generic check */
data work.inds;
infile datalines4 dsd;
input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
OPERATOR_NM:$10. RAW_VALUE:$4000.;
datalines4;
AND,AND,1,name,IN,"(Alfred,Alice)"
;;;;;
run;

%mp_filtercheck(work.inds,
targetds=work.class,
outds=work.badrecords,
abort=NO
)
%let syscc=0;
%mp_assertdsobs(work.badrecords,
desc=Character IN list is still validated,
test=HASOBS,
outds=work.test_results
)

/* the regex id is a working variable - it must not leak into the output */
%let re_temporal_col=1;
data _null_;
dsid=open('work.badrecords','i');
if dsid>0 then do;
call symputx('re_temporal_col',(varnum(dsid,'RE_TEMPORAL')>0));
rc=close(dsid);
end;
run;
%mp_assert(iftrue=(&re_temporal_col=0),
desc=re_temporal is not in the bad records table,
outds=work.test_results
)

Loading