SAS Documentation
SASĀ® Solution for Stress Testing
Reference manual - version 08.2021
Loading...
Searching...
No Matches
irmst_node_dq_init.sas
Go to the documentation of this file.
1/*
2 Copyright (C) 2018 SAS Institute Inc. Cary, NC, USA
3*/
4
5/** \file
6 \brief Initialize Data Quality parameters
7
8 \param[in] ds_in_dq_config parameter table containing the list of AnalysisData keys and related RuleSet keys
9 \param[out] ds_out_dq_config Output table containing a record for each combination of AnalysisData key, RuleSet key and rule JSON
10
11 \details
12
13 This node converts the structure of the input table into a table with one record for each combination of AnalysisData key and the corresponding RuleSet key and rule JSON
14
15 The structure of the input table RMC_CFG.DQ_CONFIG is as follows:
16
17 | CONFIG_NAME | CONFIG_VALUE | CONFIG_VALUE_DESC |
18 |-------------------------|---------------------------------------------------------------|-------------------------------------------------------------------------------|
19 | ANALYSIS_DATA_KEY_LIST | <AnalysisDataKey1> <AnalysisDataKey2> .. <AnalysisDataKeyN> | Space separated list of Analysis Data Keys |
20 | DQ_RULE_SET_KEY_LIST | <RuleSetKey1> <RuleSetKey2> .. <RuleSetKeyN> | Space separated list of Rule Set Keys |
21 | DQ_DETAILS_FLG | <Y/N> | Controls whether to produce detailed data quality information |
22 | RULES_JSON | JSON string | JSON string describing all rules |
23 | GL_RECONCILIATION_FLG | <Y/N> | Controls whether to aggregate portfolio data and compare with GL Account data |
24 | GENERAL_LEDGER_KEY | <GLAnalysisDataKey> | GL Analysis Data Key |
25
26 The structure of the output table ST_STG.DQ_CONFIG is as follows:
27
28 | ANALYSIS_DATA_KEY | DQ_RULE_SET_KEY | DQ_DETAILS_FLG | RULE_JSON | GL_RECONCILIATION_FLG | GL_ANALYSIS_DATA_KEY |
29 |-------------------------|------------------|----------------|------------------------|-----------------------|----------------------|
30 | <AnalysisDataKey1> | <RuleSetKey1> | <Y/N> | JSON describing rule 1 | <Y/N> | <GLAnalysisDataKey1> |
31 | <AnalysisDataKey2> | <RuleSetKey2> | <Y/N> | JSON describing rule 2 | <Y/N> | <GLAnalysisDataKey2> |
32 | ... | ... | <Y/N> | | <Y/N> | ... |
33 | <AnalysisDataKeyN> | <RuleSetKeyN> | <Y/N> | JSON describing rule N | <Y/N> | <GLAnalysisDataKeyN> |
34
35
36 In addition the following macro utilities are called:
37
38 | Macro name | Description | Further information |
39 |---------------------------|--------------------------------------------------------------------------------------------------------------|-----------------------------------------------|
40 | irm_session_prepare | Reads RUN_OPTION table and sets logging options | \link irm_session_prepare.sas \endlink |
41 | irm_session_cleanup | Removes all non-IRM WORK datasets/views and deletes all user-created macro variables from workspace session | \link irm_session_cleanup.sas \endlink |
42
43 \ingroup nodes
44 \author SAS Institute Inc.
45 \date 2018
46*/
47
48/* Initialize session */
49%irm_session_prepare();
50
51%macro split_rule_json_by_rule_id();
52 %if(/* analysis_data_key_list is not missing */
53 %sysevalf(%superq(analysis_data_key_list) ne, boolean)
54 /* the lists have the same number of items */
55 and %sysfunc(countw(&analysis_data_key_list., %str( ))) = %sysfunc(countw(&dq_rule_set_key_list., %str( )))
56 ) %then %do;
57
58 filename dqrsjson temp;
59 filename onersjsn temp;
60
61 /* set up for parsing using JSON libname engine */
62 data _null_;
63 file dqrsjson;
64 length JSON $10000;
65 JSON = symget('RULES_JSON');
66 put JSON;
67 run;
68
69 libname jsonlibn JSON fileref=dqrsjson;
70
71 /* the libname engine only creates rules_lookup if lookup tables are present, handle both cases here */
72 %if(%rsk_dsexist(jsonlibn.ruleset_lookup)) %then %do;
73 proc sql noprint;
74 create table parsedJson as
75 select a.key, a.analysisData as primAnalysisData, b.ruleId, b.ruleRowKey, b.dataDef, b.analysisData
76 from jsonlibn.ruleset a left join jsonlibn.ruleset_lookup b
77 on a.ordinal_ruleSet = b.ordinal_ruleSet
78 ;
79 quit;
80 %end;
81 %else %do;
82 data parsedJson;
83 set jsonlibn.ruleset(rename=(analysisData=primAnalysisData));
84 keep key primAnalysisData dataDef;
85 dataDef = .;
86 run;
87 %end;
88
89 %local i next_dq_rule_set_key;
90 %let i=1;
91
92 /* create individual JSON representation of each rule in the rule set, key-by-key */
93 %do %while (%scan(&dq_rule_set_key_list, &i) ne );
94 %let next_dq_rule_set_key = %scan(&dq_rule_set_key_list, &i);
95
96 data parsedJsonOneRule;
97 set parsedJson;
98 where key eq &next_dq_rule_set_key.;
99 call symput("primAnalysisData", primAnalysisData);
100 run;
101
102 /* only write lookup array in JSON if lookups are present, test is in PROC JSON call */
103 proc sql noprint;
104 select count(dataDef) into :numLookups
105 from parsedJsonOneRule;
106 quit;
107
108 proc json out=onersjsn nosastags;
109 write open object;
110 write values "ruleSet";
111 write open array;
112 write open object;
113 write values "key" &next_dq_rule_set_key.;
114 write values "analysisData" &primAnalysisData.;
115 %if &numLookups. gt 0 %then %do;
116 write values "lookup";
117 write open array ;
118 export parsedJsonOneRule(drop=key primAnalysisData);
119 write close;
120 %end;
121 write close;
122 write close;
123 write close;
124 run;
125
126 /* write individual rule JSON to symbol table */
127 data _null_;
128 infile onersjsn;
129 input;
130 put _infile_;
131 call symputx(compress('RULE_JSON_' || &i.), _infile_, 'G');
132 run;
133
134 %let i = %eval(&i + 1);
135
136 %end;
137%end;
138%mend;
139
140%macro create_rule_json_from_keys();
141 %if(/* analysis_data_key_list is not missing */
142 %sysevalf(%superq(analysis_data_key_list) ne, boolean)
143 /* the lists have the same number of items */
144 and %sysfunc(countw(&analysis_data_key_list., %str( ))) = %sysfunc(countw(&dq_rule_set_key_list., %str( )))
145 ) %then %do;
146
147 data _null_;
148 length json $1000;
149 array analysis_data_list {%sysfunc(countw(&analysis_data_key_list., %str( )))} _temporary_ (&analysis_data_key_list.);
150 array dq_rule_set_list {%sysfunc(countw(&dq_rule_set_key_list., %str( )))} _temporary_ (&dq_rule_set_key_list.);
151 drop i;
152 do i = 1 to dim(analysis_data_list);
153 analysis_data_key = analysis_data_list[i];
154 dq_rule_set_key = dq_rule_set_list[i];
155 json = compress('{"ruleSet":[{"key":' || dq_rule_set_key || ',"analysisData":' || analysis_data_key || '}]}');
156 call symputx(compress('RULE_JSON_' || i), json, 'G');
157 end;
158 run;
159 %end;
160%mend;
161
162%if %length(&RULES_JSON.) > 0 %then %do;
163 %split_rule_json_by_rule_id();
164%end;
165%else %do;
166 %create_rule_json_from_keys();
167%end;
168
169data &ds_out_dq_config.;
170 length
171 analysis_data_key 8.
172 dq_rule_set_key 8.
173 dq_details_flg $3.
174 rule_json $10000.
175 gl_reconciliation_flg $3.
176 general_ledger_key 8.
177 ;
178
179 dq_details_flg = "%upcase(&dq_details_flg.)";
180
181 /* Stress Testing code */
182 gl_reconciliation_flg = "%upcase(&gl_reconciliation_flg.)";
183 %if(%sysevalf(%upcase(%superq(gl_reconciliation_flg)) eq Y, boolean)) %then %do;
184 general_ledger_key = &general_ledger_key.;
185 %end;
186 /* End Stress Testing code */
187
188 %if(/* analysis_data_key_list is not missing */
189 %sysevalf(%superq(analysis_data_key_list) ne, boolean)
190 /* the lists have the same number of items */
191 and %sysfunc(countw(&analysis_data_key_list., %str( ))) = %sysfunc(countw(&dq_rule_set_key_list., %str( )))
192 ) %then %do;
193
194 array analysis_data_list {%sysfunc(countw(&analysis_data_key_list., %str( )))} _temporary_ (&analysis_data_key_list.);
195 array dq_rule_set_list {%sysfunc(countw(&dq_rule_set_key_list., %str( )))} _temporary_ (&dq_rule_set_key_list.);
196 drop i;
197 do i = 1 to dim(analysis_data_list);
198 analysis_data_key = analysis_data_list[i];
199 dq_rule_set_key = dq_rule_set_list[i];
200
201 rule_json = symget(compress('RULE_JSON_' || i));
202 output;
203 end;
204 %end;
205 %else %do;
206 /* Create empty table */
207 stop;
208 %end;
209
210run;
211
212/* Cleanup session */
213%irm_session_cleanup;