SAS Documentation
SASĀ® Solution for Stress Testing
Reference manual - version 08.2021
Loading...
Searching...
No Matches
irmc_script_data_quality.sas
1/* ********************************************* */
2/* Data Quality Script: Input Parameters */
3/* ********************************************* */
4
5/* TGT Authentication Ticket */
6%let rgf_tgt_ticket = ${globals.ticket};
7
8/* Get the root location of the SAS Risk Workgroup Application */
9%let sas_risk_workgroup_dir = ${globals.sas_risk_workgroup_dir};
10
11/* Cycle Id */
12%let cycle_id = ${context.cycle.key};
13
14/* Analysis Run Id */
15%let analysis_run_id = ${context.analysisRun.key};
16
17/* IRM Configuration Set Id */
18%let config_set_id = ${params.CONFIGSETID};
19
20/* Portfolio key */
21%let analysis_data_key = ${params.PORTFOLIO.key};
22
23/* RuleSet Key */
24%let ruleSet_key = ${params.RULESET.key};
25
26/* Generate Data Quality Details table? Y/N */
27%let dq_details_flg = ${params.DQDETAILS};
28
29/* Perform GL Reconciliation? Y/N */
30%let gl_reconciliation_flg = ${params.GLRECONCILIATIONFLG};
31
32/* General Ledger key */
33%let general_ledger_key = ${params.GENERALLEDGER.key};
34
35/* Log Level: 1-4 */
36%let log_level = ${params.LOGLEVEL};
37
38/* Maximum wait time for IRM job completion: <number of seconds> */
39%let max_wait = ${params.MAXWAIT};
40
41/* LASR load method (Append/Replace) */
42%let lasr_load_method = ${params.LASRLOADMETHOD};
43
44/* List of analysis data objecs to reload (if lasr_load_method = replace) */
45%let reload_key_list = ${params.RELOADSUMMARYKEYLIST} ${params.RELOADDETAILSKEYLIST};
46
47/* Convert to space separated list: ["<key1>","<key2>",...,"<keyN>"] --> <key1> <key2> ... <keyN> */
48%let reload_key_list = %sysfunc(prxchange(s/\W+/ /i, -1, %superq(reload_key_list)));
49
50/* Analysis Data and Rule Set keys */
51${function:ProcessSpreadsheetParameter(params.RULESETANALYSISDATAMAPPING,"rulesData")}
52data rulesData;
53 set rulesData;
54 dataKey = prxchange('s/.+"key":(\d+).*/$1/', 1, analysisData);
55 ruleKey = prxchange('s/.+"key":(\d+).*/$1/', 1, ruleSet);
56run;
57
58%let rule_set_keys=;
59%let analysis_data_keys=;
60proc SQL noprint;
61 select distinct
62 ruleKey,
63 dataKey
64 into
65 : rule_set_keys separated by " ",
66 : analysis_data_keys separated by " "
67 from
68 rulesData
69 ;
70quit;
71
72%put &rule_set_keys.;
73%put &analysis_data_keys.;
74
75${function:ProcessSpreadsheetParameter(params.RULESETANALYSISLOOKUPDATAMAPPING,"lookupRulesData")}
76data lookupRulesData;
77 set lookupRulesData;
78 dataKey = prxchange('s/.+"key":(\d+).*/$1/', 1, analysisData);
79 ruleKey = prxchange('s/.+"key":(\d+).*/$1/', 1, ruleSet);
80run;
81
82data rulesData;
83 set rulesData;
84 ruleKey = prxchange('s/.+"key":(\d+).*/$1/', 1, ruleSet);
85 primaryAnalysisData = analysisData;
86run;
87
88data lookupRulesData;
89 set lookupRulesData;
90 ruleKey = prxchange('s/.+"key":(\d+).*/$1/', 1, ruleSet);
91 lookupAnalysisData = analysisData;
92run;
93
94proc sql nowarn noprint;
95 create table combinedRulesData as
96 select * from rulesData
97 left join lookupRulesData
98 on rulesData.ruleKey eq lookupRulesData.ruleKey;
99quit;
100
101proc sort data=combinedRulesData;
102by ruleKey;
103run;
104
105data individualRuleJson;
106keep json;
107length json $1000;
108set combinedRulesData;
109by ruleKey;
110retain json;
111retain needToCloseLookup;
112if first.ruleKey then do;
113 * start this rule array member with key and analysisData;
114 analysisDataKey = prxchange('s/.+"key":(\d+).*/$1/', 1, analysisData);
115 json = compress('{"key":' || ruleKey || ',"analysisData":' || analysisDataKey);
116 needToCloseLookup = 0;
117 * if there is at least one lookup start the lookup array;
118 if dataDefinition ne '' then do;
119 needToCloseLookup = 1;
120 json = compress(json || ',"lookup":[');
121 dataDefKey = prxchange('s/.+"key":(\d+).*/$1/', 1, dataDefinition);
122 analysisDataKey = prxchange('s/.+"key":(\d+).*/$1/', 1, lookupAnalysisData);
123 json = trim(json) || compress('{"ruleId":"' || rule_id || '",');
124 json = trim(json) || compress('"ruleRowKey":"' || RULESET_ROW_KEY || '",');
125 json = trim(json) || compress('"dataDef":' || dataDefKey || ',');
126 json = trim(json) || compress('"analysisData":' || analysisDataKey || '}');
127 end;
128end;
129
130if not first.ruleKey then do;
131* continue with additional lookup array elements;
132 if dataDefinition ne '' then do;
133 dataDefKey = prxchange('s/.+"key":(\d+).*/$1/', 1, dataDefinition);
134 analysisDataKey = prxchange('s/.+"key":(\d+).*/$1/', 1, lookupAnalysisData);
135 json = trim(json) || compress(',{"ruleId":"' || rule_id || '",');
136 json = trim(json) || compress('"ruleRowKey":"' || RULESET_ROW_KEY || '",');
137 json = trim(json) || compress('"dataDef":' || dataDefKey || ',');
138 json = trim(json) || compress('"analysisData":' || analysisDataKey || '}');
139 end;
140end;
141
142if last.ruleKey then do;
143 if needToCloseLookup then do;
144 json = trim(json) || ']';
145 end;
146 json = trim(json) || '}';
147 output;
148end;
149run;
150
151data _null_;
152 keep ruleSetJson;
153 retain ruleSetJson;
154 length ruleSetJson $10000;
155 set individualRuleJson end=last;
156
157 if _n_ eq 1 then do;
158 ruleSetJson = '{"ruleSet":[' || trim (json);
159 end;
160 else do;
161 ruleSetJson = trim(ruleSetJson) || ',' || trim(json);
162 end;
163
164 if last then do;
165 ruleSetJson = trim(ruleSetJson) || ']}';
166 ruleSetJson = tranwrd(ruleSetJson,'"','""');
167 call symput('rules_json', ruleSetJson);
168 end;
169run;
170
171/* ********************************************* */
172
173/* Initialize the environment */
174%include "&sas_risk_workgroup_dir./groups/Public/SASRiskManagementCore/cycles/&cycle_id./init.sas" / lrecl = 32000 source2;
175
176/* Set logging options (based on the value of LOG_LEVEL macro variable) */
177%irm_set_logging_options();
178
179/* Set Instance Parameters */
180%let jobflow_category = Analytics;
181%let jobflow_type = irm%lowcase(&solutionId.)_data_quality;
182%let instance_date = %sysfunc(date(), yymmddp10.);
183%let instance_time = %sysfunc(prxchange(s/(\d+):(\d+):(\d+)/$1h$2m$3s/i, -1, %sysfunc(time(), tod8.)));
184%let instanceName = Data Quality &instance_date._&instance_time. Cycle &cycle_id.;
185%let instanceDesc = Data Quality;
186
187/* Process LASR load method */
188%let reload_lasr_flg = %irmc_process_lasr_load_method(&lasr_load_method.);
189
190/* Create parameters table */
191data work.run_option;
192 length
193 config_name $32.
194 config_value $10000.
195 ;
196 config_name = "RMC_FA_ID"; config_value = "&rmc_fa_id."; output;
197 config_name = "CYCLE_ID"; config_value = "&cycle_id."; output;
198 config_name = "ANALYSIS_RUN_ID"; config_value = "&analysis_run_id."; output;
199 config_name = "MAX_WAIT"; config_value = "&max_wait."; output;
200 config_name = "LOG_LEVEL"; config_value = "&log_level."; output;
201 config_name = "RELOAD_LASR_FLG"; config_value = "&reload_lasr_flg."; output;
202 config_name = "TGT_TICKET"; config_value = "&rgf_tgt_ticket."; output;
203 config_name = "RELOAD_KEY_LIST"; config_value = "&reload_key_list."; output;
204run;
205
206 /*%local quote_option;*/
207 /* Get the current system option */
208 %let quote_option = %sysfunc(getoption(quotelenmax));
209 /* Avoid message about unbalanced quotes */
210 option noquotelenmax;
211
212data work.dq_config;
213 length
214 config_name $32.
215 config_value $10000.
216 ;
217 config_name = "ANALYSIS_DATA_KEY_LIST"; config_value = "&analysis_data_keys."; output;
218 config_name = "DQ_RULE_SET_KEY_LIST"; config_value = "&rule_set_keys."; output;
219 config_name = "DQ_DETAILS_FLG"; config_value = "&dq_details_flg."; output;
220 config_name = "RULES_JSON"; config_value = "&rules_json."; output;
221 config_name = "GL_RECONCILIATION_FLG"; config_value = "&gl_reconciliation_flg."; output;
222 config_name = "GENERAL_LEDGER_KEY"; config_value = "&general_ledger_key."; output;
223run;
224
225/* Restore system options */
226option &quote_option.;
227
228%let instanceKey =;
229%let jobflow_status = Unknown;
230/* Create job flow instance and wait for completion */
231%irm_rest_create_jobflow_instance(instance_name = &instanceName.
232 , description = &instanceDesc.
233 , entityId = &entity_id.
234 , entityRoleKey = &entity_role_key.
235 , category = &jobflow_category.
236 , jobflowFile = &jobflow_type.
237 , baseDate = &irm_base_dt.
238 , configSetId = &config_set_id.
239 , federatedAreaID = &irm_fa_id.
240 , sourceFederatedAreaId = &irm_fa_id.
241 , src_param_tables = work.run_option work.dq_config
242 , tgt_param_tables = &solutionLibrefPrefix._CFG.RUN_OPTION &solutionLibrefPrefix._CFG.DQ_CONFIG
243 , wait_flg = Y
244 , pollInterval = 3
245 , maxWait = &max_wait.
246 , host = &irm_protocol.://&irm_host.
247 , port = &irm_port.
248 , tgt_ticket = &rgf_tgt_ticket.
249 , outvar = instanceKey
250 , outVarStatus = jobflow_status
251 );
252
253%if(&jobflow_status. = Successful) %then %do;
254 /* Refresh the Star Schema Views */
255 %irmc_refresh_lasr_schema_view(lasr_lib = &lasr_libref.
256 , mart_table_name = &solutionId._DQ_SUMMARY
257 , rls_table_name = &solutionId._DQ_SUMMARY_RLS
258 , meta_repository = &meta_repository.
259 , lasr_library_name = &lasr_library_name.
260 , lasr_meta_folder = &lasr_meta_folder.
261 );
262
263 %irmc_refresh_lasr_schema_view(lasr_lib = &lasr_libref.
264 , mart_table_name = &solutionId._DQ_DETAILS
265 , rls_table_name = &solutionId._DQ_DETAILS_RLS
266 , meta_repository = &meta_repository.
267 , lasr_library_name = &lasr_library_name.
268 , lasr_meta_folder = &lasr_meta_folder.
269 );
270%end;
271
272/* Update the Cycle control table */
273%irmc_update_ctrl_table(cycle_id = &cycle_id.
274 , analysis_run_id = &analysis_run_id.
275 , dr_libref = &dr_libref.
276 , entry_type = IRM
277 , entry_id = &instanceKey.
278 , entry_name = &instanceName.
279 );