diff --git a/macros/tests/test_metrics.sql b/macros/tests/test_metrics.sql index 86b1ca6..072a972 100644 --- a/macros/tests/test_metrics.sql +++ b/macros/tests/test_metrics.sql @@ -1,18 +1,36 @@ {% macro metric_expression(table, metric, expression, column_name=None, condition=None) %} - select * from {{ref('re_data_base_metrics')}} + with + target as ( + select + * + from {{ ref('re_data_base_metrics') }} + where + table_name = '{{ re_data.full_table_name_values(table.identifier, table.schema, table.database)}}' + and metric = '{{ metric }}' + {% if column_name is not none %} + and column_name = '{{ column_name }}' + {% endif %} + union all + select id, table_name, null, null, null, time_window_start, time_window_end, interval_length_sec, computed_on + from {{ ref('re_data_base_metrics') }} + where + table_name = '{{ re_data.full_table_name_values(table.identifier, table.schema, table.database)}}' + limit 1 + ), + target_with_rn as ( + select row_number() over (order by column_name,metric) rn, * from target + ) + select * from target_with_rn where - table_name = '{{ re_data.full_table_name_values(table.identifier, table.schema, table.database)}}' and - metric = '{{ metric }}' and + rn = 1 and ( + ( {% if condition is not none %} {{ condition }} and {% endif %} - {% if column_name is none %} not ( {{ expression }} ) - {% else %} - column_name = '{{ column_name }}' and - not ( {{ expression }} ) - {% endif %} + ) or value is null + ) {% endmacro %} @@ -103,4 +121,4 @@ {% else %} {{ re_data.empty_table() }} {% endif %} -{% endtest %} \ No newline at end of file +{% endtest %} diff --git a/models/alerts/re_data_anomalies.sql b/models/alerts/re_data_anomalies.sql index f75efde..be5335e 100644 --- a/models/alerts/re_data_anomalies.sql +++ b/models/alerts/re_data_anomalies.sql @@ -3,6 +3,16 @@ materialized='view' ) }} + +{% set sql %} +select +name,schema,database,{{json_extract('anomaly_detector','filter')}} filter +from {{ ref('re_data_selected') }} +where {{ json_extract('anomaly_detector', 'filter') }} is not null +{% endset %} + +{%- set filters = dbt_utils.get_query_results_as_dict(sql) -%} + select z.id, z.table_name, @@ -28,21 +38,29 @@ select {{ re_data.generate_anomaly_message('z.column_name', 'z.metric', 'z.last_value', 'z.last_avg') }} as message, {{ re_data.generate_metric_value_text('z.metric', 'z.last_value') }} as last_value_text from - {{ ref('re_data_z_score')}} z -left join {{ ref('re_data_selected') }} m + {{ ref('re_data_z_score')}} z +left join {{ ref('re_data_selected') }} m on {{ split_and_return_nth_value('table_name', '.', 1) }} = m.database and {{ split_and_return_nth_value('table_name', '.', 2) }} = m.schema and {{ split_and_return_nth_value('table_name', '.', 3) }} = m.name where - case - when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score' + case + {% for filter in filters["filter"] %} + when m.database='{{ filters["database"][loop.index0] }}' and m.schema='{{ filters["schema"][loop.index0] }}' and m.name='{{ filters["name"][loop.index0] }}' + then {{ filter }} + {% endfor %} + else true + end + and + case + when {{ json_extract('m.anomaly_detector', 'name') }} = 'z_score' then abs(z_score_value) > cast({{ json_extract('m.anomaly_detector', 'threshold') }} as {{ numeric_type() }}) - when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score' + when {{ json_extract('m.anomaly_detector', 'name') }} = 'modified_z_score' then abs(modified_z_score_value) > cast( {{ json_extract('m.anomaly_detector', 'threshold') }} as {{numeric_type()}} ) - when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot' + when {{ json_extract('m.anomaly_detector', 'name') }} = 'boxplot' then ( z.last_value < z.last_first_quartile - (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) - or + or z.last_value > z.last_third_quartile + (cast( {{ json_extract('m.anomaly_detector', 'whisker_boundary_multiplier') }} as {{numeric_type()}} ) * z.last_iqr) ) else false