diff --git a/ci/.azure-pipelines.yml b/ci/.azure-pipelines.yml index 61585f9..211af25 100644 --- a/ci/.azure-pipelines.yml +++ b/ci/.azure-pipelines.yml @@ -43,6 +43,7 @@ steps: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt + pip install -r ci/ci-requirements.txt displayName: 'Create python virtual environment' @@ -67,6 +68,14 @@ steps: dbt compile displayName: 'Run dbt compile' +- script: | + source venv/bin/activate + + cd ci + /bin/bash sqlfluff-check.sh ../target/compiled + + displayName: 'Validate compiled SQL' + - script: | set -a && source .env && set +a diff --git a/ci/.sqlfluff b/ci/.sqlfluff new file mode 100644 index 0000000..bb8be3d --- /dev/null +++ b/ci/.sqlfluff @@ -0,0 +1,3 @@ +[sqlfluff] +large_file_skip_char_limit = 0 +large_file_skip_byte_limit = 0 \ No newline at end of file diff --git a/ci/ci-requirements.txt b/ci/ci-requirements.txt new file mode 100644 index 0000000..c147689 --- /dev/null +++ b/ci/ci-requirements.txt @@ -0,0 +1 @@ +sqlfluff==3.3.1 \ No newline at end of file diff --git a/ci/sqlfluff-check.sh b/ci/sqlfluff-check.sh new file mode 100644 index 0000000..bfde3f6 --- /dev/null +++ b/ci/sqlfluff-check.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +# This script gets all the .sql files in a path and checks if they are valid +# as defined by the PostgreSQL dialect. The script exits with 1 if any single +# file has an issue. The script exits with 0 if all files are valid. + +# The script takes one arg, which is the directory where the .sql files are. + +# Ensure a directory is provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +SQL_DIR=$1 + +EXIT_CODE=0 + +echo "Starting sqlfluff checks on ${SQL_DIR}" + +check_sql() { + sql_file="$1" + echo "Reading ${sql_file}" + + # Run sqlfluff parse on the file and capture the output + OUTPUT=$(sqlfluff parse --dialect postgres "$sql_file" 2>&1) + + # Check if the output contains a parsing error + if echo "$OUTPUT" | grep -q "==== parsing violations ===="; then + echo "Error in file: $sql_file" + echo "$OUTPUT" | awk '/==== parsing violations ====/{flag=1; next} /Complete/{flag=0} flag' + return 1 + fi + return 0 +} + +export -f check_sql + +# Run in parallel (adjust `-P` to change concurrency level) +# We exclude files contained in folders named schema.yml or ended in sources.yml +# because those are dbt generated test SQL snippets which, by definition, are +# valid. +find "$SQL_DIR" -type f -name "*.sql" -not -path "*/schema.yml/*" -not -path "*/*sources.yml/*" | xargs -P 4 -I {} bash -c 'check_sql "$@"' _ {} + +# Check exit status of xargs +if [ $? -ne 0 ]; then + EXIT_CODE=1 +fi + +echo "Exit code: ${EXIT_CODE}" +exit $EXIT_CODE \ No newline at end of file diff --git a/models/intermediate/kpis/int_kpis__product_guest_agg_metrics.sql b/models/intermediate/kpis/int_kpis__product_guest_agg_metrics.sql index f99c1f7..c60de4c 100644 --- a/models/intermediate/kpis/int_kpis__product_guest_agg_metrics.sql +++ b/models/intermediate/kpis/int_kpis__product_guest_agg_metrics.sql @@ -27,7 +27,7 @@ with ), date_ranges as ( select - {{ yesterday }} as current_date, + {{ yesterday }} as current_date_col, date_trunc('month', {{ yesterday }}) as current_month_start, date_trunc('week', {{ yesterday }}) as current_week_start, -- Start of the current week date_trunc('week', {{ yesterday }} - interval '1 year') as py_week_start, -- Start of the same week last year