Merged PR 4753: Add sqlfluff check

# Description

Adds another check to CI that validates that compiled SQL is valid.

Related work items: #28628

ci/.azure-pipelines.yml

@@ -43,6 +43,7 @@ steps:
     python3 -m venv venv
     source venv/bin/activate
     pip install -r requirements.txt
+    pip install -r ci/ci-requirements.txt
 
   displayName: 'Create python virtual environment'
 
@@ -67,6 +68,14 @@ steps:
     dbt compile
   displayName: 'Run dbt compile'
 
+- script: |
+    source venv/bin/activate
+
+    cd ci
+    /bin/bash sqlfluff-check.sh ../target/compiled
+
+  displayName: 'Validate compiled SQL'
+
 - script: |
     set -a && source .env && set +a
 

ci/.sqlfluff

@@ -0,0 +1,3 @@
+[sqlfluff]
+large_file_skip_char_limit = 0
+large_file_skip_byte_limit = 0

ci/ci-requirements.txt

@@ -0,0 +1 @@
+sqlfluff==3.3.1

ci/sqlfluff-check.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# This script gets all the .sql files in a path and checks if they are valid 
+# as defined by the PostgreSQL dialect. The script exits with 1 if any single
+# file has an issue. The script exits with 0 if all files are valid.
+
+# The script takes one arg, which is the directory where the .sql files are.
+
+# Ensure a directory is provided
+if [ -z "$1" ]; then
+    echo "Usage: $0 <sql_directory>"
+    exit 1
+fi
+
+SQL_DIR=$1
+
+EXIT_CODE=0
+
+echo "Starting sqlfluff checks on ${SQL_DIR}"
+
+check_sql() {
+    sql_file="$1"
+    echo "Reading ${sql_file}"
+    
+    # Run sqlfluff parse on the file and capture the output
+    OUTPUT=$(sqlfluff parse --dialect postgres "$sql_file" 2>&1)
+
+    # Check if the output contains a parsing error
+    if echo "$OUTPUT" | grep -q "==== parsing violations ===="; then
+        echo "Error in file: $sql_file"
+        echo "$OUTPUT" | awk '/==== parsing violations ====/{flag=1; next} /Complete/{flag=0} flag'
+        return 1
+    fi
+    return 0
+}
+
+export -f check_sql
+
+# Run in parallel (adjust `-P` to change concurrency level)
+# We exclude files contained in folders named schema.yml or ended in sources.yml
+# because those are dbt generated test SQL snippets which, by definition, are 
+# valid.
+find "$SQL_DIR" -type f -name "*.sql" -not -path "*/schema.yml/*" -not -path "*/*sources.yml/*" | xargs -P 4 -I {} bash -c 'check_sql "$@"' _ {}
+
+# Check exit status of xargs
+if [ $? -ne 0 ]; then
+    EXIT_CODE=1
+fi
+
+echo "Exit code: ${EXIT_CODE}"
+exit $EXIT_CODE

models/intermediate/kpis/int_kpis__product_guest_agg_metrics.sql

@@ -27,7 +27,7 @@ with
     ),
     date_ranges as (
         select
-            {{ yesterday }} as current_date,
+            {{ yesterday }} as current_date_col,
             date_trunc('month', {{ yesterday }}) as current_month_start,
             date_trunc('week', {{ yesterday }}) as current_week_start,  -- Start of the current week
             date_trunc('week', {{ yesterday }} - interval '1 year') as py_week_start,  -- Start of the same week last year