data-dwh-dbt-project/ci/sqlfluff-check.sh

#!/bin/bash

# This script gets all the .sql files in a path and checks if they are valid 
# as defined by the PostgreSQL dialect. The script exits with 1 if any single
# file has an issue. The script exits with 0 if all files are valid.

# The script takes one arg, which is the directory where the .sql files are.

# Ensure a directory is provided
if [ -z "$1" ]; then
    echo "Usage: $0 <sql_directory>"
    exit 1
fi

SQL_DIR=$1

EXIT_CODE=0

echo "Starting sqlfluff checks on ${SQL_DIR}"

check_sql() {
    sql_file="$1"
    echo "Reading ${sql_file}"
    
    # Run sqlfluff parse on the file and capture the output
    OUTPUT=$(sqlfluff parse --dialect postgres "$sql_file" 2>&1)

    # Check if the output contains a parsing error
    if echo "$OUTPUT" | grep -q "==== parsing violations ===="; then
        echo "Error in file: $sql_file"
        echo "$OUTPUT" | awk '/==== parsing violations ====/{flag=1; next} /Complete/{flag=0} flag'
        return 1
    fi
    return 0
}

export -f check_sql

# Run in parallel (adjust `-P` to change concurrency level)
# We exclude files contained in folders named schema.yml or ended in sources.yml
# because those are dbt generated test SQL snippets which, by definition, are 
# valid.
find "$SQL_DIR" -type f -name "*.sql" -not -path "*/schema.yml/*" -not -path "*/*sources.yml/*" | xargs -P 4 -I {} bash -c 'check_sql "$@"' _ {}

# Check exit status of xargs
if [ $? -ne 0 ]; then
    EXIT_CODE=1
fi

echo "Exit code: ${EXIT_CODE}"
exit $EXIT_CODE