#!/bin/bash # This script gets all the .sql files in a path and checks if they are valid # as defined by the PostgreSQL dialect. The script exits with 1 if any single # file has an issue. The script exits with 0 if all files are valid. # The script takes one arg, which is the directory where the .sql files are. # Ensure a directory is provided if [ -z "$1" ]; then echo "Usage: $0 " exit 1 fi SQL_DIR=$1 EXIT_CODE=0 echo "Starting sqlfluff checks on ${SQL_DIR}" check_sql() { sql_file="$1" echo "Reading ${sql_file}" # Run sqlfluff parse on the file and capture the output OUTPUT=$(sqlfluff parse --dialect postgres "$sql_file" 2>&1) # Check if the output contains a parsing error if echo "$OUTPUT" | grep -q "==== parsing violations ===="; then echo "Error in file: $sql_file" echo "$OUTPUT" | awk '/==== parsing violations ====/{flag=1; next} /Complete/{flag=0} flag' return 1 fi return 0 } export -f check_sql # Run in parallel (adjust `-P` to change concurrency level) # We exclude files contained in folders named schema.yml or ended in sources.yml # because those are dbt generated test SQL snippets which, by definition, are # valid. find "$SQL_DIR" -type f -name "*.sql" -not -path "*/schema.yml/*" -not -path "*/*sources.yml/*" | xargs -P 4 -I {} bash -c 'check_sql "$@"' _ {} # Check exit status of xargs if [ $? -ne 0 ]; then EXIT_CODE=1 fi echo "Exit code: ${EXIT_CODE}" exit $EXIT_CODE