parallelize with xargs

ci/sqlfluff-check.sh

@@ -18,20 +18,32 @@ EXIT_CODE=0
 
 echo "Starting sqlfluff checks on ${SQL_DIR}"
 
-while read -r sql_file; do 
+check_sql() {
+    sql_file="$1"
+    echo "Reading ${sql_file}"
     
-    echo "Parsing $sql_file"
     # Run sqlfluff parse on the file and capture the output
     OUTPUT=$(sqlfluff parse --dialect postgres "$sql_file" 2>&1)
 
-    
+    echo "Output"
     # Check if the output contains a parsing error
     if echo "$OUTPUT" | grep -q "==== parsing violations ===="; then
         echo "Error in file: $sql_file"
         echo "$OUTPUT" | awk '/==== parsing violations ====/{flag=1; next} /Complete/{flag=0} flag'
-        EXIT_CODE=1
+        return 1
     fi
-done < <(find "$SQL_DIR" -type f -name "*.sql")
+    return 0
+}
+
+export -f check_sql
+
+# Run in parallel (adjust `-P` to change concurrency level)
+find "$SQL_DIR" -type f -name "*.sql" | xargs -P 4 -I {} bash -c 'check_sql "$@"' _ {}
+
+# Check exit status of xargs
+if [ $? -ne 0 ]; then
+    EXIT_CODE=1
+fi
 
 echo "Exit code: ${EXIT_CODE}"
 exit $EXIT_CODE