Build process improvements (#3367)

* Fix issues with the container image build

* Fix asl-validator version check

* Disable filename rule when running ktlint tests

* Bump asl-validator from 2.2.1 to 3.0.8 in /dependencies

Bumps [asl-validator](https://github.com/ChristopheBougere/asl-validator) from 2.2.1 to 3.0.8.
- [Release notes](https://github.com/ChristopheBougere/asl-validator/releases)
- [Commits](https://github.com/ChristopheBougere/asl-validator/compare/2.2.1...3.0.8)

---
updated-dependencies:
- dependency-name: asl-validator
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* Show errors when using curl to call the status API

* Make the build more reproducible

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.editorconfig

@@ -28,3 +28,8 @@ spaces_around_brackets = false
 [{Makefile,go.mod,go.sum,*.go,.gitmodules}]
 indent_style = tab
 indent_size = 4
+
+[**/test/*.{kt,kts}]
+# Disable the filename rule because our test case logic requires that files used in test cases
+# follow a certain syntax which is not compatible with what ktlint currently mandates (PascalCase)
+disabled_rules = filename

Dockerfile

@@ -37,11 +37,14 @@ FROM python:3.10.7-alpine as base_image
 ARG ARM_TTK_NAME='master.zip'
 ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip'
 ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft'
+ARG CHECKSTYLE_VERSION='10.3.4'
 # Dart Linter
 ## stable dart sdk: https://dart.dev/get-dart#release-channels
 ARG DART_VERSION='2.8.4'
+ARG GOOGLE_JAVA_FORMAT_VERSION='1.15.0'
 ## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux)
 ARG GLIBC_VERSION='2.31-r0'
+ARG KTLINT_VERSION='0.47.1'
 # PowerShell & PSScriptAnalyzer linter
 ARG PSSA_VERSION='latest'
 ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell'
@@ -81,7 +84,8 @@ RUN apk add --no-cache \
     openssh-client \
     openssl-dev \
     perl perl-dev \
-    py3-setuptools python3-dev \
+    py3-setuptools python3-dev  \
+    py3-pyflakes \
     R R-dev R-doc \
     readline-dev \
     ruby ruby-dev ruby-bundler ruby-rdoc \
@@ -108,15 +112,7 @@ RUN npm config set package-lock true  \
 ##############################
 # Installs Perl dependencies #
 ##############################
-RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community \
+RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community
-    #######################
-    # Installs ActionLint #
-    #######################
-    && curl --retry 5 --retry-delay 5 -sLO https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash \
-    && chmod +x download-actionlint.bash \
-    && ./download-actionlint.bash \
-    && rm download-actionlint.bash \
-    && mv actionlint /usr/bin/actionlint
 
 ######################
 # Install shellcheck #
@@ -221,7 +217,7 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen
     ##################
     # Install ktlint #
     ##################
-    && curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \
+    && curl --retry 5 --retry-delay 5 -sSLO "https://github.com/pinterest/ktlint/releases/download/${KTLINT_VERSION}/ktlint" \
     && chmod a+x ktlint \
     && mv "ktlint" /usr/bin/ \
     && terrascan init \
@@ -250,14 +246,12 @@ RUN apk add --no-cache rakudo zef \
     ######################
     # Install CheckStyle #
     ######################
-    && curl --retry 5 --retry-delay 5 -sSL \
+    && curl --retry 5 --retry-delay 5 --show-error -sSL "https://github.com/checkstyle/checkstyle/releases/download/checkstyle-${CHECKSTYLE_VERSION}/checkstyle-${CHECKSTYLE_VERSION}-all.jar" \
-    "$(curl -s https://api.github.com/repos/checkstyle/checkstyle/releases/latest | jq -r '.assets[0].browser_download_url')" \
     --output /usr/bin/checkstyle \
     ##############################
     # Install google-java-format #
     ##############################
-    && curl --retry 5 --retry-delay 5 -sSL \
+    && curl --retry 5 --retry-delay 5 --show-error -sSL "https://github.com/google/google-java-format/releases/download/v${GOOGLE_JAVA_FORMAT_VERSION}/google-java-format-${GOOGLE_JAVA_FORMAT_VERSION}-all-deps.jar" \
-    "$(curl -s https://api.github.com/repos/google/google-java-format/releases/latest | jq -r '.assets | .[] | select(.browser_download_url | contains("all-deps.jar")) | .browser_download_url')" \
     --output /usr/bin/google-java-format \
     #################################
     # Install luacheck and luarocks #
@@ -393,7 +387,7 @@ ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins"
 ####################################################
 # Install Composer after all Libs have been copied #
 ####################################################
-RUN sh -c 'curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer'
+RUN sh -c 'curl --retry 5 --retry-delay 5 --show-error -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer'
 
 ########################################
 # Add node packages to path and dotnet #
@@ -507,7 +501,7 @@ RUN ln -s /usr/bin/rustup-init /usr/bin/rustup \
 # Slightly modified to always retrieve latest stable Powershell version
 # If changing PWSH_VERSION='latest' to a specific version, use format PWSH_VERSION='tags/v7.0.2'
 RUN mkdir -p ${PWSH_DIRECTORY} \
-    && curl --retry 5 --retry-delay 5 -s https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION} \
+    && curl --retry 5 --retry-delay 5 --show-error -s https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION} \
     | grep browser_download_url \
     | grep linux-alpine-x64 \
     | cut -d '"' -f 4 \
@@ -523,7 +517,7 @@ RUN mkdir -p ${PWSH_DIRECTORY} \
 # Reference https://github.com/Azure/arm-ttk
 # Reference https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/test-toolkit
 ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1"
-RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \
+RUN curl --retry 5 --retry-delay 5 --show-error -sLO "${ARM_TTK_URI}" \
     && unzip "${ARM_TTK_NAME}" -d "${ARM_TTK_DIRECTORY}" \
     && rm "${ARM_TTK_NAME}" \
     && ln -sTf "${ARM_TTK_PSD1}" /usr/bin/arm-ttk

dependencies/package-lock.json

@@ -12,7 +12,7 @@
         "@stoplight/spectral": "^6.1.0",
         "@typescript-eslint/eslint-plugin": "^4.33.0",
         "@typescript-eslint/parser": "^4.33.0",
-        "asl-validator": "^2.2.1",
+        "asl-validator": "^3.0.8",
         "axios": "^0.27.2",
         "babel-eslint": "^10.1.0",
         "eslint": "^7.32.0",
@@ -2530,9 +2530,9 @@
       }
     },
     "node_modules/ajv": {
-      "version": "8.9.0",
+      "version": "8.11.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
-      "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
+      "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "json-schema-traverse": "^1.0.0",
@@ -2735,39 +2735,36 @@
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
       "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
     },
-    "node_modules/asl-validator": {
+    "node_modules/asl-path-validator": {
-      "version": "2.2.1",
+      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-2.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.11.0.tgz",
-      "integrity": "sha512-GzG/vY94kbyGVm9Pw+oZPPZC74Aw4RVBOp0NXXCsS04KkiNS54LaZDlPUiWuKkHmmTWgWvJUlYnn1ztnE2cT7A==",
+      "integrity": "sha512-2kfFkqNCXInc7d8hbUoXn/XpK5fFr3//0nh4jfcZWav0VR4zo2bYVlRCwOuNKJID9yM4vIo7dMb4n0fnWrc/Xw==",
       "dependencies": {
-        "ajv": "^6.12.6",
+        "jsonpath-plus": "^7.0.0"
+      }
+    },
+    "node_modules/asl-path-validator/node_modules/jsonpath-plus": {
+      "version": "7.2.0",
+      "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz",
+      "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
+    "node_modules/asl-validator": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.0.8.tgz",
+      "integrity": "sha512-ku2hkt137ebImA6DNySVoBtymffl/62TQHWKBb54yI3twrcsQyi78fPtvRi+PMob89vLeb0BbGezr5+4rQcJ7Q==",
+      "dependencies": {
+        "ajv": "^8.11.0",
+        "asl-path-validator": "^0.11.0",
         "commander": "^5.1.0",
         "jsonpath-plus": "^7.0.0"
       },
       "bin": {
-        "asl-validator": "bin/asl-validator.js"
+        "asl-validator": "dist/bin/asl-validator.js"
       }
     },
-    "node_modules/asl-validator/node_modules/ajv": {
-      "version": "6.12.6",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-      "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
-      "dependencies": {
-        "fast-deep-equal": "^3.1.1",
-        "fast-json-stable-stringify": "^2.0.0",
-        "json-schema-traverse": "^0.4.1",
-        "uri-js": "^4.2.2"
-      },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/epoberezkin"
-      }
-    },
-    "node_modules/asl-validator/node_modules/json-schema-traverse": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
-    },
     "node_modules/asl-validator/node_modules/jsonpath-plus": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.0.0.tgz",
@@ -13429,9 +13426,9 @@
       }
     },
     "ajv": {
-      "version": "8.9.0",
+      "version": "8.11.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
-      "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==",
+      "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==",
       "requires": {
         "fast-deep-equal": "^3.1.1",
         "json-schema-traverse": "^1.0.0",
@@ -13570,32 +13567,32 @@
       "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
       "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
     },
-    "asl-validator": {
+    "asl-path-validator": {
-      "version": "2.2.1",
+      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-2.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.11.0.tgz",
-      "integrity": "sha512-GzG/vY94kbyGVm9Pw+oZPPZC74Aw4RVBOp0NXXCsS04KkiNS54LaZDlPUiWuKkHmmTWgWvJUlYnn1ztnE2cT7A==",
+      "integrity": "sha512-2kfFkqNCXInc7d8hbUoXn/XpK5fFr3//0nh4jfcZWav0VR4zo2bYVlRCwOuNKJID9yM4vIo7dMb4n0fnWrc/Xw==",
       "requires": {
-        "ajv": "^6.12.6",
+        "jsonpath-plus": "^7.0.0"
+      },
+      "dependencies": {
+        "jsonpath-plus": {
+          "version": "7.2.0",
+          "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz",
+          "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA=="
+        }
+      }
+    },
+    "asl-validator": {
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.0.8.tgz",
+      "integrity": "sha512-ku2hkt137ebImA6DNySVoBtymffl/62TQHWKBb54yI3twrcsQyi78fPtvRi+PMob89vLeb0BbGezr5+4rQcJ7Q==",
+      "requires": {
+        "ajv": "^8.11.0",
+        "asl-path-validator": "^0.11.0",
         "commander": "^5.1.0",
         "jsonpath-plus": "^7.0.0"
       },
       "dependencies": {
-        "ajv": {
-          "version": "6.12.6",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
-          "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
-          "requires": {
-            "fast-deep-equal": "^3.1.1",
-            "fast-json-stable-stringify": "^2.0.0",
-            "json-schema-traverse": "^0.4.1",
-            "uri-js": "^4.2.2"
-          }
-        },
-        "json-schema-traverse": {
-          "version": "0.4.1",
-          "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-          "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
-        },
         "jsonpath-plus": {
           "version": "7.0.0",
           "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.0.0.tgz",

dependencies/package.json

@@ -5,7 +5,7 @@
     "@stoplight/spectral": "^6.1.0",
     "@typescript-eslint/eslint-plugin": "^4.33.0",
     "@typescript-eslint/parser": "^4.33.0",
-    "asl-validator": "^2.2.1",
+    "asl-validator": "^3.0.8",
     "axios": "^0.27.2",
     "babel-eslint": "^10.1.0",
     "eslint": "^7.32.0",

lib/functions/linterVersions.sh

@@ -72,7 +72,7 @@ BuildLinterVersions() {
       if [[ ${LINTER} == "arm-ttk" ]]; then
         # Need specific command for ARM
         GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)"
-      elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then
+      elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "asl-validator" ]]; then
         # Need specific command for Protolint and editorconfig-checker
         GET_VERSION_CMD="$(echo "--version not supported")"
       elif [[ ${LINTER} == "lintr" ]]; then

lib/linter.sh

@@ -653,7 +653,7 @@ CallStatusAPI() {
     # Call the status API to create status check #
     ##############################################
     SEND_STATUS_CMD=$(
-      curl -f -s -X POST \
+      curl -f -s --show-error -X POST \
         --url "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \
         -H 'accept: application/vnd.github.v3+json' \
         -H "authorization: Bearer ${GITHUB_TOKEN}" \

test/inspec/inspec.lock

@@ -1,3 +0,0 @@
----
-lockfile_version: 1
-depends: []

test/inspec/super-linter/controls/super_linter.rb

@@ -13,6 +13,7 @@ control "super-linter-installed-packages" do
 
   packages = [
     "bash",
+    "ca-certificates",
     "coreutils",
     "curl",
     "gcc",
@@ -43,6 +44,7 @@ control "super-linter-installed-packages" do
     "openssl-dev",
     "perl-dev",
     "perl",
+    "py3-pyflakes",
     "py3-setuptools",
     "python3-dev",
     "rakudo",
@@ -92,7 +94,7 @@ control "super-linter-installed-commands" do
     { linter_name: "actionlint"},
     { linter_name: "ansible-lint"},
     { linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"},
-    { linter_name: "asl-validator"},
+    { linter_name: "asl-validator", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
     { linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command
     { linter_name: "black"},
     { linter_name: "clang-format"},
@@ -392,7 +394,7 @@ control "super-linter-validate-files" do
     "/action/lib/.automation/.scalafmt.conf",
     "/action/lib/.automation/.snakefmt.toml",
     "/action/lib/.automation/.sql-config.json",
-    "/action/lib/.automation//.sqlfluff",
+    "/action/lib/.automation/.sqlfluff",
     "/action/lib/.automation/.stylelintrc.json",
     "/action/lib/.automation/.tflint.hcl",
     "/action/lib/.automation/.yaml-lint.yml",