From 808436342ff40e12ce3ba5d9cfbad17a9c0eee99 Mon Sep 17 00:00:00 2001 From: Marco Ferrari Date: Wed, 28 Sep 2022 15:45:01 +0200 Subject: [PATCH] Build process improvements (#3367) * Fix issues with the container image build * Fix asl-validator version check * Disable filename rule when running ktlint tests * Bump asl-validator from 2.2.1 to 3.0.8 in /dependencies Bumps [asl-validator](https://github.com/ChristopheBougere/asl-validator) from 2.2.1 to 3.0.8. - [Release notes](https://github.com/ChristopheBougere/asl-validator/releases) - [Commits](https://github.com/ChristopheBougere/asl-validator/compare/2.2.1...3.0.8) --- updated-dependencies: - dependency-name: asl-validator dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Show errors when using curl to call the status API * Make the build more reproducible Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .editorconfig | 5 + Dockerfile | 30 ++--- dependencies/package-lock.json | 105 +++++++++--------- dependencies/package.json | 2 +- lib/functions/linterVersions.sh | 2 +- lib/linter.sh | 2 +- test/inspec/inspec.lock | 3 - .../super-linter/controls/super_linter.rb | 6 +- 8 files changed, 75 insertions(+), 80 deletions(-) delete mode 100644 test/inspec/inspec.lock diff --git a/.editorconfig b/.editorconfig index abf24629..06c94e90 100644 --- a/.editorconfig +++ b/.editorconfig @@ -28,3 +28,8 @@ spaces_around_brackets = false [{Makefile,go.mod,go.sum,*.go,.gitmodules}] indent_style = tab indent_size = 4 + +[**/test/*.{kt,kts}] +# Disable the filename rule because our test case logic requires that files used in test cases +# follow a certain syntax which is not compatible with what ktlint currently mandates (PascalCase) +disabled_rules = filename diff --git a/Dockerfile b/Dockerfile index aaa780c9..03d92a63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,11 +37,14 @@ FROM python:3.10.7-alpine as base_image ARG ARM_TTK_NAME='master.zip' ARG ARM_TTK_URI='https://github.com/Azure/arm-ttk/archive/master.zip' ARG ARM_TTK_DIRECTORY='/usr/lib/microsoft' +ARG CHECKSTYLE_VERSION='10.3.4' # Dart Linter ## stable dart sdk: https://dart.dev/get-dart#release-channels ARG DART_VERSION='2.8.4' +ARG GOOGLE_JAVA_FORMAT_VERSION='1.15.0' ## install alpine-pkg-glibc (glibc compatibility layer package for Alpine Linux) ARG GLIBC_VERSION='2.31-r0' +ARG KTLINT_VERSION='0.47.1' # PowerShell & PSScriptAnalyzer linter ARG PSSA_VERSION='latest' ARG PWSH_DIRECTORY='/usr/lib/microsoft/powershell' @@ -81,7 +84,8 @@ RUN apk add --no-cache \ openssh-client \ openssl-dev \ perl perl-dev \ - py3-setuptools python3-dev \ + py3-setuptools python3-dev \ + py3-pyflakes \ R R-dev R-doc \ readline-dev \ ruby ruby-dev ruby-bundler ruby-rdoc \ @@ -108,15 +112,7 @@ RUN npm config set package-lock true \ ############################## # Installs Perl dependencies # ############################## -RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community \ - ####################### - # Installs ActionLint # - ####################### - && curl --retry 5 --retry-delay 5 -sLO https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash \ - && chmod +x download-actionlint.bash \ - && ./download-actionlint.bash \ - && rm download-actionlint.bash \ - && mv actionlint /usr/bin/actionlint +RUN curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic Perl::Critic::Community ###################### # Install shellcheck # @@ -221,7 +217,7 @@ RUN wget --tries=5 -q -O kubeval-linux-amd64.tar.gz https://github.com/instrumen ################## # Install ktlint # ################## - && curl --retry 5 --retry-delay 5 -sSLO https://github.com/pinterest/ktlint/releases/latest/download/ktlint \ + && curl --retry 5 --retry-delay 5 -sSLO "https://github.com/pinterest/ktlint/releases/download/${KTLINT_VERSION}/ktlint" \ && chmod a+x ktlint \ && mv "ktlint" /usr/bin/ \ && terrascan init \ @@ -250,14 +246,12 @@ RUN apk add --no-cache rakudo zef \ ###################### # Install CheckStyle # ###################### - && curl --retry 5 --retry-delay 5 -sSL \ - "$(curl -s https://api.github.com/repos/checkstyle/checkstyle/releases/latest | jq -r '.assets[0].browser_download_url')" \ + && curl --retry 5 --retry-delay 5 --show-error -sSL "https://github.com/checkstyle/checkstyle/releases/download/checkstyle-${CHECKSTYLE_VERSION}/checkstyle-${CHECKSTYLE_VERSION}-all.jar" \ --output /usr/bin/checkstyle \ ############################## # Install google-java-format # ############################## - && curl --retry 5 --retry-delay 5 -sSL \ - "$(curl -s https://api.github.com/repos/google/google-java-format/releases/latest | jq -r '.assets | .[] | select(.browser_download_url | contains("all-deps.jar")) | .browser_download_url')" \ + && curl --retry 5 --retry-delay 5 --show-error -sSL "https://github.com/google/google-java-format/releases/download/v${GOOGLE_JAVA_FORMAT_VERSION}/google-java-format-${GOOGLE_JAVA_FORMAT_VERSION}-all-deps.jar" \ --output /usr/bin/google-java-format \ ################################# # Install luacheck and luarocks # @@ -393,7 +387,7 @@ ENV TFLINT_PLUGIN_DIR="/root/.tflint.d/plugins" #################################################### # Install Composer after all Libs have been copied # #################################################### -RUN sh -c 'curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer' +RUN sh -c 'curl --retry 5 --retry-delay 5 --show-error -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer' ######################################## # Add node packages to path and dotnet # @@ -507,7 +501,7 @@ RUN ln -s /usr/bin/rustup-init /usr/bin/rustup \ # Slightly modified to always retrieve latest stable Powershell version # If changing PWSH_VERSION='latest' to a specific version, use format PWSH_VERSION='tags/v7.0.2' RUN mkdir -p ${PWSH_DIRECTORY} \ - && curl --retry 5 --retry-delay 5 -s https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION} \ + && curl --retry 5 --retry-delay 5 --show-error -s https://api.github.com/repos/powershell/powershell/releases/${PWSH_VERSION} \ | grep browser_download_url \ | grep linux-alpine-x64 \ | cut -d '"' -f 4 \ @@ -523,7 +517,7 @@ RUN mkdir -p ${PWSH_DIRECTORY} \ # Reference https://github.com/Azure/arm-ttk # Reference https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/test-toolkit ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1" -RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \ +RUN curl --retry 5 --retry-delay 5 --show-error -sLO "${ARM_TTK_URI}" \ && unzip "${ARM_TTK_NAME}" -d "${ARM_TTK_DIRECTORY}" \ && rm "${ARM_TTK_NAME}" \ && ln -sTf "${ARM_TTK_PSD1}" /usr/bin/arm-ttk diff --git a/dependencies/package-lock.json b/dependencies/package-lock.json index cf1bbb2d..36b16295 100644 --- a/dependencies/package-lock.json +++ b/dependencies/package-lock.json @@ -12,7 +12,7 @@ "@stoplight/spectral": "^6.1.0", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", - "asl-validator": "^2.2.1", + "asl-validator": "^3.0.8", "axios": "^0.27.2", "babel-eslint": "^10.1.0", "eslint": "^7.32.0", @@ -2530,9 +2530,9 @@ } }, "node_modules/ajv": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", - "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", "dependencies": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -2735,39 +2735,36 @@ "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" }, - "node_modules/asl-validator": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-2.2.1.tgz", - "integrity": "sha512-GzG/vY94kbyGVm9Pw+oZPPZC74Aw4RVBOp0NXXCsS04KkiNS54LaZDlPUiWuKkHmmTWgWvJUlYnn1ztnE2cT7A==", + "node_modules/asl-path-validator": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.11.0.tgz", + "integrity": "sha512-2kfFkqNCXInc7d8hbUoXn/XpK5fFr3//0nh4jfcZWav0VR4zo2bYVlRCwOuNKJID9yM4vIo7dMb4n0fnWrc/Xw==", "dependencies": { - "ajv": "^6.12.6", + "jsonpath-plus": "^7.0.0" + } + }, + "node_modules/asl-path-validator/node_modules/jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/asl-validator": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.0.8.tgz", + "integrity": "sha512-ku2hkt137ebImA6DNySVoBtymffl/62TQHWKBb54yI3twrcsQyi78fPtvRi+PMob89vLeb0BbGezr5+4rQcJ7Q==", + "dependencies": { + "ajv": "^8.11.0", + "asl-path-validator": "^0.11.0", "commander": "^5.1.0", "jsonpath-plus": "^7.0.0" }, "bin": { - "asl-validator": "bin/asl-validator.js" + "asl-validator": "dist/bin/asl-validator.js" } }, - "node_modules/asl-validator/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/asl-validator/node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, "node_modules/asl-validator/node_modules/jsonpath-plus": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.0.0.tgz", @@ -13429,9 +13426,9 @@ } }, "ajv": { - "version": "8.9.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.9.0.tgz", - "integrity": "sha512-qOKJyNj/h+OWx7s5DePL6Zu1KeM9jPZhwBqs+7DzP6bGOvqzVCSf0xueYmVuaC/oQ/VtS2zLMLHdQFbkka+XDQ==", + "version": "8.11.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", + "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", "requires": { "fast-deep-equal": "^3.1.1", "json-schema-traverse": "^1.0.0", @@ -13570,32 +13567,32 @@ "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" }, - "asl-validator": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-2.2.1.tgz", - "integrity": "sha512-GzG/vY94kbyGVm9Pw+oZPPZC74Aw4RVBOp0NXXCsS04KkiNS54LaZDlPUiWuKkHmmTWgWvJUlYnn1ztnE2cT7A==", + "asl-path-validator": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/asl-path-validator/-/asl-path-validator-0.11.0.tgz", + "integrity": "sha512-2kfFkqNCXInc7d8hbUoXn/XpK5fFr3//0nh4jfcZWav0VR4zo2bYVlRCwOuNKJID9yM4vIo7dMb4n0fnWrc/Xw==", "requires": { - "ajv": "^6.12.6", + "jsonpath-plus": "^7.0.0" + }, + "dependencies": { + "jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==" + } + } + }, + "asl-validator": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/asl-validator/-/asl-validator-3.0.8.tgz", + "integrity": "sha512-ku2hkt137ebImA6DNySVoBtymffl/62TQHWKBb54yI3twrcsQyi78fPtvRi+PMob89vLeb0BbGezr5+4rQcJ7Q==", + "requires": { + "ajv": "^8.11.0", + "asl-path-validator": "^0.11.0", "commander": "^5.1.0", "jsonpath-plus": "^7.0.0" }, "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" - }, "jsonpath-plus": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.0.0.tgz", diff --git a/dependencies/package.json b/dependencies/package.json index 4d9942e2..6334dbc3 100644 --- a/dependencies/package.json +++ b/dependencies/package.json @@ -5,7 +5,7 @@ "@stoplight/spectral": "^6.1.0", "@typescript-eslint/eslint-plugin": "^4.33.0", "@typescript-eslint/parser": "^4.33.0", - "asl-validator": "^2.2.1", + "asl-validator": "^3.0.8", "axios": "^0.27.2", "babel-eslint": "^10.1.0", "eslint": "^7.32.0", diff --git a/lib/functions/linterVersions.sh b/lib/functions/linterVersions.sh index 8f125c03..ee231499 100755 --- a/lib/functions/linterVersions.sh +++ b/lib/functions/linterVersions.sh @@ -72,7 +72,7 @@ BuildLinterVersions() { if [[ ${LINTER} == "arm-ttk" ]]; then # Need specific command for ARM GET_VERSION_CMD="$(grep -iE 'version' "/usr/bin/arm-ttk" | xargs 2>&1)" - elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]]; then + elif [[ ${LINTER} == "bash-exec" ]] || [[ ${LINTER} == "gherkin-lint" ]] || [[ ${LINTER} == "asl-validator" ]]; then # Need specific command for Protolint and editorconfig-checker GET_VERSION_CMD="$(echo "--version not supported")" elif [[ ${LINTER} == "lintr" ]]; then diff --git a/lib/linter.sh b/lib/linter.sh index 5e2e1369..b459ac1d 100755 --- a/lib/linter.sh +++ b/lib/linter.sh @@ -653,7 +653,7 @@ CallStatusAPI() { # Call the status API to create status check # ############################################## SEND_STATUS_CMD=$( - curl -f -s -X POST \ + curl -f -s --show-error -X POST \ --url "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \ -H 'accept: application/vnd.github.v3+json' \ -H "authorization: Bearer ${GITHUB_TOKEN}" \ diff --git a/test/inspec/inspec.lock b/test/inspec/inspec.lock deleted file mode 100644 index e687b9b4..00000000 --- a/test/inspec/inspec.lock +++ /dev/null @@ -1,3 +0,0 @@ ---- -lockfile_version: 1 -depends: [] diff --git a/test/inspec/super-linter/controls/super_linter.rb b/test/inspec/super-linter/controls/super_linter.rb index c0b2d791..83bdccd2 100644 --- a/test/inspec/super-linter/controls/super_linter.rb +++ b/test/inspec/super-linter/controls/super_linter.rb @@ -13,6 +13,7 @@ control "super-linter-installed-packages" do packages = [ "bash", + "ca-certificates", "coreutils", "curl", "gcc", @@ -43,6 +44,7 @@ control "super-linter-installed-packages" do "openssl-dev", "perl-dev", "perl", + "py3-pyflakes", "py3-setuptools", "python3-dev", "rakudo", @@ -92,7 +94,7 @@ control "super-linter-installed-commands" do { linter_name: "actionlint"}, { linter_name: "ansible-lint"}, { linter_name: "arm-ttk", version_command: "grep -iE 'version' '/usr/bin/arm-ttk' | xargs"}, - { linter_name: "asl-validator"}, + { linter_name: "asl-validator", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command { linter_name: "bash-exec", expected_exit_status: 1}, # expect a return code = 1 because this linter doesn't support a "get linter version" command { linter_name: "black"}, { linter_name: "clang-format"}, @@ -392,7 +394,7 @@ control "super-linter-validate-files" do "/action/lib/.automation/.scalafmt.conf", "/action/lib/.automation/.snakefmt.toml", "/action/lib/.automation/.sql-config.json", - "/action/lib/.automation//.sqlfluff", + "/action/lib/.automation/.sqlfluff", "/action/lib/.automation/.stylelintrc.json", "/action/lib/.automation/.tflint.hcl", "/action/lib/.automation/.yaml-lint.yml",