Expose PEP 740 attestations functionality

PR #236

This patch adds PEP 740 attestation generation to the workflow: when the Trusted Publishing flow is used, this will generate a publish attestation for each distribution being uploaded. These generated attestations are then fed into `twine`, which newly supports them via `--attestations`.

Ref: https://github.com/pypi/warehouse/issues/15871

Dockerfile

@@ -28,6 +28,7 @@ COPY LICENSE.md .
 COPY twine-upload.sh .
 COPY print-hash.py .
 COPY oidc-exchange.py .
+COPY attestations.py .
 
 RUN chmod +x twine-upload.sh
 ENTRYPOINT ["/app/twine-upload.sh"]

README.md

@@ -99,6 +99,31 @@ filter to the job:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
 ```
 
+### Generating and uploading attestations
+
+> [!IMPORTANT]
+> Support for generating and uploading [digital attestations] is currently
+> experimental and limited only to Trusted Publishing flows using PyPI or TestPyPI.
+> Support for this feature is not yet stable; the settings and behavior described
+> below may change without prior notice.
+
+> [!NOTE]
+> Generating and uploading digital attestations currently requires
+> authentication with a [trusted publisher].
+
+You can generate signed [digital attestations] for all the distribution files and
+upload them all together by enabling the `attestations` setting:
+
+```yml
+   with:
+     attestations: true
+```
+
+This will use [Sigstore] to create attestation
+objects for each distribution package, signing them with the identity provided
+by the GitHub's OIDC token associated with the current workflow. This means
+both the trusted publishing authentication and the attestations are tied to the
+same identity.
 
 ## Non-goals
 
@@ -287,3 +312,7 @@ https://github.com/vshymanskyy/StandWithUkraine/blob/main/docs/README.md
 [configured on PyPI]: https://docs.pypi.org/trusted-publishers/adding-a-publisher/
 
 [how to specify username and password]: #specifying-a-different-username
+
+[digital attestations]: https://peps.python.org/pep-0740/
+[Sigstore]: https://www.sigstore.dev/
+[trusted publisher]: #trusted-publishing

action.yml

@@ -80,6 +80,13 @@ inputs:
       Use `print-hash` instead.
     required: false
     default: 'false'
+  attestations:
+    description: >-
+      [EXPERIMENTAL]
+      Enable experimental support for PEP 740 attestations.
+      Only works with PyPI and TestPyPI via Trusted Publishing.
+    required: false
+    default: 'false'
 branding:
   color: yellow
   icon: upload-cloud
@@ -95,3 +102,4 @@ runs:
   - ${{ inputs.skip-existing }}
   - ${{ inputs.verbose }}
   - ${{ inputs.print-hash }}
+  - ${{ inputs.attestations }}

attestations.py

@@ -0,0 +1,114 @@
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import NoReturn
+
+from pypi_attestations import Attestation, Distribution
+from sigstore.oidc import IdentityError, IdentityToken, detect_credential
+from sigstore.sign import Signer, SigningContext
+
+# Be very verbose.
+sigstore_logger = logging.getLogger('sigstore')
+sigstore_logger.setLevel(logging.DEBUG)
+sigstore_logger.addHandler(logging.StreamHandler())
+
+_GITHUB_STEP_SUMMARY = Path(os.getenv('GITHUB_STEP_SUMMARY'))
+
+# The top-level error message that gets rendered.
+# This message wraps one of the other templates/messages defined below.
+_ERROR_SUMMARY_MESSAGE = """
+Attestation generation failure:
+
+{message}
+
+You're seeing this because the action attempted to generated PEP 740
+attestations for its inputs, but failed to do so.
+"""
+
+# Rendered if OIDC identity token retrieval fails for any reason.
+_TOKEN_RETRIEVAL_FAILED_MESSAGE = """
+OpenID Connect token retrieval failed: {identity_error}
+
+This failure occurred after a successful Trusted Publishing Flow,
+suggesting a transient error.
+"""  # noqa: S105; not a password
+
+
+def die(msg: str) -> NoReturn:
+    with _GITHUB_STEP_SUMMARY.open('a', encoding='utf-8') as io:
+        print(_ERROR_SUMMARY_MESSAGE.format(message=msg), file=io)
+
+    # HACK: GitHub Actions' annotations don't work across multiple lines naively;
+    # translating `\n` into `%0A` (i.e., HTML percent-encoding) is known to work.
+    # See: https://github.com/actions/toolkit/issues/193
+    msg = msg.replace('\n', '%0A')
+    print(f'::error::Attestation generation failure: {msg}', file=sys.stderr)
+    sys.exit(1)
+
+
+def debug(msg: str):
+    print(f'::debug::{msg}', file=sys.stderr)
+
+
+def collect_dists(packages_dir: Path) -> list[Path]:
+    # Collect all sdists and wheels.
+    dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')]
+    dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl'))
+
+    # Make sure everything that looks like a dist actually is one.
+    # We do this up-front to prevent partial signing.
+    if (invalid_dists := [path for path in dist_paths if path.is_file()]):
+        invalid_dist_list = ', '.join(map(str, invalid_dists))
+        die(
+            'The following paths look like distributions but '
+            f'are not actually files: {invalid_dist_list}',
+        )
+
+    return dist_paths
+
+
+def attest_dist(dist_path: Path, signer: Signer) -> None:
+    # We are the publishing step, so there should be no pre-existing publish
+    # attestation. The presence of one indicates user confusion.
+    attestation_path = Path(f'{dist_path}.publish.attestation')
+    if attestation_path.exists():
+        die(f'{dist_path} already has a publish attestation: {attestation_path}')
+
+    dist = Distribution.from_file(dist_path)
+    attestation = Attestation.sign(signer, dist)
+
+    attestation_path.write_text(attestation.model_dump_json(), encoding='utf-8')
+    debug(f'saved publish attestation: {dist_path=} {attestation_path=}')
+
+
+def get_identity_token() -> IdentityToken:
+    # Will raise `sigstore.oidc.IdentityError` if it fails to get the token
+    # from the environment or if the token is malformed.
+    # NOTE: audience is always sigstore.
+    oidc_token = detect_credential()
+    return IdentityToken(oidc_token)
+
+
+def main() -> None:
+    packages_dir = Path(sys.argv[1])
+
+    try:
+        identity = get_identity_token()
+    except IdentityError as identity_error:
+        # NOTE: We only perform attestations in trusted publishing flows, so we
+        # don't need to re-check for the "PR from fork" error mode, only
+        # generic token retrieval errors. We also render a simpler error,
+        # since permissions can't be to blame at this stage.
+        die(_TOKEN_RETRIEVAL_FAILED_MESSAGE.format(identity_error=identity_error))
+
+    dist_paths = collect_dists(packages_dir)
+
+    with SigningContext.production().signer(identity, cache=True) as s:
+        debug(f'attesting to dists: {dist_paths}')
+        for dist_path in dist_paths:
+            attest_dist(dist_path, s)
+
+
+if __name__ == '__main__':
+    main()

requirements/runtime.in

@@ -1,9 +1,14 @@
 twine
 
-# NOTE: Used to detect an ambient OIDC credential for OIDC publishing.
+# NOTE: Used to detect an ambient OIDC credential for OIDC publishing,
+# NOTE: as well as PEP 740 attestations.
 id ~= 1.0
 
 # NOTE: This is pulled in transitively through `twine`, but we also declare
 # NOTE: it explicitly here because `oidc-exchange.py` uses it.
 # Ref: https://github.com/di/id
 requests
+
+# NOTE: Used to generate attestations.
+pypi-attestations ~= 0.0.11
+sigstore ~= 3.2.0

requirements/runtime.txt

@@ -6,16 +6,41 @@
 #
 annotated-types==0.6.0
     # via pydantic
+betterproto==2.0.0b6
+    # via sigstore-protobuf-specs
 certifi==2024.2.2
     # via requests
+cffi==1.16.0
+    # via cryptography
 charset-normalizer==3.3.2
     # via requests
+cryptography==42.0.7
+    # via
+    #   pyopenssl
+    #   pypi-attestations
+    #   sigstore
+dnspython==2.6.1
+    # via email-validator
 docutils==0.21.2
     # via readme-renderer
+email-validator==2.1.1
+    # via pydantic
+grpclib==0.4.7
+    # via betterproto
+h2==4.1.0
+    # via grpclib
+hpack==4.0.0
+    # via h2
+hyperframe==6.0.1
+    # via h2
 id==1.4.0
-    # via -r runtime.in
+    # via
+    #   -r runtime.in
+    #   sigstore
 idna==3.7
-    # via requests
+    # via
+    #   email-validator
+    #   requests
 importlib-metadata==7.1.0
     # via twine
 jaraco-classes==3.4.0
@@ -34,33 +59,77 @@ more-itertools==10.2.0
     # via
     #   jaraco-classes
     #   jaraco-functools
+multidict==6.0.5
+    # via grpclib
 nh3==0.2.17
     # via readme-renderer
+packaging==24.1
+    # via pypi-attestations
 pkginfo==1.10.0
     # via twine
+platformdirs==4.2.2
+    # via sigstore
+pyasn1==0.6.0
+    # via sigstore
+pycparser==2.22
+    # via cffi
 pydantic==2.7.1
-    # via id
+    # via
+    #   id
+    #   pypi-attestations
+    #   sigstore
+    #   sigstore-rekor-types
 pydantic-core==2.18.2
     # via pydantic
 pygments==2.18.0
     # via
     #   readme-renderer
     #   rich
+pyjwt==2.8.0
+    # via sigstore
+pyopenssl==24.1.0
+    # via sigstore
+pypi-attestations==0.0.11
+    # via -r runtime.in
+python-dateutil==2.9.0.post0
+    # via betterproto
 readme-renderer==43.0
     # via twine
-requests==2.32.0
+requests==2.32.3
     # via
     #   -r runtime.in
     #   id
     #   requests-toolbelt
+    #   sigstore
+    #   tuf
     #   twine
 requests-toolbelt==1.0.0
     # via twine
 rfc3986==2.0.0
     # via twine
+rfc8785==0.1.2
+    # via sigstore
 rich==13.7.1
-    # via twine
+    # via
-twine==5.1.0
+    #   sigstore
+    #   twine
+securesystemslib==1.0.0
+    # via tuf
+sigstore==3.2.0
+    # via
+    #   -r runtime.in
+    #   pypi-attestations
+sigstore-protobuf-specs==0.3.2
+    # via
+    #   pypi-attestations
+    #   sigstore
+sigstore-rekor-types==0.0.13
+    # via sigstore
+six==1.16.0
+    # via python-dateutil
+tuf==5.0.0
+    # via sigstore
+twine==5.1.1
     # via -r runtime.in
 typing-extensions==4.11.0
     # via

twine-upload.sh

@@ -39,6 +39,7 @@ INPUT_PACKAGES_DIR="$(get-normalized-input 'packages-dir')"
 INPUT_VERIFY_METADATA="$(get-normalized-input 'verify-metadata')"
 INPUT_SKIP_EXISTING="$(get-normalized-input 'skip-existing')"
 INPUT_PRINT_HASH="$(get-normalized-input 'print-hash')"
+INPUT_ATTESTATIONS="$(get-normalized-input 'attestations')"
 
 PASSWORD_DEPRECATION_NUDGE="::error title=Password-based uploads disabled::\
 As of 2024, PyPI requires all users to enable Two-Factor \
@@ -53,7 +54,37 @@ environments like GitHub Actions without needing to use username/password \
 combinations or API tokens to authenticate with PyPI. Read more: \
 https://docs.pypi.org/trusted-publishers"
 
-if [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] ; then
+ATTESTATIONS_WITHOUT_TP_WARNING="::warning title=attestations input ignored::\
+The workflow was run with the 'attestations: true' input, but an explicit \
+password was also set, disabling Trusted Publishing. As a result, the \
+attestations input is ignored."
+
+ATTESTATIONS_WRONG_INDEX_WARNING="::warning title=attestations input ignored::\
+The workflow was run with 'attestations: true' input, but the specified \
+repository URL does not support PEP 740 attestations. As a result, the \
+attestations input is ignored."
+
+[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \
+    && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false
+
+if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then
+    # Setting `attestations: true` without Trusted Publishing indicates
+    # user confusion, since attestations (currently) require it.
+    if ! "${TRUSTED_PUBLISHING}" ; then
+        echo "${ATTESTATIONS_WITHOUT_TP_WARNING}"
+        INPUT_ATTESTATIONS="false"
+    fi
+
+    # Setting `attestations: true` with an index other than PyPI or TestPyPI
+    # indicates user confusion, since attestations are not supported on other
+    # indices presently.
+    if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org ]] ; then
+        echo "${ATTESTATIONS_WRONG_INDEX_WARNING}"
+        INPUT_ATTESTATIONS="false"
+    fi
+fi
+
+if "${TRUSTED_PUBLISHING}" ; then
     # No password supplied by the user implies that we're in the OIDC flow;
     # retrieve the OIDC credential and exchange it for a PyPI API token.
     echo "::debug::Authenticating to ${INPUT_REPOSITORY_URL} via Trusted Publishing"
@@ -130,6 +161,15 @@ if [[ ${INPUT_VERBOSE,,} != "false" ]] ; then
     TWINE_EXTRA_ARGS="--verbose $TWINE_EXTRA_ARGS"
 fi
 
+if [[ ${INPUT_ATTESTATIONS,,} != "false" ]] ; then
+    # NOTE: Intentionally placed after `twine check`, to prevent attestation
+    # NOTE: generation on distributions with invalid metadata.
+    echo "::notice::Generating and uploading digital attestations"
+    python /app/attestations.py "${INPUT_PACKAGES_DIR%%/}"
+
+    TWINE_EXTRA_ARGS="--attestations $TWINE_EXTRA_ARGS"
+fi
+
 if [[ ${INPUT_PRINT_HASH,,} != "false" || ${INPUT_VERBOSE,,} != "false" ]] ; then
     python /app/print-hash.py ${INPUT_PACKAGES_DIR%%/}
 fi