[Draft] MPL review gap

.github/actions/polymorph_codegen/action.yml

@@ -82,6 +82,12 @@ runs:
       run: |
         make polymorph_dotnet
 
+    - name: Regenerate Python code using smithy-dafny
+      working-directory: ./${{ inputs.library }}
+      shell: bash
+      run: |
+        make polymorph_python
+
     - name: Check regenerated code against commited code
       # Composite action inputs seem to not actually support booleans properly for some reason
       if: inputs.diff-generated-code == 'true'

.github/workflows/daily_ci.yml

@@ -43,6 +43,13 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  daily-ci-python:
+    needs: getVersion
+    if: github.event_name != 'schedule' || github.repository_owner == 'aws'
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      # TODO: Once minimum Dafny version is 4.7.0, point this at `getVersion` output
+      dafny: 4.7.0
   daily-interop-test:
     needs: getVersion
     if: github.event_name != 'schedule' || github.repository_owner == 'aws'

.github/workflows/library_codegen.yml

@@ -25,6 +25,10 @@ jobs:
         # Note dotnet is only used for formatting generated code
         # in this workflow
         dotnet-version: ["6.0.x"]
+        # Note dotnet is only used for formatting generated code
+        # in this workflow.
+        # The workflow installs black formatter and runs it on generated code.
+        python-version: [3.11]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     defaults:
@@ -66,6 +70,15 @@ jobs:
       - name: Install Smithy-Dafny codegen dependencies
         uses: ./.github/actions/install_smithy_dafny_codegen_dependencies
 
+      - name: Setup Python and black for code formatting
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          python -m pip install --upgrade black
+
       - uses: ./.github/actions/polymorph_codegen
         with:
           dafny: ${{ env.DAFNY_VERSION }}

.github/workflows/library_interop_tests.yml

@@ -25,7 +25,7 @@ jobs:
             ubuntu-latest,
             macos-12,
           ]
-        language: [java, net]
+        language: [java, net, python]
         # https://taskei.amazon.dev/tasks/CrypTool-5284
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
@@ -57,6 +57,16 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Dafny
         uses: dafny-lang/[email protected]
         with:
@@ -89,6 +99,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python
+
       - name: Setup gradle
         if: matrix.language == 'java'
         uses: gradle/gradle-build-action@v2
@@ -128,8 +147,8 @@ jobs:
             ubuntu-latest,
             macos-12,
           ]
-        encrypting_language: [java, net]
-        decrypting_language: [java, net]
+        encrypting_language: [java, net, python]
+        decrypting_language: [java, net, python]
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
     permissions:
@@ -167,6 +186,16 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Dafny
         uses: dafny-lang/[email protected]
         with:
@@ -199,6 +228,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_net
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.decrypting_language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python
+
       - name: Download Encrypt Manifest Artifact
         uses: actions/download-artifact@v4
         with:

.github/workflows/library_python_tests.yml

@@ -0,0 +1,84 @@
+# This workflow performs tests in Python.
+name: Library Python tests
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+      regenerate-code:
+        description: "Regenerate code using smithy-dafny"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  testPython:
+    strategy:
+      fail-fast: false
+      matrix:
+        library:
+          [
+            StandardLibrary,
+            AwsCryptographyPrimitives,
+            ComAmazonawsKms,
+            ComAmazonawsDynamodb,
+            AwsCryptographicMaterialProviders,
+            TestVectorsAwsCryptographicMaterialProviders,
+          ]
+        python-version: ["3.11"]
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Support longpaths on Git checkout
+        run: |
+          git config --global core.longpaths true
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-MPL-Dafny-Role-us-west-2
+          role-session-name: PythonTests
+
+      - uses: actions/checkout@v4
+      # The specification submodule is private so we don't have access, but we don't need
+      # it to verify the Dafny code. Instead we manually pull the submodules we DO need.
+      - run: git submodule update --init libraries
+      - run: git submodule update --init smithy-dafny
+
+      - name: Setup Dafny
+        uses: dafny-lang/[email protected]
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Setup Python ${{ matrix.python-version }} for running tests
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Build ${{ matrix.library }} implementation
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
+      - name: Test ${{ matrix.library }}
+        working-directory: ./${{ matrix.library }}
+        shell: bash
+        run: |
+          make test_python

.github/workflows/manual.yml

@@ -37,6 +37,12 @@ jobs:
     with:
       dafny: ${{ inputs.dafny }}
       regenerate-code: ${{ inputs.regenerate-code }}
+  manual-ci-python:
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      # TODO: Once minimum Dafny version is 4.7.0, point this at `getVersion` output
+      dafny: 4.7.0
+      regenerate-code: ${{ inputs.regenerate-code }}
   manual-interop-test:
     uses: ./.github/workflows/library_interop_tests.yml
     with:

.github/workflows/nightly_dafny.yml

@@ -37,6 +37,12 @@ jobs:
     with:
       dafny: "nightly-latest"
       regenerate-code: true
+  dafny-nightly-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      # TODO: Once minimum Dafny version is 4.7.0, point this at `getVersion` output
+      dafny: 4.7.0
 
   cut-issue-on-failure:
     runs-on: ubuntu-latest

.github/workflows/pull.yml

@@ -34,6 +34,12 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  pr-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      # TODO: Once minimum Dafny version is 4.7.0, point this at `getVersion` output
+      dafny: 4.7.0
   pr-interop-test:
     needs: getVersion
     uses: ./.github/workflows/library_interop_tests.yml

.github/workflows/push.yml

@@ -36,6 +36,12 @@ jobs:
     uses: ./.github/workflows/library_net_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  push-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/library_python_tests.yml
+    with:
+      # TODO: Once minimum Dafny version is 4.7.0, point this at `getVersion` output
+      dafny: 4.7.0
   pr-interop-test:
     needs: getVersion
     uses: ./.github/workflows/library_interop_tests.yml

.gitignore

@@ -2,11 +2,16 @@
 .vscode/*
 *.sln?*
 **/.DS_Store
-build/*
 test/**/Output/*
 /package-lock.json
 /node_modules
 
+# Build Artifacts
+build
+build/*
+doc/build
+AwsCryptographicMaterialProviders/runtimes/python/doc/generated/*
+
 # Duvet output
 specification_compliance_report.html
 

AwsCryptographicMaterialProviders/codebuild/release-python/prod-release.yml

@@ -4,7 +4,7 @@ env:
   variables:
     BRANCH: "main"
   secrets-manager:
-    TWINE_USERNAME: PyPiAPIToken:username 
+    TWINE_USERNAME: PyPiAPIToken:username
     TWINE_PASSWORD: PyPiAPIToken:password
 
 phases:
@@ -17,20 +17,18 @@ phases:
   pre_build:
     commands:
       - git checkout $COMMIT_ID
-      # TODO (Ritvik) -- update version for mpl
-      # - FOUND_VERSION=$(sed -n 's/__version__ = "\(.*\)"/\1/p' src/aws_encryption_sdk/identifiers.py)
-      # - |
-      #   if expr ${FOUND_VERSION} != ${VERSION}; then
-      #     echo "identifiers.py version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping"
-      #     exit 1;
-      #   fi
+      - FOUND_VERSION=$(sed -n 's/version = "\(.*\)"/\1/p' AwsCryptographicMaterialProviders/runtimes/python/pyproject.toml)
+      - |
+        if expr ${FOUND_VERSION} != ${VERSION}; then
+          echo "pyproject.toml version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping"
+          exit 1;
+        fi
   build:
     commands:
-      - tox -e park
-      - tox -e release
+      - echo "Skipping prod release until we are ready to release the MPL"
 
 batch:
-  fast-fail: true 
+  fast-fail: true
   build-graph:
     - identifier: release_to_prod
     - identifier: validate_prod_release

AwsCryptographicMaterialProviders/codebuild/release-python/test-release.yml

@@ -4,7 +4,7 @@ env:
   variables:
     BRANCH: "main"
   secrets-manager:
-    TWINE_USERNAME: TestPyPiAPIToken:username 
+    TWINE_USERNAME: TestPyPiAPIToken:username
     TWINE_PASSWORD: TestPyPiAPIToken:password
 
 phases:
@@ -17,20 +17,17 @@ phases:
   pre_build:
     commands:
       - git checkout $COMMIT_ID
-      # TODO (Ritvik) -- update version for mpl
-      # - FOUND_VERSION=$(sed -n 's/__version__ = "\(.*\)"/\1/p' src/aws_encryption_sdk/identifiers.py)
-      # - |
-      #   if expr ${FOUND_VERSION} != ${VERSION}; then
-      #     echo "identifiers.py version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping"
-      #     exit 1;
-      #   fi
+      - FOUND_VERSION=$(sed -n 's/version = "\(.*\)"/\1/p' AwsCryptographicMaterialProviders/runtimes/python/pyproject.toml)
+      - |
+        if expr ${FOUND_VERSION} != ${VERSION}; then
+          echo "pyproject.toml version (${FOUND_VERSION}) does not match expected version (${VERSION}), stopping"
+          exit 1;
+        fi
   build:
     commands:
       - cd AwsCryptographicMaterialProviders/runtimes/python/
-      - tox -e park
       - tox -e test-release
 
-
 batch:
   fast-fail: true
   build-graph: