install ARO with custom disk encryption set

ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__installer-rehearse-4.14.yaml

@@ -61,6 +61,13 @@ tests:
     env:
       BASE_DOMAIN: qe.azure.devcluster.openshift.com
     workflow: cucushift-installer-rehearse-azure-ipi-private
+- as: installer-rehearse-aro-byok
+  cron: '@yearly'
+  steps:
+    cluster_profile: azure-qe
+    env:
+      ARO_CLUSTER_VERSION: 4.14.16
+    workflow: cucushift-installer-rehearse-azure-aro-byok
 - as: installer-rehearse-azure-stack
   cluster: build01
   cron: '@yearly'

ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-master-periodics.yaml

@@ -12320,6 +12320,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build01
+  cron: '@yearly'
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: verification-tests
+  labels:
+    ci-operator.openshift.io/cloud: azure4
+    ci-operator.openshift.io/cloud-cluster-profile: azure-qe
+    ci-operator.openshift.io/variant: installer-rehearse-4.14
+    ci.openshift.io/generator: prowgen
+    job-release: "4.14"
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-verification-tests-master-installer-rehearse-4.14-installer-rehearse-aro-byok
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/installer-rehearse-aro-byok-cluster-profile
+      - --target=installer-rehearse-aro-byok
+      - --variant=installer-rehearse-4.14
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/installer-rehearse-aro-byok-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-azure-qe
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build05
   cron: '@yearly'

ci-operator/step-registry/aro/provision/cluster/aro-provision-cluster-commands.sh

@@ -9,7 +9,7 @@ RESOURCEGROUP=${RESOURCEGROUP:=$(cat "${SHARED_DIR}/resourcegroup")}
 VNET=${VNET:=$(cat "$SHARED_DIR/vnet")}
 LOCATION=${LOCATION:=${LEASED_RESOURCE}}
 PULL_SECRET_FILE=${PULL_SECRET_FILE:="${CLUSTER_PROFILE_DIR}/pull-secret"}
-DISK_ENCRYPTION_SET_ENABLE=${DISK_ENCRYPTION_SET_ENABLE:=no}
+#DISK_ENCRYPTION_SET_ENABLE=${DISK_ENCRYPTION_SET_ENABLE:=no}
 AZURE_AUTH_LOCATION="${CLUSTER_PROFILE_DIR}/osServicePrincipal.json"
 AZURE_AUTH_CLIENT_ID="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientId)"
 AZURE_AUTH_CLIENT_SECRET="$(<"${AZURE_AUTH_LOCATION}" jq -r .clientSecret)"
@@ -53,8 +53,8 @@ fi
 
 # if azure_des file exists assume we want to use des for our aro cluster
 if [ -f "${SHARED_DIR}/azure_des" ]; then
-    des=$(cat ${SHARED_DIR}/azure_des)
-    des_id=$(az disk-encryption-set show -n ${des} -g ${RESOURCEGROUP} --query "[id]" -o tsv)
+    des_id=$(cat ${SHARED_DIR}/azure_des_id)
+    #des_id=$(az disk-encryption-set show -n ${des} -g ${RESOURCEGROUP} --query "[id]" -o tsv)
     CREATE_CMD="${CREATE_CMD} --disk-encryption-set ${des_id} --master-encryption-at-host --worker-encryption-at-host "
 fi
 

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byok/OWNERS

@@ -0,0 +1 @@
+../OWNERS

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byok/cucushift-installer-rehearse-azure-aro-byok-workflow.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "cucushift/installer/rehearse/azure/aro/byok/cucushift-installer-rehearse-azure-aro-byok-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"yunjiang29",
+			"jinyunma",
+			"mgahagan73"
+		],
+		"reviewers": [
+			"jianlinliu",
+			"yunjiang29",
+			"jinyunma",
+			"mgahagan73"
+		]
+	}
+}

ci-operator/step-registry/cucushift/installer/rehearse/azure/aro/byok/cucushift-installer-rehearse-azure-aro-byok-workflow.yaml

@@ -0,0 +1,15 @@
+workflow:
+  as: cucushift-installer-rehearse-azure-aro-byok
+  steps: 
+    pre:
+    - ref: azure-provision-resourcegroup
+    - ref: azure-provision-disk-encryption-set
+    - ref: aro-provision-vnet
+    - ref: aro-provision-cluster
+    - ref: ipi-install-rbac
+    - ref: openshift-cluster-bot-rbac
+    - ref: enable-qe-catalogsource
+    post:
+    - ref: aro-deprovision
+  documentation: |-
+    This is the workflow to trigger Prow's rehearsal test when submitting installer steps/chain/workflow for aro with a custom disk encryption set. Control plane and workers will use this custom des for data encryption.