Merge pull request #1276 from GoogleCloudPlatform/release-candidate

Release v1.17.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,10 @@ assignees: ''
 
 ---
 
+**If you find a similar existing issue, please comment on that issue instead of creating a new one.**
+
+**If you are submitting a feature request, please start a [discussion](https://github.com/GoogleCloudPlatform/hpc-toolkit/discussions/new?category=ideas-and-feature-requests) instead of creating an issue.**
+
 ### Describe the bug
 
 A clear and concise description of what the bug is.

.github/pull_request_template.md

@@ -1,9 +1,11 @@
 ### Submission Checklist
 
-* [ ] Have you installed and run this change against pre-commit? (`pre-commit
-  install`)
-* [ ] Are all tests passing? (`make tests`)
-* [ ] Have you written unit tests to cover this change?
-* [ ] Is unit test coverage still above 80%?
-* [ ] Have you updated all applicable documentation?
-* [ ] Have you followed the guidelines in our Contributing document?
+Please take the following actions before submitting this pull request.
+
+* Fork your PR branch from the Toolkit "develop" branch (not main)
+* Test all changes with pre-commit in a local branch [#](https://goo.gle/hpc-toolkit#development)
+* Confirm that "make tests" passes all tests
+* Add or modify unit tests to cover code changes
+* Ensure that unit test coverage remains above 80%
+* Update all applicable documentation
+* Follow Cloud HPC Toolkit Contribution guidelines [#](https://goo.gle/hpc-toolkit-contributing)

.gitignore

@@ -57,3 +57,7 @@ terraform.rc
 #### PACKER
 packer-manifest.json
 *.auto.pkrvars.hcl
+
+#### Exclude from gitingore
+!tools/validate_configs/golden_copies/*/*/*/defaults.auto.pkrvars.hcl
+!tools/validate_configs/golden_copies/*/*/terraform.tfvars

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
     entry: tools/autodoc/terraform_docs.sh
     language: script
     types: [terraform]
-    exclude: \.terraform|pkg\/.*$
+    exclude: (\.terraform|pkg\/.*$)|(tools/validate_configs/golden_copies/.*)
     pass_filenames: true
     require_serial: true
   - id: packer-readme
@@ -103,3 +103,4 @@ repos:
   rev: v4.4.0
   hooks:
   - id: end-of-file-fixer
+exclude: tools/validate_configs/golden_copies/.*

.tflint.hcl

@@ -13,7 +13,7 @@
 // limitations under the License.
 plugin "google" {
   enabled = true
-  version = "0.20.0"
+  version = "0.23.0"
   source  = "github.com/terraform-linters/tflint-ruleset-google"
 }
 rule "terraform_deprecated_index" {

Makefile

@@ -6,7 +6,7 @@ MIN_GOLANG_VERSION=1.18 # for building ghpc
 .PHONY: install install-user tests format add-google-license install-dev-deps \
         warn-go-missing warn-terraform-missing warn-packer-missing \
         warn-go-version warn-terraform-version warn-packer-version \
-        test-engine validate_configs packer-check \
+        test-engine validate_configs validate_golden_copy packer-check \
         terraform-format packer-format \
         check-tflint check-pre-commit
 
@@ -47,7 +47,7 @@ install: install-user
 
 endif
 
-tests: warn-terraform-version warn-packer-version test-engine validate_configs packer-check
+tests: warn-terraform-version warn-packer-version test-engine validate_golden_copy validate_configs packer-check
 
 format: warn-go-version warn-terraform-version warn-packer-version terraform-format packer-format
 	$(info **************** formatting go code *******************)
@@ -157,6 +157,10 @@ validate_configs: ghpc
 	$(info *********** running basic integration tests ***********)
 	tools/validate_configs/validate_configs.sh
 
+validate_golden_copy: ghpc
+	$(info *********** running "Golden copy" tests ***********)
+	tools/validate_configs/validate_golden_copy.sh
+
 terraform-format:
 	$(info *********** cleaning terraform files syntax and generating terraform documentation ***********)
 	@for folder in ${TERRAFORM_FOLDERS}; do \

cmd/create.go

@@ -101,7 +101,7 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 	if err := deploymentConfig.ExpandConfig(); err != nil {
 		log.Fatal(err)
 	}
-	if err := modulewriter.WriteDeployment(&deploymentConfig.Config, outputDir, overwriteDeployment); err != nil {
+	if err := modulewriter.WriteDeployment(deploymentConfig, outputDir, overwriteDeployment); err != nil {
 		var target *modulewriter.OverwriteDeniedError
 		if errors.As(err, &target) {
 			fmt.Printf("\n%s\n", err.Error())

cmd/root.go

@@ -50,7 +50,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.16.0",
+		Version:     "v1.17.0",
 		Annotations: annotation,
 	}
 )

community/examples/gke.yaml

@@ -0,0 +1,45 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+blueprint_name: small-gke
+
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: cluster-01
+  region: us-central1
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: network1
+    source: modules/network/vpc
+    settings:
+      subnetwork_name: gke-subnet
+      secondary_ranges:
+        gke-subnet:
+        - range_name: pods
+          ip_cidr_range: 10.4.0.0/14
+        - range_name: services
+          ip_cidr_range: 10.0.32.0/20
+
+  - id: gke_cluster
+    source: community/modules/scheduler/gke-cluster
+    use: [network1]
+    outputs: [instructions]
+
+  - id: compute_pool
+    source: community/modules/compute/gke-node-pool
+    use: [gke_cluster]

community/examples/quantum-circuit-simulator.yaml

@@ -67,7 +67,7 @@ deployment_groups:
           pip install pybind11 cirq
           git clone https://github.com/quantumlib/qsim.git /opt/qsim
           cd /opt/qsim
-          export CUQUANTUM_DIR=/opt/conda/envs/qsim
+          export CUQUANTUM_ROOT=/opt/conda/envs/qsim
           make
           pip install .
       - type: data

community/examples/slurm-chromedesktop.yaml

@@ -0,0 +1,127 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+blueprint_name: slurm-crd
+
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: slurm-chromedesktop
+  region: us-central1
+  zone: us-central1-c
+
+# Documentation for each of the modules used below can be found at
+# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: network1
+    source: modules/network/vpc
+
+  - id: homefs
+    source: modules/file-system/filestore
+    use: [network1]
+    settings:
+      local_mount: /home
+
+  - id: remote-desktop
+    source: community/modules/remote-desktop/chrome-remote-desktop
+    use: [network1]
+    settings:
+      install_nvidia_driver: true
+      # instance_count: 0 will create installation scripts only
+      # which can be used with slurm node provisioning
+      instance_count: 0
+
+  - id: remote_desktop_startup
+    source: modules/scripts/startup-script
+    settings:
+      runners:
+      - type: ansible-local
+        source: modules/chrome-remote-desktop/scripts/configure-grid-drivers.yml
+        destination: /tmp/configure-grid-drivers.yml
+      - type: ansible-local
+        source: modules/chrome-remote-desktop/scripts/configure-chrome-desktop.yml
+        destination: /tmp/configure-chrome-desktop.yml
+      - type: ansible-local
+        source: modules/chrome-remote-desktop/scripts/disable-sleep.yml
+        destination: /tmp/disable-sleep.yml
+
+  - id: crd_node_group
+    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
+    settings:
+      machine_type: n1-standard-8
+      node_count_dynamic_max: 1
+      disable_public_ips: false
+      instance_image:
+        family: schedmd-v5-slurm-22-05-8-ubuntu-2004-lts
+        project: projects/schedmd-slurm-public/global/images/family
+      guest_accelerator:
+      - type: nvidia-tesla-t4-vws
+        count: 1
+
+  - id: crd_partition
+    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
+    use:
+    - network1
+    - homefs
+    - remote_desktop_startup
+    - crd_node_group
+    settings:
+      partition_name: desktop
+      enable_placement: false
+      partition_startup_scripts_timeout: 900
+
+  - id: compute_node_group
+    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
+    settings:
+      machine_type: n2-standard-4
+      node_count_dynamic_max: 1
+
+  - id: compute_partition
+    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
+    use:
+    - network1
+    - homefs
+    - compute_node_group
+    settings:
+      partition_name: compute
+
+  - id: slurm_controller
+    source: community/modules/scheduler/schedmd-slurm-gcp-v5-controller
+    use:
+    - network1
+    - homefs
+    - crd_partition
+    - compute_partition
+    settings:
+      disable_controller_public_ips: false
+      compute_startup_scripts_timeout: 900
+      cloud_parameters:
+        resume_rate: 0
+        resume_timeout: 900
+        suspend_rate: 0
+        suspend_timeout: 300
+        no_comma_params: false
+
+  - id: slurm_login
+    source: community/modules/scheduler/schedmd-slurm-gcp-v5-login
+    use:
+    - network1
+    - slurm_controller
+    settings:
+      machine_type: n2-standard-4
+      disable_login_public_ips: false

community/examples/slurm-gcp-v5-high-io.yaml

@@ -19,8 +19,8 @@ blueprint_name: hpc-cluster-high-io-v5
 vars:
   project_id:  ## Set GCP Project ID Here ##
   deployment_name: high-io-slurm-gcp-v5
-  region: us-central1
-  zone: us-central1-c
+  region: us-west4
+  zone: us-west4-c
   # By default, public IPs are set in the login and controller to allow easier
   # SSH access. To turn this behavior off, set this to true.
   disable_public_ips: false

community/examples/slurm-gcp-v5-hpc-centos7.yaml

@@ -19,8 +19,8 @@ blueprint_name: slurm-gcp-v5-hpc-centos7
 vars:
   project_id:  ## Set GCP Project ID Here ##
   deployment_name: slurm-gcp-v5
-  region: us-central1
-  zone: us-central1-c
+  region: us-west4
+  zone: us-west4-c
 
 # Documentation for each of the modules used below can be found at
 # https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md

community/examples/slurm-gcp-v5-ubuntu2004.yaml

@@ -19,8 +19,8 @@ blueprint_name: slurm-gcp-v5-ubuntu2004
 vars:
   project_id:  ## Set GCP Project ID Here ##
   deployment_name: slurm-gcp-v5
-  region: us-central1
-  zone: us-central1-c
+  region: us-west4
+  zone: us-west4-c
   instance_image:
     # Please refer to the following link for the latest images:
     # https://github.com/SchedMD/slurm-gcp/blob/master/docs/images.md#supported-operating-systems