Snapshot hidden ref

images/gh-gl-sync/SpackCIBridge.py

@@ -295,6 +295,21 @@ def list_github_tags(self):
             print("    {0}".format(tag))
         return tags
 
+    def list_github_hidden_refs(self, hidden_refs):
+        if not hidden_refs:
+            return []
+
+        ref_list = []
+        for ref in hidden_refs:
+            ref_list.extend(self.py_gh_repo.get_matching_refs(ref))
+            print("Rate limit after get_matching_refs({}): {}".format(ref, self.py_github.rate_limiting[0]))
+
+        refs = sorted(set([ref.ref for ref in ref_list]))
+        print("Refs:")
+        for ref in refs:
+            print("    {0}".format(ref))
+        return refs
+
     def setup_git_repo(self):
         """Initialize a bare git repository with two remotes:
         one for GitHub and one for GitLab.
@@ -352,6 +367,12 @@ def update_refspecs_for_tags(self, tags, open_refspecs, fetch_refspecs):
             open_refspecs.append("refs/tags/{0}:refs/tags/{0}".format(tag))
         return open_refspecs, fetch_refspecs
 
+    def update_refspecs_for_hidden_refs(self, hidden_refs, open_refspecs, fetch_refspecs):
+        for ref in hidden_refs:
+            fetch_refspecs.append("+{0}:{0}".format(ref))
+            open_refspecs.append("{0}:{0}".format(ref))
+        return open_refspecs, fetch_refspecs
+
     def fetch_github_branches(self, fetch_refspecs):
         """Perform `git fetch` for a given list of refspecs."""
         print("Fetching GitHub refs for open PRs")
@@ -590,7 +611,7 @@ def create_status_for_commit(self, sha, branch, state, target_url, description):
             print(e_inst)
         print("  {0} -> {1}".format(branch, sha))
 
-    def sync(self):
+    def sync(self, hidden_refs=None):
         """Synchronize pull requests from GitHub as branches on GitLab."""
 
         print("Initial rate limit: {}".format(self.py_github.rate_limiting[0]))
@@ -634,11 +655,15 @@ def sync(self):
             # Get tags on GitHub.
             tags = self.list_github_tags()
 
+            # Get hidden refs on GitHub
+            refs = self.list_github_hidden_refs(hidden_refs)
+
             # Get refspecs for open PRs and protected branches.
             open_refspecs = self.get_open_refspecs(open_prs)
             fetch_refspecs = []
             self.update_refspecs_for_protected_branches(protected_branches, open_refspecs, fetch_refspecs)
             self.update_refspecs_for_tags(tags, open_refspecs, fetch_refspecs)
+            self.update_refspecs_for_hidden_refs(refs, open_refspecs, fetch_refspecs)
 
             # Sync open GitHub PRs and protected branches to GitLab.
             self.fetch_github_branches(fetch_refspecs)
@@ -674,6 +699,8 @@ def sync(self):
 on a commit of the main branch that is newer than the latest commit tested by GitLab.""")
     parser.add_argument("--prereq-check", nargs="+", default=False,
                         help="Only push branches that have already passed this GitHub check")
+    parser.add_argument("-r", "--hidden-ref", action="append",
+                        help="List of hidden refs to sync.")
 
     args = parser.parse_args()
 
@@ -693,4 +720,4 @@ def sync(self):
                            main_branch=args.main_branch,
                            prereq_checks=args.prereq_check)
     bridge.setup_ssh(ssh_key_base64)
-    bridge.sync()
+    bridge.sync(hidden_refs=args.hidde_ref)

images/snapshot-release-tags/requirements.txt

@@ -1,14 +1,2 @@
-certifi==2023.5.7
-cffi==1.15.1
-charset-normalizer==3.1.0
-cryptography==40.0.2
-Deprecated==1.2.13
-idna==3.4
-pycparser==2.21
 PyGithub==1.58.2
-PyJWT==2.7.0
-PyNaCl==1.5.0
-requests==2.30.0
 sentry-sdk
-urllib3==2.0.2
-wrapt==1.15.0

images/snapshot-release-tags/snapshot_release_tags.py

@@ -4,10 +4,7 @@
 from github import Github, InputGitAuthor
 import json
 import os
-import re
 import sentry_sdk
-import subprocess
-import tempfile
 import urllib.request
 
 sentry_sdk.init(
@@ -38,25 +35,22 @@
     sha = pipelines[0]["sha"]
 
     date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
-    tag_name = f"develop-{date_str}"
-    tag_msg = f"Snapshot release {date_str}"
+    ref_name = f"develop-{date_str}"
 
     # Use the GitHub API to create a tag for this commit of develop.
     github_token = os.environ.get('GITHUB_TOKEN')
     py_github = Github(github_token)
     py_gh_repo = py_github.get_repo("spack/spack", lazy=True)
-    spackbot_author = InputGitAuthor("spackbot", "[email protected]")
-    print(f"Pushing tag {tag_name} for commit {sha}")
+    print(f"Pushing ref {ref_name} for commit {sha}")
 
-    tag = py_gh_repo.create_git_tag(
-        tag=tag_name,
-        message=tag_msg,
-        object=sha,
-        type="commit",
-        tagger=spackbot_author)
+    # Create a ref for this sha using the date stamp
+    py_gh_repo.create_git_ref(
+        ref=f"refs/snapshots/{ref_name}",
+        sha=sha)
 
+    # Create a ref for this sha using the `develop-latest` tag for the GH-GL sync script
     py_gh_repo.create_git_ref(
-        ref=f"refs/tags/{tag_name}",
-        sha=tag.sha)
+        ref="refs/snapshots/develop-latest",
+        sha=sha)
 
     print("Push done!")

k8s/production/custom/gh-gl-sync/cron-jobs.yaml

@@ -49,5 +49,7 @@ spec:
               - "develop"
               - "--prereq-check"
               - "prechecks / style"
+              - "--hidden-ref"
+              - "refs/snapshots/develop-latest"
           nodeSelector:
             spack.io/node-pool: base