From 75ebacc4187d9d67e08a150997c4ff892f466763 Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Mon, 2 Oct 2023 14:40:16 -0400 Subject: [PATCH 1/8] build: replace `kit` with `task` - Task is a very popular, well-maintained, Go-based task runner with a nearly identical spec to `kit`, as can be seen in the diff - basically can do virtually all of the same things and a good number of other things, with lots of docs and active maintenances - intentionally made this commit have as few changes as possible to reduce the diff and make it look very familiar - used one-line arrays instead of multi-line to be similar to `kit`'s `make`-like task dep list - used singular `cmd` syntax instead of the usual plural `cmds` syntax Signed-off-by: Anton Gilgur --- tasks.yaml | 193 ++++++++++++++++++++++++++--------------------------- 1 file changed, 95 insertions(+), 98 deletions(-) diff --git a/tasks.yaml b/tasks.yaml index 5df2a78c74a8..42471d6a8f2d 100644 --- a/tasks.yaml +++ b/tasks.yaml @@ -1,99 +1,96 @@ -apiVersion: kit/v1 -kind: Tasks -metadata: - annotations: - help: | - Install `kit` by following https://github.com/kitproj/kit#install. - - Run `kit up` to start argo. +# yaml-language-server: $schema: https://taskfile.dev/schema.json - - `env PROFILE=mysql kit up` to start with MySQL. - - `env PROFILE=plugins ARGO_EXECUTOR_PLUGINS=true kit up` to start with plugins. - - `env PROFILE=sso ARGO_AUTH_MODE=sso kit up` to start with SSO. - - The app will be up-and-running between 15s and 1m later (if hot compiled or cold). - Any changes made to the source code will be automatically recompiled and the app restarted, typically within a few seconds. - name: argo-workflows -spec: - tasks: - - name: go-deps - command: go mod download - mutex: downloads - - name: install - command: sh -c "make install PROFILE=$PROFILE" - env: - - PROFILE=minimal - dependencies: go-deps - watch: manifests - mutex: docker - - name: build-controller - command: make ./dist/workflow-controller - watch: cmd/workflow-controller config errors persist pkg util workflow - dependencies: go-deps - mutex: build - - name: port-forward - command: ./hack/port-forward.sh - ports: 9000 - dependencies: install - - name: controller - command: ./dist/workflow-controller - dependencies: install build-controller port-forward - env: - - ARGO_EXECUTOR_PLUGINS=false - - ARGO_NAMESPACE=argo - - ARGO_NAMESPACED=true - - ARGO_MANAGED_NAMESPACE=argo - - ARGO_LOG_LEVEL=info - - ARGO_REMOVE_PVC_PROTECTION_FINALIZER=true - - ARGO_PROGRESS_PATCH_TICK_DURATION=7s - - DEFAULT_REQUEUE_TIME=1s - - LEADER_ELECTION_IDENTITY=local - - ALWAYS_OFFLOAD_NODE_STATUS=false - - OFFLOAD_NODE_STATUS_TTL=30s - - WORKFLOW_GC_PERIOD=30s - - UPPERIO_DB_DEBUG=1 - - ARCHIVED_WORKFLOW_GC_PERIOD=30s - ports: "9090" - - name: build-argo - command: make ./dist/argo - dependencies: go-deps - env: - - STATIC_FILES=false - watch: cmd/argo config errors persist pkg util server workflow - mutex: build - - name: server - command: ./dist/argo server - dependencies: build-argo port-forward - env: - - ARGO_X_FRAME_OPTIONS=SAMEORIGIN - - ARGO_SECURE=false - - ARGO_NAMESPACE=argo - - ARGO_NAMESPACED=true - - ARGO_LOG_LEVEL=info - - ARGO_AUTH_MODE=hybrid - - ARGO_MANAGED_NAMESPACE=argo - - UPPERIO_DB_DEBUG=1 - ports: "2746" - - name: ui-deps - command: yarn install - workingDir: ui - mutex: downloads - - name: ui - command: yarn start - workingDir: ui - dependencies: ui-deps server - ports: "8080" - - name: executor - command: make argoexec-image - watch: cmd/argoexec config errors pkg util workflow - mutex: docker - - name: example - command: kubectl create -f examples/hello-world.yaml - dependencies: install - mutex: docker - - name: build - dependencies: build-controller build-argo - - name: pre-up - dependencies: build install executor example - - name: up - dependencies: pre-up controller server ui +# Install `task` by following https://taskfile.dev/installation/ +# Run `task up` to start argo. + +# - `env PROFILE=mysql task up` to start with MySQL. +# - `env PROFILE=plugins ARGO_EXECUTOR_PLUGINS=true task up` to start with plugins. +# - `env PROFILE=sso ARGO_AUTH_MODE=sso task up` to start with SSO. + +# The app will be up-and-running between 15s and 1m later (if hot compiled or cold). +# When using `--watch`, any changes made to the source code will be automatically recompiled and the app restarted, typically within a few seconds. + +version: 3 + +tasks: + go-deps: + cmd: go mod download + # mutex: download + install: + cmd: sh -c "make install PROFILE=$PROFILE" + env: + PROFILE: minimal + deps: [go-deps] + sources: [manifests] + # mutex: docker + build-controller: + cmd: make ./dist/workflow-controller + sources: [cmd/workflow-controller, config, errors, persist, pkg, util, workflow] + deps: [go-deps] + # mutex: build + port-forward: + cmd: ./hack/port-forward.sh + # ports: 9000 + deps: [install] + controller: + cmd: ./dist/workflow-controller + deps: [install, build-controller, port-forward] + env: + ARGO_EXECUTOR_PLUGINS: false + ARGO_NAMESPACE: argo + ARGO_NAMESPACED: true + ARGO_MANAGED_NAMESPACE: argo + ARGO_LOG_LEVEL: info + ARGO_REMOVE_PVC_PROTECTION_FINALIZER: true + ARGO_PROGRESS_PATCH_TICK_DURATION: 7s + DEFAULT_REQUEUE_TIME: 1s + LEADER_ELECTION_IDENTITY: local + ALWAYS_OFFLOAD_NODE_STATUS: false + OFFLOAD_NODE_STATUS_TTL: 30s + WORKFLOW_GC_PERIOD: 30s + UPPERIO_DB_DEBUG: 1 + ARCHIVED_WORKFLOW_GC_PERIOD: 30s + # ports: "9090" + build-argo: + cmd: make ./dist/argo + deps: [go-deps] + env: + STATIC_FILES: false + sources: [cmd/argo, config, errors, persist, pkg, util, server, workflow] + # mutex: build + server: + cmd: ./dist/argo server + deps: [build-argo, port-forward] + env: + ARGO_X_FRAME_OPTIONS: SAMEORIGIN + ARGO_SECURE: false + ARGO_NAMESPACE: argo + ARGO_NAMESPACED: true + ARGO_LOG_LEVEL: info + ARGO_AUTH_MODE: hybrid + ARGO_MANAGED_NAMESPACE: argo + UPPERIO_DB_DEBUG: 1 + # ports: "2746" + ui-deps: + cmd: yarn install + dir: ui + # mutex: downloads + ui: + cmd: yarn start + dir: ui + deps: [ui-deps, server] + # ports: "8080" + executor: + cmd: make argoexec-image + sources: [cmd/argoexec, config, errors, pkg, util, workflow] + # mutex: docker + example: + cmd: kubectl create -f examples/hello-world.yaml + deps: [install] + # mutex: docker + build: + deps: [build-controller, build-argo] + pre-up: + deps: [build, install, executor, example] + up: + deps: [pre-up, controller, server, ui] From 1e9d3c5bfe9d82f89190f68b139c418848fa91e8 Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Mon, 2 Oct 2023 14:42:02 -0400 Subject: [PATCH 2/8] build: rename `tasks.yaml` -> `Taskfile.yml` - required name change - convention to use capital for the first letter: https://taskfile.dev/styleguide/#use-taskfileyml-and-not-taskfileyml Signed-off-by: Anton Gilgur --- tasks.yaml => Taskfile.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tasks.yaml => Taskfile.yml (100%) diff --git a/tasks.yaml b/Taskfile.yml similarity index 100% rename from tasks.yaml rename to Taskfile.yml From 7937cbbce9e8e356d96ea8027197e7dd4b49f0ca Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Mon, 2 Oct 2023 14:45:43 -0400 Subject: [PATCH 3/8] chore: follow Taskfile styleguide - new lines between tasks: https://taskfile.dev/styleguide/#add-spaces-between-tasks - consistent ordering of keywords: https://taskfile.dev/styleguide/#use-the-correct-order-of-keywords - strangely it doesn't mention keywords _within_ tasks, but I did `cmd` -> `dir` -> `deps` -> `sources` -> `env` consistently - similar to how I ordered the GH Actions keywords too Signed-off-by: Anton Gilgur --- Taskfile.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index 42471d6a8f2d..cea5f58a353d 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -16,22 +16,26 @@ tasks: go-deps: cmd: go mod download # mutex: download + install: cmd: sh -c "make install PROFILE=$PROFILE" - env: - PROFILE: minimal deps: [go-deps] sources: [manifests] + env: + PROFILE: minimal # mutex: docker + build-controller: cmd: make ./dist/workflow-controller - sources: [cmd/workflow-controller, config, errors, persist, pkg, util, workflow] deps: [go-deps] + sources: [cmd/workflow-controller, config, errors, persist, pkg, util, workflow] # mutex: build + port-forward: cmd: ./hack/port-forward.sh - # ports: 9000 deps: [install] + # ports: 9000 + controller: cmd: ./dist/workflow-controller deps: [install, build-controller, port-forward] @@ -51,13 +55,15 @@ tasks: UPPERIO_DB_DEBUG: 1 ARCHIVED_WORKFLOW_GC_PERIOD: 30s # ports: "9090" + build-argo: cmd: make ./dist/argo deps: [go-deps] + sources: [cmd/argo, config, errors, persist, pkg, util, server, workflow] env: STATIC_FILES: false - sources: [cmd/argo, config, errors, persist, pkg, util, server, workflow] # mutex: build + server: cmd: ./dist/argo server deps: [build-argo, port-forward] @@ -71,26 +77,33 @@ tasks: ARGO_MANAGED_NAMESPACE: argo UPPERIO_DB_DEBUG: 1 # ports: "2746" + ui-deps: cmd: yarn install dir: ui # mutex: downloads + ui: cmd: yarn start dir: ui deps: [ui-deps, server] # ports: "8080" + executor: cmd: make argoexec-image sources: [cmd/argoexec, config, errors, pkg, util, workflow] # mutex: docker + example: cmd: kubectl create -f examples/hello-world.yaml deps: [install] # mutex: docker + build: deps: [build-controller, build-argo] + pre-up: deps: [build, install, executor, example] + up: deps: [pre-up, controller, server, ui] From 3040183371384c785251a56e322d8e64de3a701a Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Mon, 2 Oct 2023 14:54:03 -0400 Subject: [PATCH 4/8] build: replace all installation and usage of `kit` with `task` - following the installation guide: https://taskfile.dev/installation/ - use `npm` for the `Makefile` install as it's compatible on different OSes and matches some of our existing tools (like `mdspell`, `markdownlint`, etc) - NOTE: this does still need adding to the Nix dir - replace `kit` in `Makefile`, `pre-build.sh`, and `ci-build.yaml` with `task` Signed-off-by: Anton Gilgur --- .devcontainer/pre-build.sh | 6 +++--- .github/workflows/ci-build.yaml | 2 +- Makefile | 23 +++++++++-------------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/.devcontainer/pre-build.sh b/.devcontainer/pre-build.sh index f7b60189a41f..14f06e711e03 100755 --- a/.devcontainer/pre-build.sh +++ b/.devcontainer/pre-build.sh @@ -12,8 +12,8 @@ chmod +x ./kubectl sudo mv ./kubectl /usr/local/bin/kubectl kubectl cluster-info -# install kit -curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | sh +# install task +sh -c "$(curl --location https://taskfile.dev/install.sh)" -- -d -b /usr/local/bin # install protocol buffer compiler (protoc) sudo apt update @@ -23,7 +23,7 @@ sudo apt install -y protobuf-compiler sudo chown -R vscode:vscode /home/vscode/go # download dependencies and do first-pass compile -CI=1 kit pre-up +CI=1 task pre-up # Patch CoreDNS to have host.docker.internal inside the cluster available kubectl get cm coredns -n kube-system -o yaml | sed "s/ NodeHosts: |/ NodeHosts: |\n `grep host.docker.internal /etc/hosts`/" | kubectl apply -f - diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index a8dd8bb06748..d2eba5f80b7c 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -149,7 +149,7 @@ jobs: - name: Install manifests run: make install PROFILE=${{matrix.profile}} STATIC_FILES=false - name: Build controller - run: make controller kit STATIC_FILES=false + run: make controller task STATIC_FILES=false - name: Build CLI run: make cli STATIC_FILES=false if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} diff --git a/Makefile b/Makefile index 40da27f2e8e8..d64be6c711ec 100644 --- a/Makefile +++ b/Makefile @@ -517,27 +517,22 @@ dist/argosay: mkdir -p dist cp test/e2e/images/argosay/v2/argosay dist/ -.PHONY: kit -kit: -ifeq ($(shell command -v kit),) -ifeq ($(shell uname),Darwin) - brew tap kitproj/kit --custom-remote https://github.com/kitproj/kit - brew install kit -else - curl -q https://raw.githubusercontent.com/kitproj/kit/main/install.sh | tag=v0.1.8 sh -endif +.PHONY: task +task: +# update this in Nix when upgrading it here +ifneq ($(USE_NIX), true) + npm i -g @go-task/cli@3.30.1 endif - .PHONY: start ifeq ($(RUN_MODE),local) ifeq ($(API),true) -start: install controller kit cli +start: install controller task cli else -start: install controller kit +start: install controller task endif else -start: install kit +start: install task endif @echo "starting STATIC_FILES=$(STATIC_FILES) (DEV_BRANCH=$(DEV_BRANCH), GIT_BRANCH=$(GIT_BRANCH)), AUTH_MODE=$(AUTH_MODE), RUN_MODE=$(RUN_MODE), MANAGED_NAMESPACE=$(MANAGED_NAMESPACE)" ifneq ($(API),true) @@ -558,7 +553,7 @@ endif grep '127.0.0.1.*postgres' /etc/hosts grep '127.0.0.1.*mysql' /etc/hosts ifeq ($(RUN_MODE),local) - env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) PROFILE=$(PROFILE) kit $(TASKS) + env DEFAULT_REQUEUE_TIME=$(DEFAULT_REQUEUE_TIME) ARGO_SECURE=$(SECURE) ALWAYS_OFFLOAD_NODE_STATUS=$(ALWAYS_OFFLOAD_NODE_STATUS) ARGO_LOG_LEVEL=$(LOG_LEVEL) UPPERIO_DB_DEBUG=$(UPPERIO_DB_DEBUG) ARGO_AUTH_MODE=$(AUTH_MODE) ARGO_NAMESPACED=$(NAMESPACED) ARGO_NAMESPACE=$(KUBE_NAMESPACE) ARGO_MANAGED_NAMESPACE=$(MANAGED_NAMESPACE) ARGO_EXECUTOR_PLUGINS=$(PLUGINS) PROFILE=$(PROFILE) task $(TASKS) endif .PHONY: wait From 6aaf99ec2e7f46aebd21fce3b8d8984a3b6495c6 Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Tue, 3 Oct 2023 19:12:26 -0400 Subject: [PATCH 5/8] add task checksums to gitignore Signed-off-by: Anton Gilgur --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c24fdb1b5f25..eb63ee748019 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ Pipfile .idea/ .node-version .DS_Store +# checksums etc that `task` stores for caching / memoizing +.task/ vendor/ dist/ # delve debug binaries @@ -45,6 +47,6 @@ sdks/python/client/dist/* manifests/install.yaml manifests/namespace-install.yaml /logs -node_modules +node_modules result .devenv From bf07cea24ad9fbff0e9503547ac30db6830e6b4a Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Tue, 3 Oct 2023 19:12:42 -0400 Subject: [PATCH 6/8] add back `./hack/port-forward.sh` to `ci-build.yaml` - it was removed when `goreman` was replaced by `kit` in 05e13182bb6be97ba6f91a24ea0876cc349d9ab6 - add it back since `task` does not do port-forwarding - should hopefully fix all the CI issues in E2E tests Signed-off-by: Anton Gilgur --- .github/workflows/ci-build.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index d2eba5f80b7c..4a936fb9dfa1 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -153,6 +153,8 @@ jobs: - name: Build CLI run: make cli STATIC_FILES=false if: ${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} + - name: Start port forward + run: ./hack/port-forward.sh - name: Start controller/API run: make start PROFILE=${{matrix.profile}} AUTH_MODE=client STATIC_FILES=false LOG_LEVEL=info API=${{matrix.test == 'test-api' || matrix.test == 'test-cli' || matrix.test == 'test-java-sdk' || matrix.test == 'test-python-sdk'}} UI=false > /tmp/argo.log 2>&1 & - name: Wait for controller to be up From 26d51febd62979ba11daef1aa5d0bb85b53bb00c Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Tue, 3 Oct 2023 19:50:16 -0400 Subject: [PATCH 7/8] properly background continuous tasks - i.e. `controller`, `server`, and `ui`, which all just run forever - previously `task` was waiting for these to complete as `deps`, but that would never happen because they run forever Signed-off-by: Anton Gilgur --- Taskfile.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index cea5f58a353d..d30419bdeaa5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -37,7 +37,7 @@ tasks: # ports: 9000 controller: - cmd: ./dist/workflow-controller + cmd: ./dist/workflow-controller & deps: [install, build-controller, port-forward] env: ARGO_EXECUTOR_PLUGINS: false @@ -65,7 +65,7 @@ tasks: # mutex: build server: - cmd: ./dist/argo server + cmd: ./dist/argo server & deps: [build-argo, port-forward] env: ARGO_X_FRAME_OPTIONS: SAMEORIGIN @@ -84,7 +84,7 @@ tasks: # mutex: downloads ui: - cmd: yarn start + cmd: yarn start & dir: ui deps: [ui-deps, server] # ports: "8080" From 8a1c81f749e766916e8aa161fab78f3862632c03 Mon Sep 17 00:00:00 2001 From: Anton Gilgur Date: Tue, 3 Oct 2023 19:51:48 -0400 Subject: [PATCH 8/8] optimize `ui-deps` task to not rerun unnecessarily - it was running all the time before in `kit` and now `task`, but it should only run when there are dep changes Signed-off-by: Anton Gilgur --- Taskfile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Taskfile.yml b/Taskfile.yml index d30419bdeaa5..8587662c782b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -81,6 +81,7 @@ tasks: ui-deps: cmd: yarn install dir: ui + sources: [ui/package.json, ui/yarn.lock, ui/node_modules] # mutex: downloads ui: