From 8faa97a8bb3c44c4c2b86da79c3dc12a315c171c Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Fri, 10 Oct 2025 13:33:11 +0100
Subject: [PATCH 1/8] feat: env INSTALL_LONGHORN

Add Ansible playbooks for Longhorn, MetalLB, and Traefik installation

conditional on presence of INSTALL_LONGHORN=true
---
 vagrant/dev/ubuntu/Vagrantfile                |  6 +-
 .../dev/ubuntu/ansible/install_longhorn.yaml  | 16 ++++
 .../dev/ubuntu/ansible/install_metallb.yaml   | 16 ++++
 .../dev/ubuntu/ansible/install_traefik.yaml   | 20 +++++
 .../ubuntu/ansible/provision_workstation.sh   | 82 ++++++++++++-------
 vagrant/dev/ubuntu/scripts/install_metallb.sh | 33 +++++---
 6 files changed, 129 insertions(+), 44 deletions(-)
 create mode 100644 vagrant/dev/ubuntu/ansible/install_longhorn.yaml
 create mode 100644 vagrant/dev/ubuntu/ansible/install_metallb.yaml
 create mode 100644 vagrant/dev/ubuntu/ansible/install_traefik.yaml

diff --git a/vagrant/dev/ubuntu/Vagrantfile b/vagrant/dev/ubuntu/Vagrantfile
index eee4a3e..39abb26 100644
--- a/vagrant/dev/ubuntu/Vagrantfile
+++ b/vagrant/dev/ubuntu/Vagrantfile
@@ -117,7 +117,11 @@ Vagrant.configure("2") do |config|
       vb.cpus = 1
     end    
 
-    ws.vm.provision "shell", path: "ansible/provision_workstation.sh"
+    ws.vm.provision "shell", 
+      path: "ansible/provision_workstation.sh",
+      env: {
+        "INSTALL_LONGHORN" => ENV['INSTALL_LONGHORN'] || "false"
+      }
   end
 
 
diff --git a/vagrant/dev/ubuntu/ansible/install_longhorn.yaml b/vagrant/dev/ubuntu/ansible/install_longhorn.yaml
new file mode 100644
index 0000000..e303917
--- /dev/null
+++ b/vagrant/dev/ubuntu/ansible/install_longhorn.yaml
@@ -0,0 +1,16 @@
+---
+- name: Install longhorn using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run infctl longhorn pipeline
+      ansible.builtin.command: >
+        bash -c 'cd /home/vagrant && LOG_FILE=/tmp/longhorn_log.txt LOG_FORMAT=basic infctl -f pipelines/vagrant-longhorn.json'
+      register: longhorn_result
+      ignore_errors: false
diff --git a/vagrant/dev/ubuntu/ansible/install_metallb.yaml b/vagrant/dev/ubuntu/ansible/install_metallb.yaml
new file mode 100644
index 0000000..2129d0c
--- /dev/null
+++ b/vagrant/dev/ubuntu/ansible/install_metallb.yaml
@@ -0,0 +1,16 @@
+---
+- name: Install metallb using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run ======== infctl metallb pipeline
+      ansible.builtin.command: >
+        bash -c 'cd /home/vagrant && LOG_FILE=/tmp/metallb_log.txt LOG_FORMAT=basic infctl -f ./pipelines/vagrant-metallb.json'
+      register: metallb_result
+      ignore_errors: false
diff --git a/vagrant/dev/ubuntu/ansible/install_traefik.yaml b/vagrant/dev/ubuntu/ansible/install_traefik.yaml
new file mode 100644
index 0000000..99f5656
--- /dev/null
+++ b/vagrant/dev/ubuntu/ansible/install_traefik.yaml
@@ -0,0 +1,20 @@
+---
+- name: Install traefik using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run infctl traefik pipeline
+      ansible.builtin.command: infctl -f pipelines/vagrant-ingress.json
+      args:
+        chdir: /home/vagrant
+      environment:
+        LOG_FILE: /tmp/traefik_log.txt
+        LOG_FORMAT: none
+      register: traefik_result
+      ignore_errors: false
\ No newline at end of file
diff --git a/vagrant/dev/ubuntu/ansible/provision_workstation.sh b/vagrant/dev/ubuntu/ansible/provision_workstation.sh
index 1077193..09e598a 100644
--- a/vagrant/dev/ubuntu/ansible/provision_workstation.sh
+++ b/vagrant/dev/ubuntu/ansible/provision_workstation.sh
@@ -4,7 +4,8 @@
 sudo apt-get update
 sudo apt-get install -y software-properties-common git vim python3.10-venv jq figlet
 
-source /vagrant/.envrc
+# shellcheck disable=SC1091
+source /vagrant/.envrc 
 
 # Set up ansible environment for vagrant user
 sudo -u vagrant mkdir -p /home/vagrant/.ansible
@@ -24,10 +25,10 @@ sudo chmod +x /home/vagrant/pipelines/*.sh
 
 # Copy the Vagrant private keys (these will be synced by Vagrant)
 for i in {1..3}; do
-sudo -u vagrant cp /vagrant/.vagrant/machines/vm$i/virtualbox/private_key /home/vagrant/.ssh/vm${i}_key
-sudo -u root cp /vagrant/.vagrant/machines/vm$i/virtualbox/private_key /root/.ssh/vm${i}_key
-sudo chmod 600 /home/vagrant/.ssh/vm${i}_key
-sudo chmod 600 /root/.ssh/vm${i}_key
+sudo -u vagrant cp "/vagrant/.vagrant/machines/vm$i/virtualbox/private_key" "/home/vagrant/.ssh/vm${i}_key"
+sudo -u root cp "/vagrant/.vagrant/machines/vm$i/virtualbox/private_key" "/root/.ssh/vm${i}_key"
+sudo chmod 600 "/home/vagrant/.ssh/vm${i}_key"
+sudo chmod 600 "/root/.ssh/vm${i}_key"
 done
 
 # Disable host key checking for easier learning
@@ -46,18 +47,17 @@ cd "$ANSIBLE_DIR" || {
 if [ ! -d "venv" ]; then
     echo "Creating Python virtual environment in ./venv..."
     python3 -m venv venv
-    source "venv/bin/activate"
-    if [ $? -ne 0 ]; then
+    # shellcheck disable=SC1091
+    if ! source "venv/bin/activate"; then
         echo "Failed to activate virtual environment. Please check your Python installation."
         exit 1
     fi
     echo "Virtual environment created and activated."
-    cp /vagrant/ansible/requirements.txt .
+    cp "/vagrant/ansible/requirements.txt" .
     if [ -f "requirements.txt" ]; then
         echo "Installing dependencies from requirements.txt..."
         pip install --upgrade pip
-        pip install -r requirements.txt
-        if [ $? -ne 0 ]; then
+        if ! pip install -r requirements.txt; then
             echo "Failed to install dependencies from requirements.txt."
             exit 1
         fi
@@ -76,7 +76,13 @@ ls -al "$ANSIBLE_VENV_DIR/bin/activate"
 
 if [ -d "$ANSIBLE_VENV_DIR" ]; then
     echo "Activating Ansible virtual environment..."
-    source "$ANSIBLE_VENV_DIR/bin/activate"
+    if [ -f "$ANSIBLE_VENV_DIR/bin/activate" ]; then
+        # shellcheck source=/dev/null
+        source "$ANSIBLE_VENV_DIR/bin/activate"
+    else
+        echo "Virtualenv activate script not found!" >&2
+        exit 1
+    fi
 else
     echo "Ansible virtual environment not found at $ANSIBLE_VENV_DIR. Please create it before running this script."
     exit 1
@@ -86,13 +92,13 @@ echo ""
 
 ansible --version
 
-if [ $? -ne 0 ]; then
+if ! ansible --version; then
     echo "Ansible is not installed or not found in the virtual environment. Please check your installation."
     exit 1
 fi
 
 
-eval `ssh-agent -s`
+eval "$(ssh-agent -s)"
 ssh-add # ~/machines/*/virtualbox/private_key
 
 BASHRC="/home/vagrant/.bashrc"
@@ -103,10 +109,11 @@ if ! grep -qF "$BLOCK_START" "$BASHRC"; then
     cat <<'EOF' >> "$BASHRC"
 
 # ADDED BY infctl provisioning
-eval `ssh-agent -s`
+eval "$(ssh-agent -s)"
 ssh-add ~/machines/*/virtualbox/private_key
 ssh-add -L
-source /vagrant/.envrc
+# shellcheck disable=SC1091
+source /vagrant/.envrc 
 EOF
 else
     echo "Provisioning block already present in $BASHRC"
@@ -114,7 +121,7 @@ fi
 
 echo 
 echo -------------------------
-echo 
+echo
 
 su - vagrant
 id
@@ -125,48 +132,63 @@ echo
 
 ssh-add ~/.ssh/vm*_key
 
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible --inventory-file /home/vagrant/ansible/ansible_inventory.ini  -m ping vm1,vm2,vm3 
-
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible --inventory-file /home/vagrant/ansible/ansible_inventory.ini  -m ping vm1,vm2,vm3; then
     echo "Ansible ping failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
 # install_keepalived.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_keepalived.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_keepalived.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 echo "Keepalived installation completed."
 
 # install_k3s_3node.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_k3s_3node.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_k3s_3node.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
 # copy_k8s_config.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook copy_k8s_config.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook copy_k8s_config.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_dnsmasq.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_dnsmasq.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
+# Wait for Kubernetes API to be ready
+echo "Waiting for 30 seconds for Kubernetes API to be ready..."
+sleep 30
+echo "done waiting for kubernetes API"
+
 # check infctl 
-cd /home/vagrant
-bash /home/vagrant/scripts/check_install_infctl.sh 
-if [ $? -ne 0 ]; then
+cd /home/vagrant || exit
+if ! bash /home/vagrant/scripts/check_install_infctl.sh; then
     echo "infctl check failed. Please check your installation."
     exit 1
 fi
 
+# Optionally install Longhorn, MetalLB, and Traefik
+if [ "${INSTALL_LONGHORN}" = "true" ]; then
+    cd /home/vagrant/ansible || { echo "Failed to change directory to /home/vagrant/ansible"; exit 1; }
 
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_longhorn.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
 
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_metallb.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
+
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_traefik.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
+fi
diff --git a/vagrant/dev/ubuntu/scripts/install_metallb.sh b/vagrant/dev/ubuntu/scripts/install_metallb.sh
index 838d23a..e23e535 100755
--- a/vagrant/dev/ubuntu/scripts/install_metallb.sh
+++ b/vagrant/dev/ubuntu/scripts/install_metallb.sh
@@ -12,24 +12,31 @@ if ! kubectl get deployment -n metallb-system controller &>/dev/null; then
     exit 1
   fi
   
-  # Wait for MetalLB components to be ready
-  echo "Waiting for MetalLB components to be ready..."
-  kubectl wait --namespace metallb-system \
-    --for=condition=ready pod \
-    --selector=app=metallb \
-    --timeout=90s
+  echo "Waiting for MetalLB pods to be in 'Running' state..."
+  MAX_RETRIES=10
+  RETRY=0
+
+  while [ $RETRY -lt $MAX_RETRIES ]; do
+    NOT_READY_PODS=$(kubectl -n metallb-system get pods --no-headers | grep -v 'Running' | wc -l)
+    if [ "$NOT_READY_PODS" -eq 0 ]; then
+      echo "All MetalLB pods are running."
+      break
+    else
+      echo "$NOT_READY_PODS MetalLB pods are not ready yet. Waiting..."
+      RETRY=$((RETRY + 1))
+      sleep 5
+    fi
+  done
+
+  if [ "$NOT_READY_PODS" -ne 0 ]; then
+    echo "Failed to get all MetalLB pods running after $MAX_RETRIES attempts."
+    exit 1
+  fi
 
 else
   echo "MetalLB is already installed."
 fi
 
-# Wait for the webhook service to be ready
-echo "Waiting for MetalLB webhook service to be ready..."
-kubectl wait --namespace metallb-system \
-  --for=condition=ready pod \
-  --selector=component=webhook \
-  --timeout=90s
-
 # Check if the IPAddressPool already exists
 if ! kubectl get ipaddresspool -n metallb-system default &>/dev/null; then
   echo "Creating MetalLB IPAddressPool..."

From bb4d0cc701393d5e40ed3cc00b214b99d3f3ca3d Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 15:58:09 +0100
Subject: [PATCH 2/8] feat: update Forgejo deployment URL and add installation
 check in startup script

---
 gcloud/tf/doit.tf                     | 0
 gcloud/tf/k3s/forgejo/deployment.yaml | 2 +-
 gcloud/tf/scripts/k3s-vm-startup.sh   | 8 ++++++++
 3 files changed, 9 insertions(+), 1 deletion(-)
 delete mode 100644 gcloud/tf/doit.tf

diff --git a/gcloud/tf/doit.tf b/gcloud/tf/doit.tf
deleted file mode 100644
index e69de29..0000000
diff --git a/gcloud/tf/k3s/forgejo/deployment.yaml b/gcloud/tf/k3s/forgejo/deployment.yaml
index cd3afdc..445ec2d 100644
--- a/gcloud/tf/k3s/forgejo/deployment.yaml
+++ b/gcloud/tf/k3s/forgejo/deployment.yaml
@@ -26,7 +26,7 @@ spec:
         - name: FORGEJO__repository__ENABLE_PUSH_CREATE_USER
           value: "true"
         - name: FORGEJO__server__ROOT_URL
-          value: "https://frg.headshed.dev/"
+          value: "https://frgdr.headshed.dev/"
         - name: FORGEJO__repository__DEFAULT_BRANCH
           value: "main"
         - name: FORGEJO__server__LFS_START_SERVER
diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index dd875fd..b22c886 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -100,3 +100,11 @@ if [[ ! -d "$INFCTL_INSTALL_DIR" ]]; then
     chown -R user:user "$INFCTL_INSTALL_DIR"
 fi
 
+# check to see if INSTALL_FORGEJO is set to "true"
+if [[ "$INSTALL_FORGEJO" == "true" ]]; then
+    # install forgejo using infctl
+    # ....
+    LOG_FORMAT=none infctl -f "${INFCTL_INSTALL_DIR}/infctl-cli/gcloud/tf/scripts/install-forgejo-pipeline.json" 
+    touch /etc/forgejo_was_installed
+
+fi

From b4c0f17b12fcf0201cf60ee2c876fe6ba798f1db Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 16:32:05 +0100
Subject: [PATCH 3/8] feat: add script to copy .env file to k3s-vm-1 after
 pre-flight checks

---
 .../tf/scripts/build-gcloud-k3s-pipeline.json |  9 ++++++
 gcloud/tf/scripts/copy_env_to_first_node.sh   | 31 +++++++++++++++++++
 2 files changed, 40 insertions(+)
 create mode 100755 gcloud/tf/scripts/copy_env_to_first_node.sh

diff --git a/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json b/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
index 179282f..d1fd61b 100644
--- a/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
+++ b/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
@@ -34,5 +34,14 @@
         ],
         "retryCount": 0,
         "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
     }
 ]
\ No newline at end of file
diff --git a/gcloud/tf/scripts/copy_env_to_first_node.sh b/gcloud/tf/scripts/copy_env_to_first_node.sh
new file mode 100755
index 0000000..5bce28a
--- /dev/null
+++ b/gcloud/tf/scripts/copy_env_to_first_node.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+source .env
+
+for i in {1..10}; do
+    # Check if the instance is running
+    INSTANCE_STATUS=$(gcloud compute instances describe k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --format='get(status)')
+    if [[ "$INSTANCE_STATUS" != "RUNNING" ]]; then
+        echo "Instance k3s-vm-1 is not running. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+        continue
+    fi
+
+    # Check if the directory exists on the remote host
+    if gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+        echo "/opt/src/infctl-cli/ exists on k3s-vm-1."
+        break
+    else
+        echo "/opt/src/infctl-cli/ does not exist yet. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if ! gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+    echo "ERROR: /opt/src/infctl-cli/ does not exist on k3s-vm-1 after 10 attempts. Exiting."
+    exit 1
+fi
+
+gcloud compute scp .env  k3s-vm-1:/opt/src/infctl-cli/.env --zone=us-central1-a --project=$PROJECT_NAME
+

From f23e1c41ff34243cb6f5fb6925cc78786ac3ed0d Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 16:50:12 +0100
Subject: [PATCH 4/8] feat: add .env file existence check and load it in
 startup script

---
 gcloud/tf/scripts/k3s-vm-startup.sh | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index b22c886..44793a1 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -100,6 +100,25 @@ if [[ ! -d "$INFCTL_INSTALL_DIR" ]]; then
     chown -R user:user "$INFCTL_INSTALL_DIR"
 fi
 
+for i in {1..100}; do
+    if [[ -f /opt/src/infctl-cli/.env ]]; then
+        echo ".env file found."
+        break
+    else
+        echo ".env file not found. Attempt $i/100. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if [[ ! -f /opt/src/infctl-cli/.env ]]; then
+    echo "ERROR: .env file not found after 10 attempts. Exiting."
+    exit 1
+fi
+
+# load .env file
+source /opt/src/infctl-cli/.env
+
 # check to see if INSTALL_FORGEJO is set to "true"
 if [[ "$INSTALL_FORGEJO" == "true" ]]; then
     # install forgejo using infctl

From 02b114e0e6b6e7dc025f2d069ecc3498cb112b22 Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 17:49:59 +0100
Subject: [PATCH 5/8] feat: add scripts for pre-flight checks and user input
 wait in k3s pipeline

---
 .../build-gcloud-k3s-pipeline-wait-dns.json   | 56 +++++++++++++++++++
 gcloud/tf/scripts/k3s-vm-startup.sh           |  4 ++
 gcloud/tf/scripts/wait_for_user_input_dns.sh  | 11 ++++
 3 files changed, 71 insertions(+)
 create mode 100644 gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
 create mode 100755 gcloud/tf/scripts/wait_for_user_input_dns.sh

diff --git a/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json b/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
new file mode 100644
index 0000000..eeb5a5f
--- /dev/null
+++ b/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
@@ -0,0 +1,56 @@
+[
+    {
+        "name": "run pre-flight checks",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/pre-flight-checks.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "list gcloud infrastructure",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/list_gloud_infra.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "create tfvars",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/create_tfvars.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "run tofu",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/run_tofu.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "wait for user input to continue",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/wait_for_user_input_dns.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    }
+]
\ No newline at end of file
diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index 44793a1..6e2fcf9 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Redirect all output to a log file for reliability
+exec > /tmp/startup.log 2>&1
+
 INFCTL_GIT_REPO="https://codeberg.org/headshed/infctl-cli.git"
 INFCTL_GIT_REPO_BRANCH="main"
 INFCTL_INSTALL_DIR="/opt/src"
@@ -123,6 +126,7 @@ source /opt/src/infctl-cli/.env
 if [[ "$INSTALL_FORGEJO" == "true" ]]; then
     # install forgejo using infctl
     # ....
+    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
     LOG_FORMAT=none infctl -f "${INFCTL_INSTALL_DIR}/infctl-cli/gcloud/tf/scripts/install-forgejo-pipeline.json" 
     touch /etc/forgejo_was_installed
 
diff --git a/gcloud/tf/scripts/wait_for_user_input_dns.sh b/gcloud/tf/scripts/wait_for_user_input_dns.sh
new file mode 100755
index 0000000..a356571
--- /dev/null
+++ b/gcloud/tf/scripts/wait_for_user_input_dns.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+echo "Please configure DNS using the IP address from the previous stage."
+echo "you have 120 seconds."
+for i in {120..1}; do
+    echo -ne "Time remaining: $i seconds\r"
+    sleep 1
+done
+echo ""
+
+exit 0
\ No newline at end of file

From 8f1955882693e0fa4659740990725ad685c83190 Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 15:58:09 +0100
Subject: [PATCH 6/8] feat: update Forgejo deployment

add DNS update step and complete forgejo deployment after build
---
 gcloud/tf/doit.tf                             |  0
 gcloud/tf/k3s/forgejo/deployment.yaml         |  2 +-
 .../build-gcloud-k3s-pipeline-wait-dns.json   | 56 +++++++++++++++++++
 .../tf/scripts/build-gcloud-k3s-pipeline.json |  9 +++
 gcloud/tf/scripts/copy_env_to_first_node.sh   | 31 ++++++++++
 gcloud/tf/scripts/k3s-vm-startup.sh           | 31 ++++++++++
 gcloud/tf/scripts/wait_for_user_input_dns.sh  | 11 ++++
 7 files changed, 139 insertions(+), 1 deletion(-)
 delete mode 100644 gcloud/tf/doit.tf
 create mode 100644 gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
 create mode 100755 gcloud/tf/scripts/copy_env_to_first_node.sh
 create mode 100755 gcloud/tf/scripts/wait_for_user_input_dns.sh

diff --git a/gcloud/tf/doit.tf b/gcloud/tf/doit.tf
deleted file mode 100644
index e69de29..0000000
diff --git a/gcloud/tf/k3s/forgejo/deployment.yaml b/gcloud/tf/k3s/forgejo/deployment.yaml
index cd3afdc..445ec2d 100644
--- a/gcloud/tf/k3s/forgejo/deployment.yaml
+++ b/gcloud/tf/k3s/forgejo/deployment.yaml
@@ -26,7 +26,7 @@ spec:
         - name: FORGEJO__repository__ENABLE_PUSH_CREATE_USER
           value: "true"
         - name: FORGEJO__server__ROOT_URL
-          value: "https://frg.headshed.dev/"
+          value: "https://frgdr.headshed.dev/"
         - name: FORGEJO__repository__DEFAULT_BRANCH
           value: "main"
         - name: FORGEJO__server__LFS_START_SERVER
diff --git a/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json b/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
new file mode 100644
index 0000000..eeb5a5f
--- /dev/null
+++ b/gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json
@@ -0,0 +1,56 @@
+[
+    {
+        "name": "run pre-flight checks",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/pre-flight-checks.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "list gcloud infrastructure",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/list_gloud_infra.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "create tfvars",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/create_tfvars.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "run tofu",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/run_tofu.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "wait for user input to continue",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/wait_for_user_input_dns.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    }
+]
\ No newline at end of file
diff --git a/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json b/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
index 179282f..d1fd61b 100644
--- a/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
+++ b/gcloud/tf/scripts/build-gcloud-k3s-pipeline.json
@@ -34,5 +34,14 @@
         ],
         "retryCount": 0,
         "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
     }
 ]
\ No newline at end of file
diff --git a/gcloud/tf/scripts/copy_env_to_first_node.sh b/gcloud/tf/scripts/copy_env_to_first_node.sh
new file mode 100755
index 0000000..5bce28a
--- /dev/null
+++ b/gcloud/tf/scripts/copy_env_to_first_node.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+source .env
+
+for i in {1..10}; do
+    # Check if the instance is running
+    INSTANCE_STATUS=$(gcloud compute instances describe k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --format='get(status)')
+    if [[ "$INSTANCE_STATUS" != "RUNNING" ]]; then
+        echo "Instance k3s-vm-1 is not running. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+        continue
+    fi
+
+    # Check if the directory exists on the remote host
+    if gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+        echo "/opt/src/infctl-cli/ exists on k3s-vm-1."
+        break
+    else
+        echo "/opt/src/infctl-cli/ does not exist yet. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if ! gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+    echo "ERROR: /opt/src/infctl-cli/ does not exist on k3s-vm-1 after 10 attempts. Exiting."
+    exit 1
+fi
+
+gcloud compute scp .env  k3s-vm-1:/opt/src/infctl-cli/.env --zone=us-central1-a --project=$PROJECT_NAME
+
diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index dd875fd..6e2fcf9 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Redirect all output to a log file for reliability
+exec > /tmp/startup.log 2>&1
+
 INFCTL_GIT_REPO="https://codeberg.org/headshed/infctl-cli.git"
 INFCTL_GIT_REPO_BRANCH="main"
 INFCTL_INSTALL_DIR="/opt/src"
@@ -100,3 +103,31 @@ if [[ ! -d "$INFCTL_INSTALL_DIR" ]]; then
     chown -R user:user "$INFCTL_INSTALL_DIR"
 fi
 
+for i in {1..100}; do
+    if [[ -f /opt/src/infctl-cli/.env ]]; then
+        echo ".env file found."
+        break
+    else
+        echo ".env file not found. Attempt $i/100. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if [[ ! -f /opt/src/infctl-cli/.env ]]; then
+    echo "ERROR: .env file not found after 10 attempts. Exiting."
+    exit 1
+fi
+
+# load .env file
+source /opt/src/infctl-cli/.env
+
+# check to see if INSTALL_FORGEJO is set to "true"
+if [[ "$INSTALL_FORGEJO" == "true" ]]; then
+    # install forgejo using infctl
+    # ....
+    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+    LOG_FORMAT=none infctl -f "${INFCTL_INSTALL_DIR}/infctl-cli/gcloud/tf/scripts/install-forgejo-pipeline.json" 
+    touch /etc/forgejo_was_installed
+
+fi
diff --git a/gcloud/tf/scripts/wait_for_user_input_dns.sh b/gcloud/tf/scripts/wait_for_user_input_dns.sh
new file mode 100755
index 0000000..a356571
--- /dev/null
+++ b/gcloud/tf/scripts/wait_for_user_input_dns.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+echo "Please configure DNS using the IP address from the previous stage."
+echo "you have 120 seconds."
+for i in {120..1}; do
+    echo -ne "Time remaining: $i seconds\r"
+    sleep 1
+done
+echo ""
+
+exit 0
\ No newline at end of file

From e0906c821dbf15c1258d3fae53d90e4fb76f0f84 Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 18:01:55 +0100
Subject: [PATCH 7/8] fix: change to repo dir

---
 gcloud/tf/scripts/k3s-vm-startup.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index 6e2fcf9..48c5d29 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -121,6 +121,7 @@ fi
 
 # load .env file
 source /opt/src/infctl-cli/.env
+cd $INFCTL_INSTALL_DIR/infctl-cli || "echo 'Failed to change directory to $INFCTL_INSTALL_DIR/infctl-cli' ; exit 1"
 
 # check to see if INSTALL_FORGEJO is set to "true"
 if [[ "$INSTALL_FORGEJO" == "true" ]]; then

From 2f118907795233ec869f18a02c512bd3059860f1 Mon Sep 17 00:00:00 2001
From: jon brookes <marshyon@gmail.com>
Date: Tue, 14 Oct 2025 18:15:32 +0100
Subject: [PATCH 8/8] feat: add .env file check and load it in startup script

---
 gcloud/tf/scripts/k3s-vm-startup.sh | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/gcloud/tf/scripts/k3s-vm-startup.sh b/gcloud/tf/scripts/k3s-vm-startup.sh
index 658ecac..48c5d29 100644
--- a/gcloud/tf/scripts/k3s-vm-startup.sh
+++ b/gcloud/tf/scripts/k3s-vm-startup.sh
@@ -103,3 +103,32 @@ if [[ ! -d "$INFCTL_INSTALL_DIR" ]]; then
     chown -R user:user "$INFCTL_INSTALL_DIR"
 fi
 
+for i in {1..100}; do
+    if [[ -f /opt/src/infctl-cli/.env ]]; then
+        echo ".env file found."
+        break
+    else
+        echo ".env file not found. Attempt $i/100. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if [[ ! -f /opt/src/infctl-cli/.env ]]; then
+    echo "ERROR: .env file not found after 10 attempts. Exiting."
+    exit 1
+fi
+
+# load .env file
+source /opt/src/infctl-cli/.env
+cd $INFCTL_INSTALL_DIR/infctl-cli || "echo 'Failed to change directory to $INFCTL_INSTALL_DIR/infctl-cli' ; exit 1"
+
+# check to see if INSTALL_FORGEJO is set to "true"
+if [[ "$INSTALL_FORGEJO" == "true" ]]; then
+    # install forgejo using infctl
+    # ....
+    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+    LOG_FORMAT=none infctl -f "${INFCTL_INSTALL_DIR}/infctl-cli/gcloud/tf/scripts/install-forgejo-pipeline.json" 
+    touch /etc/forgejo_was_installed
+
+fi