feat: add .env file check and load it in startup script

add DNS update step and complete forgejo deployment after build

gcloud/tf/k3s/forgejo/deployment.yaml

@@ -26,7 +26,7 @@ spec:
         - name: FORGEJO__repository__ENABLE_PUSH_CREATE_USER
           value: "true"
         - name: FORGEJO__server__ROOT_URL
-          value: "https://frg.headshed.dev/"
+          value: "https://frgdr.headshed.dev/"
         - name: FORGEJO__repository__DEFAULT_BRANCH
           value: "main"
         - name: FORGEJO__server__LFS_START_SERVER

gcloud/tf/scripts/build-gcloud-k3s-pipeline-wait-dns.json

@@ -0,0 +1,56 @@
+[
+    {
+        "name": "run pre-flight checks",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/pre-flight-checks.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "list gcloud infrastructure",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/list_gloud_infra.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "create tfvars",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/create_tfvars.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "run tofu",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/run_tofu.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "wait for user input to continue",
+        "function": "RunCommand",
+        "params": [
+            "./gcloud/tf/scripts/wait_for_user_input_dns.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
+    }
+]

gcloud/tf/scripts/build-gcloud-k3s-pipeline.json

@@ -34,5 +34,14 @@
         ],
         "retryCount": 0,
         "shouldAbort": true
+    },
+    {
+        "name": "copy .env to k3s-vm-1",
+        "function": "RunCommand",
+        "params": [
+            "gcloud/tf/scripts/copy_env_to_first_node.sh"
+        ],
+        "retryCount": 0,
+        "shouldAbort": true
     }
 ]

gcloud/tf/scripts/copy_env_to_first_node.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+source .env
+
+for i in {1..10}; do
+    # Check if the instance is running
+    INSTANCE_STATUS=$(gcloud compute instances describe k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --format='get(status)')
+    if [[ "$INSTANCE_STATUS" != "RUNNING" ]]; then
+        echo "Instance k3s-vm-1 is not running. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+        continue
+    fi
+
+    # Check if the directory exists on the remote host
+    if gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+        echo "/opt/src/infctl-cli/ exists on k3s-vm-1."
+        break
+    else
+        echo "/opt/src/infctl-cli/ does not exist yet. Attempt $i/10. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if ! gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project="$PROJECT_NAME" --command="test -d /opt/src/infctl-cli/"; then
+    echo "ERROR: /opt/src/infctl-cli/ does not exist on k3s-vm-1 after 10 attempts. Exiting."
+    exit 1
+fi
+
+gcloud compute scp .env  k3s-vm-1:/opt/src/infctl-cli/.env --zone=us-central1-a --project=$PROJECT_NAME
+

gcloud/tf/scripts/k3s-vm-startup.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+# Redirect all output to a log file for reliability
+exec > /tmp/startup.log 2>&1
+
 INFCTL_GIT_REPO="https://codeberg.org/headshed/infctl-cli.git"
 INFCTL_GIT_REPO_BRANCH="main"
 INFCTL_INSTALL_DIR="/opt/src"
@@ -100,3 +103,32 @@ if [[ ! -d "$INFCTL_INSTALL_DIR" ]]; then
     chown -R user:user "$INFCTL_INSTALL_DIR"
 fi
 
+for i in {1..100}; do
+    if [[ -f /opt/src/infctl-cli/.env ]]; then
+        echo ".env file found."
+        break
+    else
+        echo ".env file not found. Attempt $i/100. Waiting 5 seconds..."
+        sleep 5
+    fi
+done
+
+# Final check after loop
+if [[ ! -f /opt/src/infctl-cli/.env ]]; then
+    echo "ERROR: .env file not found after 10 attempts. Exiting."
+    exit 1
+fi
+
+# load .env file
+source /opt/src/infctl-cli/.env
+cd $INFCTL_INSTALL_DIR/infctl-cli || "echo 'Failed to change directory to $INFCTL_INSTALL_DIR/infctl-cli' ; exit 1"
+
+# check to see if INSTALL_FORGEJO is set to "true"
+if [[ "$INSTALL_FORGEJO" == "true" ]]; then
+    # install forgejo using infctl
+    # ....
+    export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+    LOG_FORMAT=none infctl -f "${INFCTL_INSTALL_DIR}/infctl-cli/gcloud/tf/scripts/install-forgejo-pipeline.json" 
+    touch /etc/forgejo_was_installed
+
+fi

gcloud/tf/scripts/wait_for_user_input_dns.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+echo "Please configure DNS using the IP address from the previous stage."
+echo "you have 120 seconds."
+for i in {120..1}; do
+    echo -ne "Time remaining: $i seconds\r"
+    sleep 1
+done
+echo ""
+
+exit 0

vagrant/dev/ubuntu/Vagrantfile

@@ -117,7 +117,11 @@ Vagrant.configure("2") do |config|
       vb.cpus = 1
     end    
 
-    ws.vm.provision "shell", path: "ansible/provision_workstation.sh"
+    ws.vm.provision "shell", 
+      path: "ansible/provision_workstation.sh",
+      env: {
+        "INSTALL_LONGHORN" => ENV['INSTALL_LONGHORN'] || "false"
+      }
   end
 
 

vagrant/dev/ubuntu/ansible/install_longhorn.yaml

@@ -0,0 +1,16 @@
+---
+- name: Install longhorn using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run infctl longhorn pipeline
+      ansible.builtin.command: >
+        bash -c 'cd /home/vagrant && LOG_FILE=/tmp/longhorn_log.txt LOG_FORMAT=basic infctl -f pipelines/vagrant-longhorn.json'
+      register: longhorn_result
+      ignore_errors: false

vagrant/dev/ubuntu/ansible/install_metallb.yaml

@@ -0,0 +1,16 @@
+---
+- name: Install metallb using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run ======== infctl metallb pipeline
+      ansible.builtin.command: >
+        bash -c 'cd /home/vagrant && LOG_FILE=/tmp/metallb_log.txt LOG_FORMAT=basic infctl -f ./pipelines/vagrant-metallb.json'
+      register: metallb_result
+      ignore_errors: false

vagrant/dev/ubuntu/ansible/install_traefik.yaml

@@ -0,0 +1,20 @@
+---
+- name: Install traefik using infctl
+  hosts: localhost
+  become: true
+  become_user: vagrant
+  serial: 1 # Ensure tasks are executed one host at a time
+  vars_files:
+    - vars.yaml
+
+  tasks:
+
+    - name: run infctl traefik pipeline
+      ansible.builtin.command: infctl -f pipelines/vagrant-ingress.json
+      args:
+        chdir: /home/vagrant
+      environment:
+        LOG_FILE: /tmp/traefik_log.txt
+        LOG_FORMAT: none
+      register: traefik_result
+      ignore_errors: false

vagrant/dev/ubuntu/ansible/provision_workstation.sh

@@ -4,7 +4,8 @@
 sudo apt-get update
 sudo apt-get install -y software-properties-common git vim python3.10-venv jq figlet
 
-source /vagrant/.envrc
+# shellcheck disable=SC1091
+source /vagrant/.envrc 
 
 # Set up ansible environment for vagrant user
 sudo -u vagrant mkdir -p /home/vagrant/.ansible
@@ -24,10 +25,10 @@ sudo chmod +x /home/vagrant/pipelines/*.sh
 
 # Copy the Vagrant private keys (these will be synced by Vagrant)
 for i in {1..3}; do
-sudo -u vagrant cp /vagrant/.vagrant/machines/vm$i/virtualbox/private_key /home/vagrant/.ssh/vm${i}_key
-sudo -u root cp /vagrant/.vagrant/machines/vm$i/virtualbox/private_key /root/.ssh/vm${i}_key
-sudo chmod 600 /home/vagrant/.ssh/vm${i}_key
-sudo chmod 600 /root/.ssh/vm${i}_key
+sudo -u vagrant cp "/vagrant/.vagrant/machines/vm$i/virtualbox/private_key" "/home/vagrant/.ssh/vm${i}_key"
+sudo -u root cp "/vagrant/.vagrant/machines/vm$i/virtualbox/private_key" "/root/.ssh/vm${i}_key"
+sudo chmod 600 "/home/vagrant/.ssh/vm${i}_key"
+sudo chmod 600 "/root/.ssh/vm${i}_key"
 done
 
 # Disable host key checking for easier learning
@@ -46,18 +47,17 @@ cd "$ANSIBLE_DIR" || {
 if [ ! -d "venv" ]; then
     echo "Creating Python virtual environment in ./venv..."
     python3 -m venv venv
-    source "venv/bin/activate"
-    if [ $? -ne 0 ]; then
+    # shellcheck disable=SC1091
+    if ! source "venv/bin/activate"; then
         echo "Failed to activate virtual environment. Please check your Python installation."
         exit 1
     fi
     echo "Virtual environment created and activated."
-    cp /vagrant/ansible/requirements.txt .
+    cp "/vagrant/ansible/requirements.txt" .
     if [ -f "requirements.txt" ]; then
         echo "Installing dependencies from requirements.txt..."
         pip install --upgrade pip
-        pip install -r requirements.txt
-        if [ $? -ne 0 ]; then
+        if ! pip install -r requirements.txt; then
             echo "Failed to install dependencies from requirements.txt."
             exit 1
         fi
@@ -76,7 +76,13 @@ ls -al "$ANSIBLE_VENV_DIR/bin/activate"
 
 if [ -d "$ANSIBLE_VENV_DIR" ]; then
     echo "Activating Ansible virtual environment..."
-    source "$ANSIBLE_VENV_DIR/bin/activate"
+    if [ -f "$ANSIBLE_VENV_DIR/bin/activate" ]; then
+        # shellcheck source=/dev/null
+        source "$ANSIBLE_VENV_DIR/bin/activate"
+    else
+        echo "Virtualenv activate script not found!" >&2
+        exit 1
+    fi
 else
     echo "Ansible virtual environment not found at $ANSIBLE_VENV_DIR. Please create it before running this script."
     exit 1
@@ -86,13 +92,13 @@ echo ""
 
 ansible --version
 
-if [ $? -ne 0 ]; then
+if ! ansible --version; then
     echo "Ansible is not installed or not found in the virtual environment. Please check your installation."
     exit 1
 fi
 
 
-eval `ssh-agent -s`
+eval "$(ssh-agent -s)"
 ssh-add # ~/machines/*/virtualbox/private_key
 
 BASHRC="/home/vagrant/.bashrc"
@@ -103,10 +109,11 @@ if ! grep -qF "$BLOCK_START" "$BASHRC"; then
     cat <<'EOF' >> "$BASHRC"
 
 # ADDED BY infctl provisioning
-eval `ssh-agent -s`
+eval "$(ssh-agent -s)"
 ssh-add ~/machines/*/virtualbox/private_key
 ssh-add -L
-source /vagrant/.envrc
+# shellcheck disable=SC1091
+source /vagrant/.envrc 
 EOF
 else
     echo "Provisioning block already present in $BASHRC"
@@ -114,7 +121,7 @@ fi
 
 echo 
 echo -------------------------
-echo 
+echo
 
 su - vagrant
 id
@@ -125,48 +132,63 @@ echo
 
 ssh-add ~/.ssh/vm*_key
 
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible --inventory-file /home/vagrant/ansible/ansible_inventory.ini  -m ping vm1,vm2,vm3 
-
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible --inventory-file /home/vagrant/ansible/ansible_inventory.ini  -m ping vm1,vm2,vm3; then
     echo "Ansible ping failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
 # install_keepalived.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_keepalived.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_keepalived.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 echo "Keepalived installation completed."
 
 # install_k3s_3node.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_k3s_3node.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_k3s_3node.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
 # copy_k8s_config.yaml
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook copy_k8s_config.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook copy_k8s_config.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
-ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_dnsmasq.yaml --inventory-file ansible_inventory.ini 
-if [ $? -ne 0 ]; then
+if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_dnsmasq.yaml --inventory-file ansible_inventory.ini; then
     echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
     exit 1
 fi
 
+# Wait for Kubernetes API to be ready
+echo "Waiting for 30 seconds for Kubernetes API to be ready..."
+sleep 30
+echo "done waiting for kubernetes API"
+
 # check infctl 
-cd /home/vagrant
-bash /home/vagrant/scripts/check_install_infctl.sh 
-if [ $? -ne 0 ]; then
+cd /home/vagrant || exit
+if ! bash /home/vagrant/scripts/check_install_infctl.sh; then
     echo "infctl check failed. Please check your installation."
     exit 1
 fi
 
+# Optionally install Longhorn, MetalLB, and Traefik
+if [ "${INSTALL_LONGHORN}" = "true" ]; then
+    cd /home/vagrant/ansible || { echo "Failed to change directory to /home/vagrant/ansible"; exit 1; }
 
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_longhorn.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
 
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_metallb.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
+
+    if ! ANSIBLE_SUPPRESS_INTERPRETER_DISCOVERY_WARNING=1 ANSIBLE_HOST_KEY_CHECKING=False ansible-playbook install_traefik.yaml --inventory-file ansible_inventory.ini; then
+        echo "Ansible playbook failed. Please check your Vagrant VMs and network configuration."
+        exit 1
+    fi
+fi

vagrant/dev/ubuntu/scripts/install_metallb.sh

@@ -12,24 +12,31 @@ if ! kubectl get deployment -n metallb-system controller &>/dev/null; then
     exit 1
   fi
   
-  # Wait for MetalLB components to be ready
-  echo "Waiting for MetalLB components to be ready..."
-  kubectl wait --namespace metallb-system \
-    --for=condition=ready pod \
-    --selector=app=metallb \
-    --timeout=90s
+  echo "Waiting for MetalLB pods to be in 'Running' state..."
+  MAX_RETRIES=10
+  RETRY=0
+
+  while [ $RETRY -lt $MAX_RETRIES ]; do
+    NOT_READY_PODS=$(kubectl -n metallb-system get pods --no-headers | grep -v 'Running' | wc -l)
+    if [ "$NOT_READY_PODS" -eq 0 ]; then
+      echo "All MetalLB pods are running."
+      break
+    else
+      echo "$NOT_READY_PODS MetalLB pods are not ready yet. Waiting..."
+      RETRY=$((RETRY + 1))
+      sleep 5
+    fi
+  done
+
+  if [ "$NOT_READY_PODS" -ne 0 ]; then
+    echo "Failed to get all MetalLB pods running after $MAX_RETRIES attempts."
+    exit 1
+  fi
 
 else
   echo "MetalLB is already installed."
 fi
 
-# Wait for the webhook service to be ready
-echo "Waiting for MetalLB webhook service to be ready..."
-kubectl wait --namespace metallb-system \
-  --for=condition=ready pod \
-  --selector=component=webhook \
-  --timeout=90s
-
 # Check if the IPAddressPool already exists
 if ! kubectl get ipaddresspool -n metallb-system default &>/dev/null; then
   echo "Creating MetalLB IPAddressPool..."