From 2804824444e68e65c525d9adf211abfd6bb049f3 Mon Sep 17 00:00:00 2001 From: jon brookes Date: Fri, 3 Oct 2025 18:09:26 +0100 Subject: [PATCH] Add comprehensive guides for gcloud setups; include instructions for cluster creation, smoke testing, and infrastructure management. --- .../guides/{k3d => 1. k3d}/003-local-k3d.md | 0 .../{k3d => 1. k3d}/004-infctl-first-run.md | 0 .../009-development-releases.md | 0 .../005-local-vagrant-cluster.md | 0 .../006-local-vagrant-smoke-test.md | 0 .../010-local-vagrant-cluster-storage.md | 0 .../015-local-vagrant-cluster-ingress.md | 0 .../{vagrant => 2. vagrant}/020-security.md | 0 .../docs/guides/3. gcloud/005-gcloud-intro.md | 90 +++++++++++++ .../3. gcloud/020-create-gcloud-infra.md | 95 ++++++++++++++ .../3. gcloud/030-smoke-test-cluster.md | 57 +++++++++ .../guides/3. gcloud/040-install-forgejo.md | 118 ++++++++++++++++++ .../guides/3. gcloud/050-clean-up-infra.md | 43 +++++++ 13 files changed, 403 insertions(+) rename src/content/docs/guides/{k3d => 1. k3d}/003-local-k3d.md (100%) rename src/content/docs/guides/{k3d => 1. k3d}/004-infctl-first-run.md (100%) rename src/content/docs/guides/{k3d => 1. k3d}/009-development-releases.md (100%) rename src/content/docs/guides/{vagrant => 2. vagrant}/005-local-vagrant-cluster.md (100%) rename src/content/docs/guides/{vagrant => 2. vagrant}/006-local-vagrant-smoke-test.md (100%) rename src/content/docs/guides/{vagrant => 2. vagrant}/010-local-vagrant-cluster-storage.md (100%) rename src/content/docs/guides/{vagrant => 2. vagrant}/015-local-vagrant-cluster-ingress.md (100%) rename src/content/docs/guides/{vagrant => 2. vagrant}/020-security.md (100%) create mode 100644 src/content/docs/guides/3. gcloud/005-gcloud-intro.md create mode 100644 src/content/docs/guides/3. gcloud/020-create-gcloud-infra.md create mode 100644 src/content/docs/guides/3. gcloud/030-smoke-test-cluster.md create mode 100644 src/content/docs/guides/3. gcloud/040-install-forgejo.md create mode 100644 src/content/docs/guides/3. gcloud/050-clean-up-infra.md diff --git a/src/content/docs/guides/k3d/003-local-k3d.md b/src/content/docs/guides/1. k3d/003-local-k3d.md similarity index 100% rename from src/content/docs/guides/k3d/003-local-k3d.md rename to src/content/docs/guides/1. k3d/003-local-k3d.md diff --git a/src/content/docs/guides/k3d/004-infctl-first-run.md b/src/content/docs/guides/1. k3d/004-infctl-first-run.md similarity index 100% rename from src/content/docs/guides/k3d/004-infctl-first-run.md rename to src/content/docs/guides/1. k3d/004-infctl-first-run.md diff --git a/src/content/docs/guides/k3d/009-development-releases.md b/src/content/docs/guides/1. k3d/009-development-releases.md similarity index 100% rename from src/content/docs/guides/k3d/009-development-releases.md rename to src/content/docs/guides/1. k3d/009-development-releases.md diff --git a/src/content/docs/guides/vagrant/005-local-vagrant-cluster.md b/src/content/docs/guides/2. vagrant/005-local-vagrant-cluster.md similarity index 100% rename from src/content/docs/guides/vagrant/005-local-vagrant-cluster.md rename to src/content/docs/guides/2. vagrant/005-local-vagrant-cluster.md diff --git a/src/content/docs/guides/vagrant/006-local-vagrant-smoke-test.md b/src/content/docs/guides/2. vagrant/006-local-vagrant-smoke-test.md similarity index 100% rename from src/content/docs/guides/vagrant/006-local-vagrant-smoke-test.md rename to src/content/docs/guides/2. vagrant/006-local-vagrant-smoke-test.md diff --git a/src/content/docs/guides/vagrant/010-local-vagrant-cluster-storage.md b/src/content/docs/guides/2. vagrant/010-local-vagrant-cluster-storage.md similarity index 100% rename from src/content/docs/guides/vagrant/010-local-vagrant-cluster-storage.md rename to src/content/docs/guides/2. vagrant/010-local-vagrant-cluster-storage.md diff --git a/src/content/docs/guides/vagrant/015-local-vagrant-cluster-ingress.md b/src/content/docs/guides/2. vagrant/015-local-vagrant-cluster-ingress.md similarity index 100% rename from src/content/docs/guides/vagrant/015-local-vagrant-cluster-ingress.md rename to src/content/docs/guides/2. vagrant/015-local-vagrant-cluster-ingress.md diff --git a/src/content/docs/guides/vagrant/020-security.md b/src/content/docs/guides/2. vagrant/020-security.md similarity index 100% rename from src/content/docs/guides/vagrant/020-security.md rename to src/content/docs/guides/2. vagrant/020-security.md diff --git a/src/content/docs/guides/3. gcloud/005-gcloud-intro.md b/src/content/docs/guides/3. gcloud/005-gcloud-intro.md new file mode 100644 index 0000000..aa9549c --- /dev/null +++ b/src/content/docs/guides/3. gcloud/005-gcloud-intro.md @@ -0,0 +1,90 @@ +--- +title: Google Cloud MVK +description: A guide to creating a virtualized Google Cloud k3s instance. +--- + +###### Use case :badge[development]{variant=success} :badge[pre-production]{variant=caution} + +# Introduction + +Google Cloud can be used to host a lab that can be cheaper to run if the hardware you use for your day to day is not sufficiently capable and can also offer an alternative that is close or identical to production. + +The approach adopted by MVK strongly supports the use of *infrastructure as code*. A key benefit of this being its ability to build and more imortantly *rebuild* infrastructure from scratch. + +This means that you can **destroy** infrastructure when it is not needed. + +Fort test, development and validation work, this has many benefits. + +* Reduced costs - your not paying for hosting when you don't need it + +* Increased confidence - in your infrastructure being recoverable in the event of a disaster + +* Better testing - of both your infrastructure and your applications as each test cycle is guaranteed to have a 'clean', consistent and predictable environment. + +# Prerequisites + +If you have not [already installed `infctl`](/infctl/quick-start/) please do so. + +If your a Windows user, be sure to check [Notes for Windows Users](https://mvk.headshed.dev/guides/001-dev-env/#note-for-windows-users) to ensure you have [Git and Bash](https://git-scm.com/downloads) installed. + +Suffice to say, you will need to run the following in a bash capable console: + +```bash +curl -L https://codeberg.org/headshed/infctl-cli/raw/branch/main/install.sh | bash +``` + +## Install OpenTofu + +We will build Google Cloud infrastructure with Terrarorm and using OpenTofu which you can find download and installation instructions for all major platforms here: + +https://opentofu.org/docs/intro/install + +## Sign Up for Google Cloud Console + +To create and manage Google Cloud resources, you need a Google Cloud account. + +1. Visit the [Google Cloud Console](https://console.cloud.google.com/). +2. Sign in with your Google account or create one if you don't have one yet. +3. Follow the prompts to set up billing (a free trial may be available). +4. Once signed in, you can create a new project by clicking on the project dropdown at the top and selecting "New Project". +5. Name your project and click "Create". + +You will use this project to build and manage your cloud resources. + +## Install the Google Cloud CLI + +To interact with Google Cloud resources, you need to install the `gcloud` command line interface. + +`gcloud` is available for all major platforms, be they Windows, Linux or Mac. + +You can find an installation and download for each at : + +https://cloud.google.com/sdk/docs/install + +After installation, open a new Command Prompt and initialize the CLI: + +```cmd +gcloud init +``` + +Refer to the [official documentation](https://cloud.google.com/sdk/docs/install#windows) for troubleshooting and advanced options. + + +## Sign up for Hosted DNS + +To access your lab in Gcloud, you need to buy or already have a domain name and the ability to manage where its 'A records' point to. + +You can use any DNS provider of your choosing that can do this for you but here is a quick guide to sign up for a free Cloudflare account where you can both buy and manage existing or new domain names. + +Cloudflare offers free DNS management and security features for your domains. + +1. Go to [Cloudflare Sign Up](https://dash.cloudflare.com/sign-up). +2. Enter your email address and create a password. +3. Click "Create Account". +4. After logging in if you already have a domain name, click "Add a Site" and enter your existing domain name. +5. Select the free plan and continue. +6. Cloudflare will scan your DNS records; review and confirm them and give instructions to complete the process. + +Once sign up is complete, you can buy or import other Domain names and manage their records and settings from the Cloudflare dashboard. + +Next we will run a pipeline to create a single node development cluster running k3s in gcloud. \ No newline at end of file diff --git a/src/content/docs/guides/3. gcloud/020-create-gcloud-infra.md b/src/content/docs/guides/3. gcloud/020-create-gcloud-infra.md new file mode 100644 index 0000000..96c314c --- /dev/null +++ b/src/content/docs/guides/3. gcloud/020-create-gcloud-infra.md @@ -0,0 +1,95 @@ +--- +title: Create gcloud infrastructure +description: A guide to creating gcloud infrastructure. +--- + +Clone the `infctl` repo if you have not yet done so. We will work on the assumption that we are working in a home directory called `projects`. + +```bash +cd ~/projects + +if [ ! -d "infctl-cli" ]; then + git clone https://codeberg.org/headshed/infctl-cli + cd infctl-cli +else + cd infctl-cli +fi +``` + +we need to configure some environment variables to let our build know some things about our project. + +```bash +cp .env.gcloud-example .env +``` + +edit our newly created `.env` file and give it values that are appropriate to our gcloud account, for example: + +``` +PROJECT_NAME="my-very-own-dev-lab" +EMAIL="your.email@mailsomewhere.com" +APP_DOMAIN_NAME="atestdr.yourdomain.com" +``` + +where each of these variables represent: +* your project name, often called the project id in gcloud +* your email that you wish to be identified with for DNS and TLS encryption +* the domain name you want to use for this project when it is on line + +Activate these in the current shell with: + +```bash +source .env +``` + +Before we go any further, we need to confirm we have our gcloud environment ready and configured: + +```bash +gcloud compute instances list --project="$PROJECT_NAME" \ +&& gcloud compute disks list --project="$PROJECT_NAME" \ +&& gcloud compute firewall-rules list --project="$PROJECT_NAME" \ +&& gcloud storage buckets list --project="$PROJECT_NAME" +``` + +We should see pretty much an empty list but for default rules assigned for us by Google to our project to accept SSH, RDP and ICMP traffic - 4 lines in all. + +To build the infrastructure : + +```bash +LOGFORMAT=none infctl -f gcloud/tf/scripts/build-gcloud-k3s-pipeline.json +``` +You should see a successful build ending with something like: + +```bash +Apply complete! Resources: 0 added, 0 changed, 0 destroyed. + +Outputs: +k3s_vm_public_ip = "xxx.xxx.xxx.xxx" +project_number = "..233434.." +✅ Step completed: run tofu +✅ 🚀 Pipeline completed successfully +``` + +Take a note of public IP address assigned to your VM as in `xxx.xxx.xxx.xxx` in the above example. + +You need to create an `A record` in your DNS console to point to this address, using the environment name you set for your app earlier + +```bash +echo $APP_DOMAIN_NAME + +``` + +### Setting an A Record in your chosen or Cloudflare DNS + +If you are using Cloudflare DNS, you can follow the below or use as a guide for your chosen DNS console... + +1. Log in to your Cloudflare dashboard. +2. Select your domain. +3. Go to the **DNS** tab. +4. Click **Add record**. +5. Choose **A** as the record type. +6. Enter your subdomain (e.g., `atestdr`) in the **Name** field. +7. Enter your VM's public IP address in the **IPv4 address** field. +8. Set the record to be DNS only and a low TTL, for example 5 min. +9. Click **Save**. + +Your domain should now point to your VM's public IP. \ No newline at end of file diff --git a/src/content/docs/guides/3. gcloud/030-smoke-test-cluster.md b/src/content/docs/guides/3. gcloud/030-smoke-test-cluster.md new file mode 100644 index 0000000..0d9e9bd --- /dev/null +++ b/src/content/docs/guides/3. gcloud/030-smoke-test-cluster.md @@ -0,0 +1,57 @@ +--- +title: Smoke test the cluster +description: A guide to smoke testing our gcloud infrastructure. +--- + +# Smoke test the cluster + +from our git repo directory that we checked out in [create gcloud earlier](/guides/3-gcloud/020-create-gcloud-infra/) + + +```bash +source .env +gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project=$PROJECT_NAME +``` + +We should be shelled into our vm. + +```bash +user@k3s-vm-1:~$ k version +Client Version: v1.32.8+k3s1 +Kustomize Version: v5.5.0 +Server Version: v1.32.8+k3s1 +``` + +Kubectl is giving us a version + +```bash +user@k3s-vm-1:~$ kubectl get nodes +NAME STATUS ROLES AGE VERSION +k3s-vm-1 Ready control-plane,etcd,master 7m41s v1.32.8+k3s1 +``` + +`kubectl` which is also aliased to `k` ( we can use either ) is telling us we have a single node k3s node. + +```bash +k get events -A +.... +kube-system 8m53s Normal ApplyingManifest addon/runtimes Applying manifest at "/var/lib/rancher/k3s/server/manifests/runtimes.yaml" +kube-system 8m53s Normal AppliedManifest addon/runtimes Applied manifest at "/var/lib/rancher/k3s/server/manifests/runtimes.yaml" +kube-system 8m52s Normal DeletingManifest addon/traefik Deleting manifest at "/var/lib/rancher/k3s/server/manifests/traefik.yaml" +``` + +There should be a bunch of events - the cluster just started up, and no alarming warnings or errors are present. + +We should have pods running: + +```bash +user@k3s-vm-1:~$ k get pods -A +NAMESPACE NAME READY STATUS RESTARTS AGE +kube-system coredns-64fd4b4794-gt5mp 1/1 Running 0 11m +kube-system local-path-provisioner-774c6665dc-h5xws 1/1 Running 0 11m +kube-system metrics-server-7bfffcd44-tfwnw 1/1 Running 0 11m +``` + +Not many, this is true but k3s is lightweight. This is good. + +Ok, so we have a functioning single node k3s, next we can install an application that can use our Domain name we configured earlier, using [forgejo](https://forgejo.org/), a code forge application, much like github, gitlab but self-hosted and efficient. \ No newline at end of file diff --git a/src/content/docs/guides/3. gcloud/040-install-forgejo.md b/src/content/docs/guides/3. gcloud/040-install-forgejo.md new file mode 100644 index 0000000..c8aaebd --- /dev/null +++ b/src/content/docs/guides/3. gcloud/040-install-forgejo.md @@ -0,0 +1,118 @@ +--- +title: Install Forgejo +description: A guide to installing forgejo to our cluster. +--- + +From our host system, in another shell and from the location of the `infctl-cli` git repo, we need to copy our `.env` file to the gcloud vm: + +```bash +gcloud compute scp .env k3s-vm-1:/opt/src/infctl-cli/.env --zone=us-central1-a --project=$PROJECT_NAME +``` + +From a shell into our cluster from [earlier](/guides/3-gcloud/030-smoke-test-cluster/) or by reconnecting with + +```bash +gcloud compute ssh k3s-vm-1 --zone=us-central1-a --project=$PROJECT_NAME +``` + +Change directory to `/opt/src/infctl-cli/` and `source` the '.env' file that was just copied + +```bash +cd /opt/src/infctl-cli/ +source .env +``` + +From here we can run a pipeline to install foregejo: + +```bash +LOG_FORMAT=none infctl -f gcloud/tf/scripts/install-forgejo-pipeline.json +``` + +if the pipeline completes successfully we can test to see if the site is responding with + +```bash +curl -k $APP_DOMAIN_NAME +``` + +we should see an HTML page returned to us from foregejo. + +This should work also from our host or anywhere that is internet connected. + +NB: the use of `-k` in the curl request: this is to tell `curl` to ignore TLS certificates that are not fully signed. + +In the manifests used to install forgejo, this was intentionally set to use a staging service with LetsEncrypt: + +```bash +user@k3s-vm-1:/opt/src/infctl-cli$ cat gcloud/tf/k3s/forgejo/issuer.yaml +``` +... + +```yaml +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: le-cluster-issuer-http + namespace: forgejo +spec: + acme: + email: marshyon@gmail.com + # We use the staging server here for testing to avoid throttling. + server: https://acme-staging-v02.api.letsencrypt.org/directory + # server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: http-issuer-account-key + solvers: + - http01: + ingress: + class: traefik +``` + +you can see that this has worked also by running + +```bash +printf "Q" | openssl s_client -connect "$APP_DOMAIN_NAME":443 -servername "$APP_DOMAIN_NAME" -showcerts +``` + +Look for the section that looks like this: + +``` +depth=1 C = US, O = (STAGING) Let's Encrypt, CN = (STAGING) Tenuous Tomato R13 +``` + +This certificate will not be trusted by Chrome or other modern browsers, so, to bypass this and to complete our tests, we can download a certificate root file from LetsEncrypt and install this into our browser. + +Step 1: Download the Staging Root Certificate + +The specific certificate you need is the "Let's Encrypt Staging Root X1". Here is the direct link to download the file from the official Let's Encrypt website: + +Direct Download Link: https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem + +download this letsencrypt-stg-root-x1.pem file and save it to your computer. + +Step 2: Import the Certificate into Your Browser + +For security reasons, it is highly recommended to do this in a new, separate browser profile that you only use for testing. + +For Google Chrome: + +1. Create a New Test Profile: +1. Click your profile picture in the top-right of Chrome. +1. Click Add > Continue without an account. +1. Give the profile a name like "Staging Test" and click Done. A new Chrome window will open using this profile. +1. Open Certificate Settings: +1. In this new profile window, go to Settings > Privacy and security > Security. +1. Scroll down and click on Manage device certificates. +1. Import the Certificate: +1. Click the Authorities tab. +1. Click the Import... button. +1. Select the letsencrypt-stg-root-x1.pem file you downloaded earlier. +1. In the pop-up window, check the box for "Trust this certificate for identifying websites." +1. Click OK. + + +Test: + +Close and reopen the "Staging Test" profile Chrome window. + +Navigate to your site. It should load correctly with a padlock icon, without any security warnings. + diff --git a/src/content/docs/guides/3. gcloud/050-clean-up-infra.md b/src/content/docs/guides/3. gcloud/050-clean-up-infra.md new file mode 100644 index 0000000..6174adc --- /dev/null +++ b/src/content/docs/guides/3. gcloud/050-clean-up-infra.md @@ -0,0 +1,43 @@ +--- +title: Shut down and clean up +description: A guide to destroying unwanted infrastructure +--- + +## Clean up (destroy) the infrastructure + +Now we no longer have use for the test infrastructure it is time to clear it down to save hosting costs. + +From our host machine and within the `infctl-cli` repository we have checked out: + +```bash +cd gcloud/tf +tofu destroy +``` + +this will ask us if we want to proceed: + +```bash +Plan: 0 to add, 0 to change, 5 to destroy. + +... + +Do you really want to destroy all resources? + OpenTofu will destroy all your managed infrastructure, as shown above. + There is no undo. Only 'yes' will be accepted to confirm. + + Enter a value: +``` + +Enter `yes` and press return. + +this should compete with + +```bash +...... +Google_compute_disk.app_data_disk: Destruction complete after 2s + +Destroy complete! Resources: 5 destroyed. +``` + +No further costs will be incurred on your Google Cloud account for this test infrastructure and you can rebuild whenever you need to run further tests or carry out disaster recovery rehearsals. +