readme: document nixos-anywhere install for builder

Readme.org

@@ -36,3 +36,11 @@ ssh-to-age < /etc/ssh/ssh_host_ed25519_key.pub
 #+begin_src sh
 sops updatekeys secrets/*
 #+end_src
+
+* Installation of builder
+- Start recent nixos installer in VM
+- Set password for root
+  
+#+begin_src sh
+nix run github:nix-community/nixos-anywhere -- --flake .#builder --target-host root@<ip>
+#+end_src

features-nixos/optional/hydra.nix

@@ -0,0 +1,49 @@
+# Host hydra
+{...}: {
+  services.hydra = {
+    enable = true;
+    hydraURL = "http://hydra.julian-mutter.de"; # externally visible URL
+    port = 3000;
+    notificationSender = "hydra@julian-mutter.de"; # e-mail of hydra service
+    # a standalone hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
+    # buildMachinesFiles = [ ];
+    # you will probably also want, otherwise *everything* will be built from scratch
+    useSubstitutes = true;
+
+    minimumDiskFree = 5; # in GB
+    minimumDiskFreeEvaluator = 4; # in GB
+  };
+
+  # Uris allowed as flake inputs, otherwise hydra does not fetch them
+  nix.settings.allowed-uris = [
+    "github:"
+    "gitlab:"
+    "git+https://github.com/hyprwm/Hyprland"
+    "https://github.com/hyprwm/Hyprland"
+    "https://github"
+    "https://gitlab"
+    "https://gitlab.julian-mutter.de"
+    "git+https://gitlab.julian-mutter.de"
+  ];
+
+  services.nginx = {
+    enable = true;
+    recommendedProxySettings = true;
+    # recommendedTlsSettings = true;
+    # other Nginx options
+    virtualHosts."hydra.julian-mutter.de" = {
+      # enableACME = true;
+      # forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:3000";
+        # proxyWebsockets = true; # needed if you need to use WebSocket
+        # extraConfig =
+        #   # required when the target is also TLS server with multiple hosts
+        #   "proxy_ssl_server_name on;" +
+        #   # required when the server wants to use HTTP Authentication
+        #   "proxy_pass_header Authorization;"
+        #   ;
+      };
+    };
+  };
+}

features-nixos/optional/jenkins-agent.nix

@@ -0,0 +1,52 @@
+# Setup the device as a jenkins agent
+{pkgs, ...}: {
+  services.openssh = {
+    enable = true;
+    # require public key authentication for better security
+    settings.PasswordAuthentication = false;
+    settings.KbdInteractiveAuthentication = false;
+    settings.PermitRootLogin = "yes";
+    # Add older algorithms for jenkins ssh-agents-plugin to be compatible
+    settings.Macs = [
+      "hmac-sha2-512-etm@openssh.com"
+      "hmac-sha2-256-etm@openssh.com"
+      "umac-128-etm@openssh.com"
+      "hmac-sha2-512"
+      "hmac-sha2-256"
+      "umac-128@openssh.com"
+    ];
+    settings.KexAlgorithms = [
+      "diffie-hellman-group-exchange-sha1"
+      "diffie-hellman-group14-sha1"
+      "mlkem768x25519-sha256"
+      "sntrup761x25519-sha512"
+      "sntrup761x25519-sha512@openssh.com"
+      "curve25519-sha256"
+      "curve25519-sha256@libssh.org"
+      "diffie-hellman-group-exchange-sha256"
+    ];
+  };
+
+  users.users.jenkins = {
+    createHome = true;
+    home = "/var/lib/jenkins";
+    group = "jenkins";
+    isNormalUser = true;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ36sQhVz3kUEi8754G7r3rboihhG4iqFK/UvQm6SING jenkins@home"
+    ];
+    packages = with pkgs; [
+      git
+      devenv
+    ];
+    extraGroups = [
+      "docker"
+    ];
+  };
+
+  users.groups.jenkins = {};
+  programs.java = {
+    enable = true;
+    package = pkgs.jdk21; # Same as jenkins version on home
+  };
+}

hosts/builder/default.nix

@@ -4,18 +4,37 @@
 {
   pwd,
   config,
-  pkgs,
+  inputs,
+  outputs,
+  lib,
   ...
 }: {
-  imports = [
+  imports =
-    ./hardware-configuration.nix
+    [
+      inputs.disko.nixosModules.disko
 
-    "${pwd}/features-nixos/global/fish.nix" # fish for admin
+      ./disko.nix
-    "${pwd}/features-nixos/global/locale.nix"
+      ./hardware-configuration.nix
-    "${pwd}/features-nixos/global/nix.nix"
+
-    "${pwd}/features-nixos/global/sops.nix"
+      "${pwd}/features-nixos/global/fish.nix" # fish for admin
-    "${pwd}/features-nixos/global/root.nix"
+      "${pwd}/features-nixos/global/locale.nix"
-  ];
+      "${pwd}/features-nixos/global/nix.nix"
+      "${pwd}/features-nixos/global/sops.nix"
+      "${pwd}/features-nixos/global/root.nix"
+
+      # "${pwd}/features-nixos/optional/hydra.nix"
+      # "${pwd}/features-nixos/optional/jenkins-agent.nix"
+    ]
+    ++ (builtins.attrValues outputs.nixosModules);
+
+  frajul.gitlab-runner = {
+    enable = false;
+    secretsFile = ./secrets.yaml;
+  };
+  frajul.gitea-runner = {
+    enable = false;
+    secretsFile = ./secrets.yaml;
+  };
 
   networking.hostName = "builder";
   system.stateVersion = "23.11";
@@ -66,22 +85,22 @@
     fallback = true;
   };
 
-  # system.autoUpgrade = {
+  system.autoUpgrade = {
-  #   enable = true;
+    enable = true;
-  #   flake = "git+https://gitlab.julian-mutter.de/julian/dotfiles";
+    flake = "git+https://gitlab.julian-mutter.de/julian/dotfiles";
-  #   flags = [
+    flags = [
-  #     "--recreate-lock-file" # update lock file
+      "--recreate-lock-file" # update lock file
-  #   ];
+    ];
-  #   dates = "02:13";
+    dates = "02:13";
-  # };
+  };
 
   # optimize store by hardlinking store files
-  nix.optimise.automatic = true;
+  nix.optimise.automatic = lib.mkForce true;
-  nix.optimise.dates = ["03:15"];
+  nix.optimise.dates = lib.mkForce ["03:15"];
 
-  # nix.gc.automatic = true;
+  nix.gc.automatic = lib.mkForce true;
-  # nix.gc.dates = "daily";
+  nix.gc.dates = lib.mkForce "daily";
-  # nix.gc.options = "--delete-old";
+  nix.gc.options = lib.mkForce "--delete-old";
 
   # nix.settings.keep-derivations = false;
   # nix.settings.keep-outputs = true;
@@ -101,55 +120,12 @@
     OOMScoreAdjust = 500;
   };
 
-  # Ollama used by open-webui as llm backend
-  # services.ollama = {
-  #   enable = true;
-  #   # acceleration = "rocm";
-  #   openFirewall = true;
-  # };
-
-  # services.nextjs-ollama-llm-ui = {
-  #   enable = true;
-  #   hostname = "192.168.3.118";
-  #   port = 3001;
-  # };
-  # services.open-webui = {
-  #   enable = true;
-  #   port = 8080;
-  #   openFirewall = true;
-  #   host = "builder.julian-mutter.de";
-  # };
-
-  networking.firewall.allowedTCPPorts = [
-    80
-    3001 # ollama-ui
-  ];
-
   services.openssh = {
     enable = true;
     # require public key authentication for better security
     settings.PasswordAuthentication = false;
     settings.KbdInteractiveAuthentication = false;
     settings.PermitRootLogin = "yes";
-    # Add older algorithms for jenkins ssh-agents-plugin to be compatible
-    settings.Macs = [
-      "hmac-sha2-512-etm@openssh.com"
-      "hmac-sha2-256-etm@openssh.com"
-      "umac-128-etm@openssh.com"
-      "hmac-sha2-512"
-      "hmac-sha2-256"
-      "umac-128@openssh.com"
-    ];
-    settings.KexAlgorithms = [
-      "diffie-hellman-group-exchange-sha1"
-      "diffie-hellman-group14-sha1"
-      "mlkem768x25519-sha256"
-      "sntrup761x25519-sha512"
-      "sntrup761x25519-sha512@openssh.com"
-      "curve25519-sha256"
-      "curve25519-sha256@libssh.org"
-      "diffie-hellman-group-exchange-sha256"
-    ];
   };
   users.users."root".openssh.authorizedKeys.keys = [
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjSZYdoF/51F+ykcBAYVCzCPTF5EEigWBL1APiR0h+H julian@aspi"
@@ -163,20 +139,6 @@
 
   # security.pam.sshAgentAuth.enable = true; # enable sudo via ssh
 
-  services.hydra = {
-    enable = true;
-    hydraURL = "http://hydra.julian-mutter.de"; # externally visible URL
-    port = 3000;
-    notificationSender = "hydra@julian-mutter.de"; # e-mail of hydra service
-    # a standalone hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
-    # buildMachinesFiles = [ ];
-    # you will probably also want, otherwise *everything* will be built from scratch
-    useSubstitutes = true;
-
-    minimumDiskFree = 5; # in GB
-    minimumDiskFreeEvaluator = 4; # in GB
-  };
-
   # add builder itpwd as build machine so system emulation is properly supported
   # nix.distributedBuilds = true;
   nix.buildMachines = [
@@ -199,93 +161,25 @@
     }
   ];
 
-  # Uris allowed as flake inputs, otherwise hydra does not fetch them
+  networking.firewall.allowedTCPPorts = [
-  nix.settings.allowed-uris = [
+    80
-    "github:"
-    "gitlab:"
-    "git+https://github.com/hyprwm/Hyprland"
-    "https://github.com/hyprwm/Hyprland"
-    "https://github"
-    "https://gitlab"
-    "https://gitlab.julian-mutter.de"
-    "git+https://gitlab.julian-mutter.de"
   ];
-
   services.nginx = {
     enable = true;
     recommendedProxySettings = true;
-    # recommendedTlsSettings = true;
-    # other Nginx options
-    virtualHosts."hydra.julian-mutter.de" = {
-      # enableACME = true;
-      # forceSSL = true;
-      locations."/" = {
-        proxyPass = "http://127.0.0.1:3000";
-        # proxyWebsockets = true; # needed if you need to use WebSocket
-        # extraConfig =
-        #   # required when the target is also TLS server with multiple hosts
-        #   "proxy_ssl_server_name on;" +
-        #   # required when the server wants to use HTTP Authentication
-        #   "proxy_pass_header Authorization;"
-        #   ;
-      };
-    };
 
+    # nix-serve
     virtualHosts."binarycache.julian-mutter.de" = {
       locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
     };
 
+    # attic
     clientMaxBodySize = "2G";
     virtualHosts."cache.julian-mutter.de" = {
       locations."/".proxyPass = "http://127.0.0.1:8080";
     };
   };
 
-  # =========== Gitea actions ==========
-  services.gitea-actions-runner.instances."builder" = {
-    enable = true;
-    url = "https://gitlab.julian-mutter.de";
-    name = "builder";
-    tokenFile = config.sops.secrets."gitea_token".path;
-    labels = [
-      # fake the ubuntu name, because node provides no ubuntu builds
-      "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest"
-      # my custom nix+devenv ci container
-      "nix-ci:docker://gitlab.julian-mutter.de/julian/nix-ci-container:latest"
-      # devenv
-      "devenv:docker://ghcr.io/cachix/devenv/devenv:latest"
-      # provide native execution on the host
-      "nixos:host"
-    ];
-    # Packages are intjected into PATH for "nixos:host"
-    hostPackages = with pkgs; [
-      bash
-      coreutils
-      curl
-      gawk
-      gitMinimal
-      nodejs # Required by many standard actions (like actions/checkout)
-      docker
-      devenv
-      wget
-      nix
-    ];
-  };
-
-  virtualisation.docker.enable = true;
-
-  # TODO: podman fails with: "cannot resolve hostname"
-  # virtualisation.podman = {
-  #   enable = true;
-  #   dockerCompat = true;
-  #   defaultNetwork.settings.dns_enabled = true;
-  # };
-
-  sops.secrets."gitea_token" = {
-    owner = config.users.users.nix.name;
-    sopsFile = ./secrets.yaml;
-  };
-
   # =========== Binary Cache ==========
   services.nix-serve = {
     enable = true;
@@ -327,41 +221,4 @@
       };
     };
   };
-
-  services.gitlab-runner.enable = true;
-  # runner for everything else
-  #
-  sops.secrets."gitlab_runner_token".sopsFile = ./secrets.yaml;
-  services.gitlab-runner.services.default = {
-    # File should contain at least these two variables:
-    authenticationTokenConfigFile = config.sops.secrets."gitlab_runner_token".path;
-    dockerImage = "alpine:latest";
-    dockerVolumes = [
-      "/var/run/docker.sock:/var/run/docker.sock"
-    ];
-  };
-
-  ### Jenkins node
-  users.users.jenkins = {
-    createHome = true;
-    home = "/var/lib/jenkins";
-    group = "jenkins";
-    isNormalUser = true;
-    openssh.authorizedKeys.keys = [
-      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJ36sQhVz3kUEi8754G7r3rboihhG4iqFK/UvQm6SING jenkins@home"
-    ];
-    packages = with pkgs; [
-      git
-      devenv
-    ];
-    extraGroups = [
-      "docker"
-    ];
-  };
-
-  users.groups.jenkins = {};
-  programs.java = {
-    enable = true;
-    package = pkgs.jdk21; # Same as jenkins version on home
-  };
 }

hosts/builder/disko.nix

@@ -0,0 +1,56 @@
+# Example to create a bios compatible gpt partition
+{ lib, ... }:
+{
+  disko.devices = {
+    disk.disk1 = {
+      device = lib.mkDefault "/dev/sda";
+      type = "disk";
+      content = {
+        type = "gpt";
+        partitions = {
+          boot = {
+            name = "boot";
+            size = "1M";
+            type = "EF02";
+          };
+          esp = {
+            name = "ESP";
+            size = "500M";
+            type = "EF00";
+            content = {
+              type = "filesystem";
+              format = "vfat";
+              mountpoint = "/boot";
+            };
+          };
+          root = {
+            name = "root";
+            size = "100%";
+            content = {
+              type = "lvm_pv";
+              vg = "pool";
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      pool = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "100%FREE";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [
+                "defaults"
+              ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/builder/hardware-configuration.nix

@@ -7,43 +7,13 @@
     "sd_mod"
     "sr_mod"
   ];
-  # boot.initrd.kernelModules = [ "amdgpu" ]; # GPU support
   boot.kernelModules = [];
   boot.extraModulePackages = [];
 
-  fileSystems."/" = {
-    device = "/dev/disk/by-uuid/f088fe8e-bf3d-4a89-98bd-ead9852d381f";
-    fsType = "ext4";
-  };
-
-  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
-  # (the default) this is the recommended approach. When using systemd-networkd it's
-  # still possible to use this option, but it's recommended to use it in conjunction
-  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
   networking.useDHCP = lib.mkDefault true;
-  # networking.interfaces.ens18.useDHCP = lib.mkDefault true;
-
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
 
-  # hardware.graphics = {
-  #   enable = true;
-  #   extraPackages = with pkgs; [
-  #     rocmPackages.clr.icd
-  #     linuxPackages.amdgpu-pro
-  #   ];
-  # };
-
-  # boot.kernelParams = [
-  #   "radeon.si_support=0"
-  #   "radeon.cik_support=1"
-  #   "amdgpu.si_support=0"
-  #   "amdgpu.cik_support=1"
-  # ];
-  # boot.extraModulePackages = with config.boot.kernelPackages; [ amdgpu-pro ];
-  # boot.blacklistedKernelModules = [ "radeon" ];
-
   boot.loader.grub.enable = true;
-  boot.loader.grub.device = "/dev/sda";
 
   # Emulated systems used as alternative to cross-compiling
   boot.binfmt.emulatedSystems = ["aarch64-linux"];

hosts/kardorf/default.nix

@@ -11,7 +11,7 @@
     "${pwd}/features-nixos/users/wolfi"
     "${pwd}/features-nixos/optional/binarycaches.nix"
 
-    "${pwd}/features-nixos/optional/remote-builder.nix"
+    # "${pwd}/features-nixos/optional/remote-builder.nix"
     "${pwd}/features-nixos/optional/boot-efi.nix"
 
     "${pwd}/features-nixos/optional/greetd.nix"

modules/nixos/default.nix

@@ -3,4 +3,6 @@
   syncthing = import ./syncthing.nix;
   frajulAutoUpgrade = import ./frajul-auto-upgrade.nix;
   pianoLEDVisualizer = import ./piano-led-visualizer.nix;
+  gitea-runner = import ./gitea-runner.nix;
+  gitlab-runner = import ./gitlab-runner.nix;
 }

modules/nixos/gitea-runner.nix

@@ -0,0 +1,57 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  cfg = config.frajul.gitea-runner;
+in {
+  options = {
+    frajul.gitea-runner = {
+      enable = lib.mkEnableOption "gitea-runner";
+      secretsFile = lib.mkOption {
+        type = lib.types.path;
+        description = "A sops encrpyted file containing a 'gitea_token' secret";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    virtualisation.docker.enable = true;
+
+    sops.secrets."gitea_token" = {
+      owner = config.users.users.nix.name;
+      sopsFile = cfg.secretsFile;
+    };
+
+    services.gitea-actions-runner.instances."builder" = {
+      enable = true;
+      url = "https://gitlab.julian-mutter.de";
+      name = "builder";
+      tokenFile = config.sops.secrets."gitea_token".path;
+      labels = [
+        # fake the ubuntu name, because node provides no ubuntu builds
+        "ubuntu-latest:docker://docker.gitea.com/runner-images:ubuntu-latest"
+        # my custom nix+devenv ci container
+        "nix-ci:docker://gitlab.julian-mutter.de/julian/nix-ci-container:latest"
+        # devenv
+        "devenv:docker://ghcr.io/cachix/devenv/devenv:latest"
+        # provide native execution on the host
+        "nixos:host"
+      ];
+      # Packages are intjected into PATH for "nixos:host"
+      hostPackages = with pkgs; [
+        bash
+        coreutils
+        curl
+        gawk
+        gitMinimal
+        nodejs # Required by many standard actions (like actions/checkout)
+        docker
+        devenv
+        wget
+        nix
+      ];
+    };
+  };
+}

modules/nixos/gitlab-runner.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  cfg = config.frajul.gitlab-runner;
+in {
+  options = {
+    frajul.gitlab-runner = {
+      enable = lib.mkEnableOption "gitlab-runner";
+      secretsFile = lib.mkOption {
+        type = lib.types.path;
+        description = "A sops encrpyted file containing a 'gitlab_runner_token' secret";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.gitlab-runner.enable = true;
+
+    sops.secrets."gitlab_runner_token".sopsFile = cfg.secretsFile;
+    services.gitlab-runner.services.default = {
+      # File should contain at least these two variables:
+      authenticationTokenConfigFile = config.sops.secrets."gitlab_runner_token".path;
+      dockerImage = "alpine:latest";
+      dockerVolumes = [
+        "/var/run/docker.sock:/var/run/docker.sock"
+      ];
+    };
+  };
+}