dotfiles/hosts/builder/default.nix

# sudo nixos-rebuild switch --flake .#builder --target-host root@192.168.3.118
# or
# deploy .#builder
{
  config,
  pkgs,
  ...
}: {
  imports = [
    ./hardware-configuration.nix

    ../common/global
  ];

  # hardware.graphics = {
  #   enable = true;
  #   extraPackages = with pkgs; [
  #     rocmPackages.clr.icd
  #     linuxPackages.amdgpu-pro
  #   ];
  # };

  # boot.kernelParams = [
  #   "radeon.si_support=0"
  #   "radeon.cik_support=1"
  #   "amdgpu.si_support=0"
  #   "amdgpu.cik_support=1"
  # ];
  # boot.extraModulePackages = with config.boot.kernelPackages; [ amdgpu-pro ];
  # boot.blacklistedKernelModules = [ "radeon" ];

  boot.loader.grub.enable = true;
  boot.loader.grub.device = "/dev/sda";

  # Emulated systems used as alternative to cross-compiling
  boot.binfmt.emulatedSystems = ["aarch64-linux"];

  networking.hostName = "builder";

  modules = {
    keymap.enable = true;
    locales.enable = true;
    sops.enable = true;
  };

  users.users.nix = {
    isNormalUser = true;
    description = "Nix";
    extraGroups = [
      "networkmanager"
      "wheel"
    ];
  };

  nix.settings.experimental-features = [
    "nix-command"
    "flakes"
  ];

  # Setup binary caches
  nix.settings = {
    substituters = [
      "https://nix-community.cachix.org"
      "https://cache.nixos.org/"
    ];
    trusted-public-keys = ["nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs="];

    trusted-users = ["nix"];
    max-jobs = "auto";
    cores = 0;
  };

  # optimize store by hardlinking store files
  nix.optimise.automatic = true;
  nix.optimise.dates = ["03:15"];

  # nix.gc.automatic = true;
  # nix.gc.dates = "daily";
  # nix.gc.options = "--delete-old";

  # nix.settings.keep-derivations = false;
  # nix.settings.keep-outputs = true;

  # Garbage collect up to 100 GiB when only 20 GiB storage left
  nix.extraOptions = ''
    min-free = ${toString (20 * 1024 * 1024 * 1024)}
    max-free = ${toString (100 * 1024 * 1024 * 1024)}
  '';

  environment.systemPackages = with pkgs; [
    vim
    htop
    mc

    # # AMD GPU tools
    # pciutils
    # nvtopPackages.amd
    # linuxPackages.amdgpu-pro
  ];

  nix.nrBuildUsers = 64;

  # prevent memory to get filled
  systemd.services.nix-daemon.serviceConfig = {
    MemoryAccounting = true;
    MemoryMax = "90%";
    OOMScoreAdjust = 500;
  };

  # Ollama used by open-webui as llm backend
  # services.ollama = {
  #   enable = true;
  #   # acceleration = "rocm";
  # };
  # services.open-webui = {
  #   enable = true;
  #   port = 8080;
  #   openFirewall = true;
  #   host = "builder.julian-mutter.de";
  # };

  networking.firewall.allowedTCPPorts = [
    80
  ];

  services.openssh = {
    enable = true;
    # require public key authentication for better security
    settings.PasswordAuthentication = true;
    settings.KbdInteractiveAuthentication = false;
    settings.PermitRootLogin = "yes";
  };
  users.users."root".openssh.authorizedKeys.keys = [
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjSZYdoF/51F+ykcBAYVCzCPTF5EEigWBL1APiR0h+H julian@aspi"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGFcS+3d1tNgHmYCjueymCV9Bd2LcJcKGhVobrDe3r0s julian@kardorf"
  ];
  users.users."nix".openssh.authorizedKeys.keys = [
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFjSZYdoF/51F+ykcBAYVCzCPTF5EEigWBL1APiR0h+H julian@aspi"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAIQ+qMuXvyoxO1DuCR3/x+IQRfSA2WyMuzuotWZjCye root@aspi"
    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHnfLJnS2SKUs47J0qpLTkk0LQA5quOuAhnxE6yppUDm root@kardorf"
  ];

  # security.pam.sshAgentAuth.enable = true; # enable sudo via ssh

  services.hydra = {
    enable = true;
    hydraURL = "http://hydra.julian-mutter.de"; # externally visible URL
    port = 3000;
    notificationSender = "hydra@julian-mutter.de"; # e-mail of hydra service
    # a standalone hydra will require you to unset the buildMachinesFiles list to avoid using a nonexistant /etc/nix/machines
    # buildMachinesFiles = [ ];
    # you will probably also want, otherwise *everything* will be built from scratch
    useSubstitutes = true;

    minimumDiskFree = 5; # in GB
    minimumDiskFreeEvaluator = 4; # in GB
  };

  # add builder itself as build machine so system emulation is properly supported
  # nix.distributedBuilds = true;
  nix.buildMachines = [
    {
      hostName = "localhost";
      protocol = null;
      # sshUser = "nix";
      systems = [
        "x86_64-linux"
        "aarch64-linux"
      ];
      maxJobs = 4;
      speedFactor = 3;
      supportedFeatures = [
        "nixos-test"
        "benchmark"
        "big-parallel"
        "kvm"
      ];
    }
  ];

  # Uris allowed as flake inputs, otherwise hydra does not fetch them
  nix.settings.allowed-uris = [
    "github:"
    "gitlab:"
    "git+https://github.com/hyprwm/Hyprland"
    "https://github.com/hyprwm/Hyprland"
    "https://github"
    "https://gitlab"
    "https://gitlab.julian-mutter.de"
    "git+https://gitlab.julian-mutter.de"
  ];

  services.nginx = {
    enable = true;
    recommendedProxySettings = true;
    # recommendedTlsSettings = true;
    # other Nginx options
    virtualHosts."hydra.julian-mutter.de" = {
      # enableACME = true;
      # forceSSL = true;
      locations."/" = {
        proxyPass = "http://127.0.0.1:3000";
        # proxyWebsockets = true; # needed if you need to use WebSocket
        # extraConfig =
        #   # required when the target is also TLS server with multiple hosts
        #   "proxy_ssl_server_name on;" +
        #   # required when the server wants to use HTTP Authentication
        #   "proxy_pass_header Authorization;"
        #   ;
      };
    };

    virtualHosts."binarycache.julian-mutter.de" = {
      locations."/".proxyPass = "http://${config.services.nix-serve.bindAddress}:${toString config.services.nix-serve.port}";
    };

    clientMaxBodySize = "2G";
    virtualHosts."cache.julian-mutter.de" = {
      locations."/".proxyPass = "http://127.0.0.1:8080";
    };
  };

  # =========== Gitea actions ==========
  services.gitea-actions-runner.instances."builder" = {
    enable = true;
    url = "https://gitlab.julian-mutter.de";
    name = "builder";
    tokenFile = config.sops.secrets."gitea_token".path;
    labels = []; # use default labels
  };

  virtualisation.docker.enable = true;

  # TODO: podman fails with: "cannot resolve hostname"
  # virtualisation.podman = {
  #   enable = true;
  #   dockerCompat = true;
  #   defaultNetwork.settings.dns_enabled = true;
  # };

  sops.secrets."gitea_token" = {
    owner = config.users.users.nix.name;
    sopsFile = ../../../secrets/secrets-builder.yaml;
  };

  # =========== Binary Cache ==========
  services.nix-serve = {
    enable = true;
    secretKeyFile = "/var/cache-priv-key.pem";
  };

  # =========== Binary Cache with attic ==========
  sops.secrets."attic_token".sopsFile = ../../../secrets/secrets-builder.yaml;

  services.atticd = {
    enable = true;
    environmentFile = config.sops.secrets."attic_token".path;
    settings = {
      listen = "[::]:8080";

      jwt = {};

      # Data chunking
      #
      # Warning: If you change any of the values here, it will be
      # difficult to reuse existing chunks for newly-uploaded NARs
      # since the cutpoints will be different. As a result, the
      # deduplication ratio will suffer for a while after the change.
      chunking = {
        # The minimum NAR size to trigger chunking
        #
        # If 0, chunking is disabled entirely for newly-uploaded NARs.
        # If 1, all NARs are chunked.
        nar-size-threshold = 64 * 1024; # 64 KiB

        # The preferred minimum size of a chunk, in bytes
        min-size = 16 * 1024; # 16 KiB

        # The preferred average size of a chunk, in bytes
        avg-size = 64 * 1024; # 64 KiB

        # The preferred maximum size of a chunk, in bytes
        max-size = 256 * 1024; # 256 KiB
      };
    };
  };

  # ======================== DO NOT CHANGE THIS ========================
  system.stateVersion = "23.11";
  # ======================== DO NOT CHANGE THIS ========================
}