rewrite sops backend for secret generation and add tests

nixosModules/clanCore/secrets/default.nix

@@ -18,14 +18,17 @@
             type = lib.types.str;
             default = secret.config._module.args.name;
             description = ''
-              namespace of the secret
+              Namespace of the secret
             '';
           };
           generator = lib.mkOption {
-            type = lib.types.nullOr lib.types.str;
+            type = lib.types.str;
             description = ''
-              script to generate the secret.
+              Script to generate the secret.
-              can be set to null. then the user has to provide the secret via the clan cli
+              The script will be called with the following variables:
+              - facts: path to a directory where facts can be stored
+              - secrets: path to a directory where secrets can be stored
+              The script is expected to generate all secrets and facts defined in the module.
             '';
           };
           secrets = lib.mkOption {
@@ -63,7 +66,11 @@
                 };
                 value = lib.mkOption {
                   defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
-                  default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}";
+                  default =
+                    if builtins.pathExists "${config.clanCore.clanDir}/${fact.config.path}" then
+                      builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"
+                    else
+                      "";
                 };
               };
             }));

nixosModules/clanCore/secrets/sops.nix

@@ -19,62 +19,33 @@ let
 
   groups = builtins.attrNames (filterDir (containsMachine groupsDir) groupsDir);
   secrets = filterDir containsMachineOrGroups secretsDir;
+  systems = [ "i686-linux" "x86_64-linux" "riscv64-linux" "aarch64-linux" "x86_64-darwin" ];
 in
 {
   config = lib.mkIf (config.clanCore.secretStore == "sops") {
-    system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
+    system.clan = lib.genAttrs systems (system:
-      #!/bin/sh
+      let
-      set -efu
+        # Maybe use inputs.nixpkgs.legacyPackages here?
-
+        # don't reimport nixpkgs if we are on the same system (optimization)
-      test -d "$CLAN_DIR"
+        pkgs' = if pkgs.hostPlatform.system == system then pkgs else import pkgs.path { system = system; };
-
+      in
-      PATH=$PATH:${lib.makeBinPath [
+      {
-        config.clanCore.clanPkgs.clan-cli
+        generateSecrets = pkgs.writeScript "generate-secrets" ''
-      ]}
+          #!${pkgs'.python3}/bin/python
-
+          import json
-      # initialize secret store
+          from clan_cli.secrets.generate import generate_secrets_from_nix
-      if ! clan secrets machines list | grep -q ${config.clanCore.machineName}; then (
+          args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; secret_submodules = config.clanCore.secrets; })})
-        INITTMP=$(mktemp -d)
+          generate_secrets_from_nix(**args)
-        trap 'rm -rf "$INITTMP"' EXIT
+        '';
-        ${pkgs.age}/bin/age-keygen -o "$INITTMP/secret" 2> "$INITTMP/public"
+        uploadSecrets = pkgs.writeScript "upload-secrets" ''
-        PUBKEY=$(cat "$INITTMP/public" | sed 's/.*: //')
+          #!${pkgs'.python3}/bin/python
-        clan secrets machines add ${config.clanCore.machineName} "$PUBKEY"
+          import json
-        tail -1 "$INITTMP/secret" | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-age.key
+          from clan_cli.secrets.upload import upload_age_key_from_nix
-      ) fi
+          # the second toJSON is needed to escape the string for the python
-
+          args = json.loads(${builtins.toJSON (builtins.toJSON { machine_name = config.clanCore.machineName; deployment_address = config.clan.networking.deploymentAddress; age_key_file = config.sops.age.keyFile; })})
-      ${lib.foldlAttrs (acc: n: v: ''
+          upload_age_key_from_nix(**args)
-        ${acc}
+        '';
-        # ${n}
+      });
-        # if any of the secrets are missing, we regenerate all connected facts/secrets
-        (if ! ${lib.concatMapStringsSep " && " (x: "clan secrets get ${config.clanCore.machineName}-${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
-
-          facts=$(mktemp -d)
-          trap "rm -rf $facts" EXIT
-          secrets=$(mktemp -d)
-          trap "rm -rf $secrets" EXIT
-          ${v.generator}
-
-          ${lib.concatMapStrings (fact: ''
-            mkdir -p "$(dirname ${fact.path})"
-            cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
-          '') (lib.attrValues v.facts)}
-
-          ${lib.concatMapStrings (secret: ''
-            cat "$secrets"/${secret.name} | clan secrets set --machine ${config.clanCore.machineName} ${config.clanCore.machineName}-${secret.name}
-          '') (lib.attrValues v.secrets)}
-        fi)
-      '') "" config.clanCore.secrets}
-    '';
-    system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
-      #!/bin/sh
-      set -efu
-
-      tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
-      trap "rm -rf $tmp_dir" EXIT
-      clan secrets get ${config.clanCore.machineName}-age.key > "$tmp_dir/key.txt"
-
-      cat "$tmp_dir/key.txt" | ssh ${config.clan.networking.deploymentAddress} 'mkdir -p "$(dirname ${lib.escapeShellArg config.sops.age.keyFile})"; cat > ${lib.escapeShellArg config.sops.age.keyFile}'
-    '';
     sops.secrets = builtins.mapAttrs
       (name: _: {
         sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";

pkgs/clan-cli/clan_cli/machines/facts.py

@@ -0,0 +1,9 @@
+from .folders import machine_folder
+
+
+def machine_has_fact(machine: str, fact: str) -> bool:
+    return (machine_folder(machine) / "facts" / fact).exists()
+
+
+def machine_get_fact(machine: str, fact: str) -> str:
+    return (machine_folder(machine) / "facts" / fact).read_text()

pkgs/clan-cli/clan_cli/nix.py

@@ -1,5 +1,8 @@
+import json
 import os
+import subprocess
 import tempfile
+from typing import Any
 
 from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
 
@@ -25,6 +28,16 @@ def nix_build(
     )
 
 
+def nix_config() -> dict[str, Any]:
+    cmd = nix_command(["show-config", "--json"])
+    proc = subprocess.run(cmd, check=True, text=True, stdout=subprocess.PIPE)
+    data = json.loads(proc.stdout)
+    config = {}
+    for key, value in data.items():
+        config[key] = value["value"]
+    return config
+
+
 def nix_eval(flags: list[str]) -> list[str]:
     default_flags = nix_command(
         [

pkgs/clan-cli/clan_cli/secrets/generate.py

@@ -1,31 +1,41 @@
 import argparse
 import os
+import shlex
+import shutil
 import subprocess
 import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any
 
 from clan_cli.errors import ClanError
 
-from ..dirs import get_clan_flake_toplevel
+from ..dirs import get_clan_flake_toplevel, module_root
-from ..nix import nix_build
+from ..nix import nix_build, nix_config
+from .folders import sops_secrets_folder
+from .machines import add_machine, has_machine
+from .secrets import encrypt_secret, has_secret
+from .sops import generate_private_key
 
 
 def generate_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel().as_posix().strip()
     env = os.environ.copy()
     env["CLAN_DIR"] = clan_dir
+    env["PYTHONPATH"] = str(module_root().parent)
+    config = nix_config()
+    system = config["system"]
 
-    proc = subprocess.run(
+    cmd = nix_build(
-        nix_build(
+        [
-            [
+            f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.{system}.generateSecrets'
-                f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets'
+        ]
-            ]
-        ),
-        capture_output=True,
-        text=True,
     )
+    proc = subprocess.run(cmd, stdout=subprocess.PIPE, text=True)
     if proc.returncode != 0:
-        print(proc.stderr, file=sys.stderr)
+        raise ClanError(
-        raise ClanError(f"failed to generate secrets:\n{proc.stderr}")
+            f"failed to generate secrets:\n{shlex.join(cmd)}\nexited with {proc.returncode}"
+        )
 
     secret_generator_script = proc.stdout.strip()
     print(secret_generator_script)
@@ -40,6 +50,87 @@ def generate_secrets(machine: str) -> None:
         print("successfully generated secrets")
 
 
+def generate_host_key(machine_name: str) -> None:
+    if has_machine(machine_name):
+        return
+    priv_key, pub_key = generate_private_key()
+    encrypt_secret(sops_secrets_folder() / f"{machine_name}-age.key", priv_key)
+    add_machine(machine_name, pub_key, False)
+
+
+def generate_secrets_group(
+    secret_group: str, machine_name: str, tempdir: Path, secret_options: dict[str, Any]
+) -> None:
+    clan_dir = get_clan_flake_toplevel()
+    secrets = secret_options["secrets"]
+    needs_regeneration = any(
+        not has_secret(f"{machine_name}-{secret['name']}")
+        for secret in secrets.values()
+    )
+    generator = secret_options["generator"]
+    subdir = tempdir / secret_group
+    if needs_regeneration:
+        facts_dir = subdir / "facts"
+        facts_dir.mkdir(parents=True)
+        secrets_dir = subdir / "secrets"
+        secrets_dir.mkdir(parents=True)
+
+        text = f"""\
+set -euo pipefail
+facts={shlex.quote(str(facts_dir))}
+secrets={shlex.quote(str(secrets_dir))}
+{generator}
+        """
+        try:
+            subprocess.run(["bash", "-c", text], check=True)
+        except subprocess.CalledProcessError:
+            msg = "failed to the following command:\n"
+            msg += text
+            raise ClanError(msg)
+        for secret in secrets.values():
+            secret_file = secrets_dir / secret["name"]
+            if not secret_file.is_file():
+                msg = f"did not generate a file for '{secret['name']}' when running the following command:\n"
+                msg += text
+                raise ClanError(msg)
+            encrypt_secret(
+                sops_secrets_folder() / f"{machine_name}-{secret['name']}",
+                secret_file.read_text(),
+            )
+        for fact in secret_options["facts"].values():
+            fact_file = facts_dir / fact["name"]
+            if not fact_file.is_file():
+                msg = f"did not generate a file for '{fact['name']}' when running the following command:\n"
+                msg += text
+                raise ClanError(msg)
+            fact_path = clan_dir.joinpath(fact["path"])
+            fact_path.parent.mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(fact_file, fact_path)
+
+
+# this is called by the sops.nix clan core module
+def generate_secrets_from_nix(
+    machine_name: str,
+    secret_submodules: dict[str, Any],
+) -> None:
+    generate_host_key(machine_name)
+    errors = {}
+    with TemporaryDirectory() as d:
+        # if any of the secrets are missing, we regenerate all connected facts/secrets
+        for secret_group, secret_options in secret_submodules.items():
+            try:
+                generate_secrets_group(
+                    secret_group, machine_name, Path(d), secret_options
+                )
+            except ClanError as e:
+                errors[secret_group] = e
+    for secret_group, error in errors.items():
+        print(f"failed to generate secrets for {machine_name}/{secret_group}:")
+        print(error, file=sys.stderr)
+    if len(errors) > 0:
+        sys.exit(1)
+
+
 def generate_command(args: argparse.Namespace) -> None:
     generate_secrets(args.machine)
 

pkgs/clan-cli/clan_cli/secrets/upload.py

@@ -1,20 +1,25 @@
 import argparse
 import json
 import subprocess
-
+import sys
-from clan_cli.errors import ClanError
+from pathlib import Path
 
 from ..dirs import get_clan_flake_toplevel
-from ..nix import nix_build, nix_eval
+from ..errors import ClanError
+from ..nix import nix_build, nix_config, nix_eval
+from ..ssh import parse_deployment_address
+from .secrets import decrypt_secret, has_secret
 
 
 def upload_secrets(machine: str) -> None:
     clan_dir = get_clan_flake_toplevel().as_posix()
+    config = nix_config()
+    system = config["system"]
 
     proc = subprocess.run(
         nix_build(
             [
-                f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets'
+                f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.{system}.uploadSecrets'
             ]
         ),
         stdout=subprocess.PIPE,
@@ -48,6 +53,34 @@ def upload_secrets(machine: str) -> None:
         print("successfully uploaded secrets")
 
 
+# this is called by the sops.nix clan core module
+def upload_age_key_from_nix(
+    machine_name: str, deployment_address: str, age_key_file: str
+) -> None:
+    secret_name = f"{machine_name}-age.key"
+    if not has_secret(secret_name):  # skip uploading the secret, not managed by us
+        return
+    secret = decrypt_secret(secret_name)
+
+    h = parse_deployment_address(machine_name, deployment_address)
+    path = Path(age_key_file)
+
+    proc = h.run(
+        [
+            "bash",
+            "-c",
+            'mkdir -p "$0" && echo -n "$1" > "$2"',
+            str(path.parent),
+            secret,
+            age_key_file,
+        ],
+        check=False,
+    )
+    if proc.returncode != 0:
+        print(f"failed to upload age key to {deployment_address}")
+        sys.exit(1)
+
+
 def upload_command(args: argparse.Namespace) -> None:
     upload_secrets(args.machine)
 

pkgs/clan-cli/tests/test_flake_with_core/flake.nix

@@ -11,6 +11,17 @@
       machines = {
         vm1 = { modulesPath, ... }: {
           imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
+          clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";
+          sops.age.keyFile = "__CLAN_SOPS_KEY_PATH__";
+
+          clanCore.secrets.testpassword = {
+            generator = ''
+              echo "secret1" > "$secrets/secret1"
+              echo "fact1" > "$facts/fact1"
+            '';
+            secrets.secret1 = { };
+            facts.fact1 = { };
+          };
         };
       };
     };

pkgs/clan-cli/tests/test_secrets_generate.py

@@ -0,0 +1,38 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from cli import Cli
+
+from clan_cli.machines.facts import machine_get_fact
+from clan_cli.secrets.folders import sops_secrets_folder
+from clan_cli.secrets.secrets import has_secret
+
+if TYPE_CHECKING:
+    from age_keys import KeyPair
+
+
+@pytest.mark.impure
+def test_upload_secret(
+    monkeypatch: pytest.MonkeyPatch,
+    test_flake_with_core: Path,
+    age_keys: list["KeyPair"],
+) -> None:
+    monkeypatch.chdir(test_flake_with_core)
+    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+    cli = Cli()
+    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
+    cli.run(["secrets", "generate", "vm1"])
+    has_secret("vm1-age.key")
+    has_secret("vm1-secret1")
+    fact1 = machine_get_fact("vm1", "fact1")
+    assert fact1 == "fact1\n"
+    age_key = sops_secrets_folder().joinpath("vm1-age.key").joinpath("secret")
+    secret1 = sops_secrets_folder().joinpath("vm1-secret1").joinpath("secret")
+    age_key_mtime = age_key.lstat().st_mtime_ns
+    secret1_mtime = secret1.lstat().st_mtime_ns
+
+    # test idempotency
+    cli.run(["secrets", "generate", "vm1"])
+    assert age_key.lstat().st_mtime_ns == age_key_mtime
+    assert secret1.lstat().st_mtime_ns == secret1_mtime

pkgs/clan-cli/tests/test_secrets_upload.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from cli import Cli
+
+from clan_cli.ssh import HostGroup
+
+if TYPE_CHECKING:
+    from age_keys import KeyPair
+
+
+@pytest.mark.impure
+def test_upload_secret(
+    monkeypatch: pytest.MonkeyPatch,
+    test_flake_with_core: Path,
+    host_group: HostGroup,
+    age_keys: list["KeyPair"],
+) -> None:
+    monkeypatch.chdir(test_flake_with_core)
+    monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
+
+    cli = Cli()
+    cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
+
+    cli.run(["secrets", "machines", "add", "vm1", age_keys[1].pubkey])
+    monkeypatch.setenv("SOPS_NIX_SECRET", age_keys[0].privkey)
+    cli.run(["secrets", "set", "vm1-age.key"])
+
+    flake = test_flake_with_core.joinpath("flake.nix")
+    host = host_group.hosts[0]
+    addr = f"{host.user}@{host.host}:{host.port}?StrictHostKeyChecking=no&UserKnownHostsFile=/dev/null&IdentityFile={host.key}"
+    new_text = flake.read_text().replace("__CLAN_DEPLOYMENT_ADDRESS__", addr)
+    sops_key = test_flake_with_core.joinpath("sops.key")
+    new_text = new_text.replace("__CLAN_SOPS_KEY_PATH__", str(sops_key))
+
+    flake.write_text(new_text)
+    cli.run(["secrets", "upload", "vm1"])
+    assert sops_key.exists()
+    assert sops_key.read_text() == age_keys[0].privkey