From fb583a7b3488ab4de9c567149a555c91bc983376 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Thu, 14 Sep 2023 15:27:53 +0200 Subject: [PATCH 01/44] Removed workflow --- .gitea/workflows/ui_assets.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/ui_assets.yaml b/.gitea/workflows/ui_assets.yaml index a9f0889..9fb58ef 100644 --- a/.gitea/workflows/ui_assets.yaml +++ b/.gitea/workflows/ui_assets.yaml @@ -1,8 +1,8 @@ name: assets -on: - push: - branches: - - main +# on: +# push: +# branches: +# - main jobs: test: runs-on: nix From 837ebda70d149e679643c2524639951c6289faa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 16:28:01 +0200 Subject: [PATCH 02/44] gitignore: use absolute paths --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 39e97f9..42dff4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .direnv result* -pkgs/clan-cli/clan_cli/nixpkgs -pkgs/clan-cli/clan_cli/webui/assets -machines +/pkgs/clan-cli/clan_cli/nixpkgs +/pkgs/clan-cli/clan_cli/webui/assets +/machines # python __pycache__ From 4905e0cecddeafbad4ad8eff3e364706dd6b93fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 16:33:40 +0200 Subject: [PATCH 03/44] move machine_flake fixture to its own file --- pkgs/clan-cli/tests/conftest.py | 36 +--------------------------- pkgs/clan-cli/tests/machine_flake.py | 34 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 35 deletions(-) create mode 100644 pkgs/clan-cli/tests/machine_flake.py diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 70ebd82..4c1ac4d 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -1,12 +1,5 @@ import os import sys -import tempfile -from pathlib import Path -from typing import Generator - -import pytest - -from clan_cli.dirs import nixpkgs_source sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) @@ -20,32 +13,5 @@ pytest_plugins = [ "command", "ports", "host_group", + "machine_flake", ] - - -@pytest.fixture(scope="module") -def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: - with pytest.MonkeyPatch.context() as mp: - yield mp - - -@pytest.fixture(scope="module") -def machine_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: - template = Path(__file__).parent / "machine_flake" - # copy the template to a new temporary location - with tempfile.TemporaryDirectory() as tmpdir_: - flake = Path(tmpdir_) - for path in template.glob("**/*"): - if path.is_dir(): - (flake / path.relative_to(template)).mkdir() - else: - (flake / path.relative_to(template)).write_text(path.read_text()) - # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake - # provided by get_clan_flake_toplevel - flake_nix = flake / "flake.nix" - flake_nix.write_text( - flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) - ) - # check that an empty config is returned if no json file exists - monkeymodule.chdir(flake) - yield flake diff --git a/pkgs/clan-cli/tests/machine_flake.py b/pkgs/clan-cli/tests/machine_flake.py new file mode 100644 index 0000000..8660c58 --- /dev/null +++ b/pkgs/clan-cli/tests/machine_flake.py @@ -0,0 +1,34 @@ +import shutil +import tempfile +from pathlib import Path +from typing import Generator + +import pytest + +from clan_cli.dirs import nixpkgs_source + + +@pytest.fixture(scope="module") +def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="module") +def machine_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: + template = Path(__file__).parent / "machine_flake" + # copy the template to a new temporary location + with tempfile.TemporaryDirectory() as tmpdir_: + home = Path(tmpdir_) + flake = home / "machine_flake" + shutil.copytree(template, flake) + # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake + # provided by get_clan_flake_toplevel + flake_nix = flake / "flake.nix" + flake_nix.write_text( + flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) + ) + # check that an empty config is returned if no json file exists + monkeymodule.chdir(flake) + monkeymodule.setenv("HOME", str(home)) + yield flake From 2048ffccb09c28d69ff3c91bdbcdb9a04075535d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 16:57:04 +0200 Subject: [PATCH 04/44] drop clan_flake fixture --- pkgs/clan-cli/tests/clan_flake.py | 23 ------------------- pkgs/clan-cli/tests/conftest.py | 1 - pkgs/clan-cli/tests/machine_flake/.clan_flake | 0 pkgs/clan-cli/tests/test_import_sops_cli.py | 2 +- pkgs/clan-cli/tests/test_machines_api.py | 2 +- pkgs/clan-cli/tests/test_machines_cli.py | 4 +++- .../tests/test_machines_update_cli.py | 4 ++-- pkgs/clan-cli/tests/test_secrets_cli.py | 20 ++++++++-------- 8 files changed, 17 insertions(+), 39 deletions(-) create mode 100644 pkgs/clan-cli/tests/machine_flake/.clan_flake diff --git a/pkgs/clan-cli/tests/clan_flake.py b/pkgs/clan-cli/tests/clan_flake.py index 23d38b4..e69de29 100644 --- a/pkgs/clan-cli/tests/clan_flake.py +++ b/pkgs/clan-cli/tests/clan_flake.py @@ -1,23 +0,0 @@ -from pathlib import Path -from typing import Iterator - -import pytest - - -@pytest.fixture -def clan_flake(temporary_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: - flake = temporary_dir / "clan-flake" - flake.mkdir() - (flake / ".clan-flake").touch() - (flake / "flake.nix").write_text( - """ -{ - description = "A flake for testing clan"; - inputs = {}; - outputs = { self }: {}; -} -""" - ) - monkeypatch.chdir(flake) - monkeypatch.setenv("HOME", str(temporary_dir)) - yield flake diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 4c1ac4d..511698a 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -6,7 +6,6 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) pytest_plugins = [ "api", "temporary_dir", - "clan_flake", "root", "age_keys", "sshd", diff --git a/pkgs/clan-cli/tests/machine_flake/.clan_flake b/pkgs/clan-cli/tests/machine_flake/.clan_flake new file mode 100644 index 0000000..e69de29 diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 6ea6a85..903b2b1 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: def test_import_sops( test_root: Path, - clan_flake: Path, + machine_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 2551fbf..6663b7d 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -3,7 +3,7 @@ from pathlib import Path from api import TestClient -def test_machines(api: TestClient, clan_flake: Path) -> None: +def test_machines(api: TestClient, machine_flake: Path) -> None: response = api.get("/api/machines") assert response.status_code == 200 assert response.json() == {"machines": []} diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index a2e3fa8..4e94d3f 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -4,7 +4,9 @@ import pytest from cli import Cli -def test_machine_subcommands(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: +def test_machine_subcommands( + machine_flake: Path, capsys: pytest.CaptureFixture +) -> None: cli = Cli() cli.run(["machines", "create", "machine1"]) diff --git a/pkgs/clan-cli/tests/test_machines_update_cli.py b/pkgs/clan-cli/tests/test_machines_update_cli.py index 62650f3..7da1001 100644 --- a/pkgs/clan-cli/tests/test_machines_update_cli.py +++ b/pkgs/clan-cli/tests/test_machines_update_cli.py @@ -8,13 +8,13 @@ from host_group import HostGroup def test_update( - clan_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch + machine_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch ) -> None: assert len(host_group.hosts) == 1 host = host_group.hosts[0] with TemporaryDirectory() as tmpdir: - host.meta["flake_uri"] = clan_flake + host.meta["flake_uri"] = machine_flake host.meta["flake_path"] = str(Path(tmpdir) / "rsync-target") host.ssh_options["SendEnv"] = "REALPATH" bin = Path(tmpdir).joinpath("bin") diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index af604a8..4cca9fc 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -14,12 +14,12 @@ if TYPE_CHECKING: def _test_identities( what: str, - clan_flake: Path, + machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"], ) -> None: cli = Cli() - sops_folder = clan_flake / "sops" + sops_folder = machine_flake / "sops" cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() @@ -60,19 +60,19 @@ def _test_identities( def test_users( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", clan_flake, capsys, age_keys) + _test_identities("users", machine_flake, capsys, age_keys) def test_machines( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", clan_flake, capsys, age_keys) + _test_identities("machines", machine_flake, capsys, age_keys) def test_groups( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = Cli() capsys.readouterr() # empty the buffer @@ -100,7 +100,7 @@ def test_groups( cli.run(["secrets", "groups", "remove-user", "group1", "user1"]) cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"]) - groups = os.listdir(clan_flake / "sops" / "groups") + groups = os.listdir(machine_flake / "sops" / "groups") assert len(groups) == 0 @@ -114,7 +114,7 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_secrets( - clan_flake: Path, + machine_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], @@ -125,7 +125,7 @@ def test_secrets( assert capsys.readouterr().out == "" monkeypatch.setenv("SOPS_NIX_SECRET", "foo") - monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(clan_flake / ".." / "age.key")) + monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(machine_flake / ".." / "age.key")) cli.run(["secrets", "key", "generate"]) capsys.readouterr() # empty the buffer cli.run(["secrets", "key", "show"]) From 02f421546c5a90303601964f78a142c005d9be9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 17:04:04 +0200 Subject: [PATCH 05/44] rename machine_flake back to clan_flake --- pkgs/clan-cli/tests/clan_flake.py | 34 +++++++++++++++++++ .../{machine_flake => clan_flake}/.clan-flake | 0 .../{machine_flake => clan_flake}/flake.nix | 0 .../nixosModules/machine1.nix | 0 pkgs/clan-cli/tests/conftest.py | 2 +- pkgs/clan-cli/tests/machine_flake.py | 34 ------------------- pkgs/clan-cli/tests/machine_flake/.clan_flake | 0 pkgs/clan-cli/tests/test_config.py | 2 +- pkgs/clan-cli/tests/test_import_sops_cli.py | 2 +- pkgs/clan-cli/tests/test_machines_api.py | 4 +-- pkgs/clan-cli/tests/test_machines_cli.py | 4 +-- pkgs/clan-cli/tests/test_machines_config.py | 4 +-- .../tests/test_machines_update_cli.py | 4 +-- pkgs/clan-cli/tests/test_secrets_cli.py | 20 +++++------ 14 files changed, 54 insertions(+), 56 deletions(-) rename pkgs/clan-cli/tests/{machine_flake => clan_flake}/.clan-flake (100%) rename pkgs/clan-cli/tests/{machine_flake => clan_flake}/flake.nix (100%) rename pkgs/clan-cli/tests/{machine_flake => clan_flake}/nixosModules/machine1.nix (100%) delete mode 100644 pkgs/clan-cli/tests/machine_flake.py delete mode 100644 pkgs/clan-cli/tests/machine_flake/.clan_flake diff --git a/pkgs/clan-cli/tests/clan_flake.py b/pkgs/clan-cli/tests/clan_flake.py index e69de29..b0bc1e7 100644 --- a/pkgs/clan-cli/tests/clan_flake.py +++ b/pkgs/clan-cli/tests/clan_flake.py @@ -0,0 +1,34 @@ +import shutil +import tempfile +from pathlib import Path +from typing import Generator + +import pytest + +from clan_cli.dirs import nixpkgs_source + + +@pytest.fixture(scope="module") +def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: + with pytest.MonkeyPatch.context() as mp: + yield mp + + +@pytest.fixture(scope="module") +def clan_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: + template = Path(__file__).parent / "clan_flake" + # copy the template to a new temporary location + with tempfile.TemporaryDirectory() as tmpdir_: + home = Path(tmpdir_) + flake = home / "clan_flake" + shutil.copytree(template, flake) + # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake + # provided by get_clan_flake_toplevel + flake_nix = flake / "flake.nix" + flake_nix.write_text( + flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) + ) + # check that an empty config is returned if no json file exists + monkeymodule.chdir(flake) + monkeymodule.setenv("HOME", str(home)) + yield flake diff --git a/pkgs/clan-cli/tests/machine_flake/.clan-flake b/pkgs/clan-cli/tests/clan_flake/.clan-flake similarity index 100% rename from pkgs/clan-cli/tests/machine_flake/.clan-flake rename to pkgs/clan-cli/tests/clan_flake/.clan-flake diff --git a/pkgs/clan-cli/tests/machine_flake/flake.nix b/pkgs/clan-cli/tests/clan_flake/flake.nix similarity index 100% rename from pkgs/clan-cli/tests/machine_flake/flake.nix rename to pkgs/clan-cli/tests/clan_flake/flake.nix diff --git a/pkgs/clan-cli/tests/machine_flake/nixosModules/machine1.nix b/pkgs/clan-cli/tests/clan_flake/nixosModules/machine1.nix similarity index 100% rename from pkgs/clan-cli/tests/machine_flake/nixosModules/machine1.nix rename to pkgs/clan-cli/tests/clan_flake/nixosModules/machine1.nix diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 511698a..7d0981d 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -12,5 +12,5 @@ pytest_plugins = [ "command", "ports", "host_group", - "machine_flake", + "clan_flake", ] diff --git a/pkgs/clan-cli/tests/machine_flake.py b/pkgs/clan-cli/tests/machine_flake.py deleted file mode 100644 index 8660c58..0000000 --- a/pkgs/clan-cli/tests/machine_flake.py +++ /dev/null @@ -1,34 +0,0 @@ -import shutil -import tempfile -from pathlib import Path -from typing import Generator - -import pytest - -from clan_cli.dirs import nixpkgs_source - - -@pytest.fixture(scope="module") -def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: - with pytest.MonkeyPatch.context() as mp: - yield mp - - -@pytest.fixture(scope="module") -def machine_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: - template = Path(__file__).parent / "machine_flake" - # copy the template to a new temporary location - with tempfile.TemporaryDirectory() as tmpdir_: - home = Path(tmpdir_) - flake = home / "machine_flake" - shutil.copytree(template, flake) - # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake - # provided by get_clan_flake_toplevel - flake_nix = flake / "flake.nix" - flake_nix.write_text( - flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) - ) - # check that an empty config is returned if no json file exists - monkeymodule.chdir(flake) - monkeymodule.setenv("HOME", str(home)) - yield flake diff --git a/pkgs/clan-cli/tests/machine_flake/.clan_flake b/pkgs/clan-cli/tests/machine_flake/.clan_flake deleted file mode 100644 index e69de29..0000000 diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 79fc2d8..3def8ce 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -50,7 +50,7 @@ def test_set_some_option( def test_configure_machine( - machine_flake: Path, + clan_flake: Path, temporary_dir: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 903b2b1..6ea6a85 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: def test_import_sops( test_root: Path, - machine_flake: Path, + clan_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 6663b7d..aced71a 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -3,7 +3,7 @@ from pathlib import Path from api import TestClient -def test_machines(api: TestClient, machine_flake: Path) -> None: +def test_machines(api: TestClient, clan_flake: Path) -> None: response = api.get("/api/machines") assert response.status_code == 200 assert response.json() == {"machines": []} @@ -21,7 +21,7 @@ def test_machines(api: TestClient, machine_flake: Path) -> None: assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]} -def test_configure_machine(api: TestClient, machine_flake: Path) -> None: +def test_configure_machine(api: TestClient, clan_flake: Path) -> None: # ensure error 404 if machine does not exist when accessing the config response = api.get("/api/machines/machine1/config") assert response.status_code == 404 diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index 4e94d3f..a2e3fa8 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -4,9 +4,7 @@ import pytest from cli import Cli -def test_machine_subcommands( - machine_flake: Path, capsys: pytest.CaptureFixture -) -> None: +def test_machine_subcommands(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: cli = Cli() cli.run(["machines", "create", "machine1"]) diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index 2484435..c7fe80c 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -3,6 +3,6 @@ from pathlib import Path from clan_cli.config import machine -def test_schema_for_machine(machine_flake: Path) -> None: - schema = machine.schema_for_machine("machine1", machine_flake) +def test_schema_for_machine(clan_flake: Path) -> None: + schema = machine.schema_for_machine("machine1", clan_flake) assert "properties" in schema diff --git a/pkgs/clan-cli/tests/test_machines_update_cli.py b/pkgs/clan-cli/tests/test_machines_update_cli.py index 7da1001..62650f3 100644 --- a/pkgs/clan-cli/tests/test_machines_update_cli.py +++ b/pkgs/clan-cli/tests/test_machines_update_cli.py @@ -8,13 +8,13 @@ from host_group import HostGroup def test_update( - machine_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch + clan_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch ) -> None: assert len(host_group.hosts) == 1 host = host_group.hosts[0] with TemporaryDirectory() as tmpdir: - host.meta["flake_uri"] = machine_flake + host.meta["flake_uri"] = clan_flake host.meta["flake_path"] = str(Path(tmpdir) / "rsync-target") host.ssh_options["SendEnv"] = "REALPATH" bin = Path(tmpdir).joinpath("bin") diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index 4cca9fc..af604a8 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -14,12 +14,12 @@ if TYPE_CHECKING: def _test_identities( what: str, - machine_flake: Path, + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"], ) -> None: cli = Cli() - sops_folder = machine_flake / "sops" + sops_folder = clan_flake / "sops" cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() @@ -60,19 +60,19 @@ def _test_identities( def test_users( - machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", machine_flake, capsys, age_keys) + _test_identities("users", clan_flake, capsys, age_keys) def test_machines( - machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", machine_flake, capsys, age_keys) + _test_identities("machines", clan_flake, capsys, age_keys) def test_groups( - machine_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = Cli() capsys.readouterr() # empty the buffer @@ -100,7 +100,7 @@ def test_groups( cli.run(["secrets", "groups", "remove-user", "group1", "user1"]) cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"]) - groups = os.listdir(machine_flake / "sops" / "groups") + groups = os.listdir(clan_flake / "sops" / "groups") assert len(groups) == 0 @@ -114,7 +114,7 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_secrets( - machine_flake: Path, + clan_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], @@ -125,7 +125,7 @@ def test_secrets( assert capsys.readouterr().out == "" monkeypatch.setenv("SOPS_NIX_SECRET", "foo") - monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(machine_flake / ".." / "age.key")) + monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(clan_flake / ".." / "age.key")) cli.run(["secrets", "key", "generate"]) capsys.readouterr() # empty the buffer cli.run(["secrets", "key", "show"]) From 2697ad54b71f330e00157bcfae67486dfbb9c7d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 17:25:46 +0200 Subject: [PATCH 06/44] drop useless test_update test This test was not finished by me? Now it's going to be broken by lassulus. --- .../tests/test_machines_update_cli.py | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 pkgs/clan-cli/tests/test_machines_update_cli.py diff --git a/pkgs/clan-cli/tests/test_machines_update_cli.py b/pkgs/clan-cli/tests/test_machines_update_cli.py deleted file mode 100644 index 62650f3..0000000 --- a/pkgs/clan-cli/tests/test_machines_update_cli.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil -from pathlib import Path -from tempfile import TemporaryDirectory - -import pytest -from host_group import HostGroup - - -def test_update( - clan_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch -) -> None: - assert len(host_group.hosts) == 1 - host = host_group.hosts[0] - - with TemporaryDirectory() as tmpdir: - host.meta["flake_uri"] = clan_flake - host.meta["flake_path"] = str(Path(tmpdir) / "rsync-target") - host.ssh_options["SendEnv"] = "REALPATH" - bin = Path(tmpdir).joinpath("bin") - bin.mkdir() - nixos_rebuild = bin.joinpath("nixos-rebuild") - bash = shutil.which("bash") - assert bash is not None - nixos_rebuild.write_text( - f"""#!{bash} -exit 0 -""" - ) - nixos_rebuild.chmod(0o755) - f"{tmpdir}/bin:{os.environ['PATH']}" - nix_state_dir = Path(tmpdir).joinpath("nix") - nix_state_dir.mkdir() - - monkeypatch.setenv("REALPATH", str(nix_state_dir)) From 87f0b090ef670c0b3e511fab1d68e597786402ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 12:31:18 +0200 Subject: [PATCH 07/44] cli: rename clan_flake to test_flake --- pkgs/clan-cli/tests/clan_flake.py | 34 ------------ pkgs/clan-cli/tests/conftest.py | 2 +- pkgs/clan-cli/tests/test_config.py | 2 +- pkgs/clan-cli/tests/test_flake.py | 54 +++++++++++++++++++ .../{clan_flake => test_flake}/.clan-flake | 0 .../{clan_flake => test_flake}/flake.nix | 6 +-- .../nixosModules/machine1.nix | 0 pkgs/clan-cli/tests/test_import_sops_cli.py | 2 +- pkgs/clan-cli/tests/test_machines_api.py | 4 +- pkgs/clan-cli/tests/test_machines_cli.py | 2 +- pkgs/clan-cli/tests/test_machines_config.py | 4 +- pkgs/clan-cli/tests/test_secrets_cli.py | 20 +++---- 12 files changed, 74 insertions(+), 56 deletions(-) delete mode 100644 pkgs/clan-cli/tests/clan_flake.py create mode 100644 pkgs/clan-cli/tests/test_flake.py rename pkgs/clan-cli/tests/{clan_flake => test_flake}/.clan-flake (100%) rename pkgs/clan-cli/tests/{clan_flake => test_flake}/flake.nix (84%) rename pkgs/clan-cli/tests/{clan_flake => test_flake}/nixosModules/machine1.nix (100%) diff --git a/pkgs/clan-cli/tests/clan_flake.py b/pkgs/clan-cli/tests/clan_flake.py deleted file mode 100644 index b0bc1e7..0000000 --- a/pkgs/clan-cli/tests/clan_flake.py +++ /dev/null @@ -1,34 +0,0 @@ -import shutil -import tempfile -from pathlib import Path -from typing import Generator - -import pytest - -from clan_cli.dirs import nixpkgs_source - - -@pytest.fixture(scope="module") -def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: - with pytest.MonkeyPatch.context() as mp: - yield mp - - -@pytest.fixture(scope="module") -def clan_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: - template = Path(__file__).parent / "clan_flake" - # copy the template to a new temporary location - with tempfile.TemporaryDirectory() as tmpdir_: - home = Path(tmpdir_) - flake = home / "clan_flake" - shutil.copytree(template, flake) - # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake - # provided by get_clan_flake_toplevel - flake_nix = flake / "flake.nix" - flake_nix.write_text( - flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) - ) - # check that an empty config is returned if no json file exists - monkeymodule.chdir(flake) - monkeymodule.setenv("HOME", str(home)) - yield flake diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 7d0981d..c502c73 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -12,5 +12,5 @@ pytest_plugins = [ "command", "ports", "host_group", - "clan_flake", + "test_flake", ] diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 3def8ce..c65ef11 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -50,7 +50,7 @@ def test_set_some_option( def test_configure_machine( - clan_flake: Path, + test_flake: Path, temporary_dir: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, diff --git a/pkgs/clan-cli/tests/test_flake.py b/pkgs/clan-cli/tests/test_flake.py new file mode 100644 index 0000000..cff8caf --- /dev/null +++ b/pkgs/clan-cli/tests/test_flake.py @@ -0,0 +1,54 @@ +import fileinput +import shutil +import tempfile +from pathlib import Path +from typing import Iterator + +import pytest +from root import PROJECT_ROOT + +from clan_cli.dirs import nixpkgs_source + + +@pytest.fixture(scope="module") +def monkeymodule() -> Iterator[pytest.MonkeyPatch]: + with pytest.MonkeyPatch.context() as mp: + yield mp + + +def create_flake( + monkeymodule: pytest.MonkeyPatch, name: str, clan_core_flake: Path | None = None +) -> Iterator[Path]: + template = Path(__file__).parent / name + # copy the template to a new temporary location + with tempfile.TemporaryDirectory() as tmpdir_: + home = Path(tmpdir_) + flake = home / name + shutil.copytree(template, flake) + # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake + # provided by get_test_flake_toplevel + flake_nix = flake / "flake.nix" + for line in fileinput.input(flake_nix, inplace=True): + line = line.replace("__NIXPKGS__", str(nixpkgs_source())) + if clan_core_flake: + line = line.replace("__CLAN_CORE__", str(clan_core_flake)) + print(line) + # check that an empty config is returned if no json file exists + monkeymodule.chdir(flake) + monkeymodule.setenv("HOME", str(home)) + yield flake + + +@pytest.fixture(scope="module") +def test_flake(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: + yield from create_flake(monkeymodule, "test_flake") + + +@pytest.fixture(scope="module") +def test_flake_with_core(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: + clan_core_flake = PROJECT_ROOT.parent.parent + if not (clan_core_flake / "flake.nix").exists(): + raise Exception( + "clan-core flake not found. This test requires the clan-core flake to be present" + ) + yield from create_flake(monkeymodule, "test_flake_with_core", clan_core_flake) diff --git a/pkgs/clan-cli/tests/clan_flake/.clan-flake b/pkgs/clan-cli/tests/test_flake/.clan-flake similarity index 100% rename from pkgs/clan-cli/tests/clan_flake/.clan-flake rename to pkgs/clan-cli/tests/test_flake/.clan-flake diff --git a/pkgs/clan-cli/tests/clan_flake/flake.nix b/pkgs/clan-cli/tests/test_flake/flake.nix similarity index 84% rename from pkgs/clan-cli/tests/clan_flake/flake.nix rename to pkgs/clan-cli/tests/test_flake/flake.nix index ff220a5..3ba6f3f 100644 --- a/pkgs/clan-cli/tests/clan_flake/flake.nix +++ b/pkgs/clan-cli/tests/test_flake/flake.nix @@ -1,8 +1,6 @@ { - inputs = { - # this placeholder is replaced by the path to nixpkgs - nixpkgs.url = "__NIXPKGS__"; - }; + # this placeholder is replaced by the path to nixpkgs + inputs.nixpkgs.url = "__NIXPKGS__"; outputs = inputs: { nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { diff --git a/pkgs/clan-cli/tests/clan_flake/nixosModules/machine1.nix b/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix similarity index 100% rename from pkgs/clan-cli/tests/clan_flake/nixosModules/machine1.nix rename to pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 6ea6a85..64f68c9 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: def test_import_sops( test_root: Path, - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index aced71a..2fba075 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -3,7 +3,7 @@ from pathlib import Path from api import TestClient -def test_machines(api: TestClient, clan_flake: Path) -> None: +def test_machines(api: TestClient, test_flake: Path) -> None: response = api.get("/api/machines") assert response.status_code == 200 assert response.json() == {"machines": []} @@ -21,7 +21,7 @@ def test_machines(api: TestClient, clan_flake: Path) -> None: assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]} -def test_configure_machine(api: TestClient, clan_flake: Path) -> None: +def test_configure_machine(api: TestClient, test_flake: Path) -> None: # ensure error 404 if machine does not exist when accessing the config response = api.get("/api/machines/machine1/config") assert response.status_code == 404 diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index a2e3fa8..6b92645 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -4,7 +4,7 @@ import pytest from cli import Cli -def test_machine_subcommands(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: +def test_machine_subcommands(test_flake: Path, capsys: pytest.CaptureFixture) -> None: cli = Cli() cli.run(["machines", "create", "machine1"]) diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index c7fe80c..4d3a0a7 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -3,6 +3,6 @@ from pathlib import Path from clan_cli.config import machine -def test_schema_for_machine(clan_flake: Path) -> None: - schema = machine.schema_for_machine("machine1", clan_flake) +def test_schema_for_machine(test_flake: Path) -> None: + schema = machine.schema_for_machine("machine1", test_flake) assert "properties" in schema diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index af604a8..7f3c816 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -14,12 +14,12 @@ if TYPE_CHECKING: def _test_identities( what: str, - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"], ) -> None: cli = Cli() - sops_folder = clan_flake / "sops" + sops_folder = test_flake / "sops" cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() @@ -60,19 +60,19 @@ def _test_identities( def test_users( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", clan_flake, capsys, age_keys) + _test_identities("users", test_flake, capsys, age_keys) def test_machines( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", clan_flake, capsys, age_keys) + _test_identities("machines", test_flake, capsys, age_keys) def test_groups( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = Cli() capsys.readouterr() # empty the buffer @@ -100,7 +100,7 @@ def test_groups( cli.run(["secrets", "groups", "remove-user", "group1", "user1"]) cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"]) - groups = os.listdir(clan_flake / "sops" / "groups") + groups = os.listdir(test_flake / "sops" / "groups") assert len(groups) == 0 @@ -114,7 +114,7 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_secrets( - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], @@ -125,7 +125,7 @@ def test_secrets( assert capsys.readouterr().out == "" monkeypatch.setenv("SOPS_NIX_SECRET", "foo") - monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(clan_flake / ".." / "age.key")) + monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake / ".." / "age.key")) cli.run(["secrets", "key", "generate"]) capsys.readouterr() # empty the buffer cli.run(["secrets", "key", "show"]) From a8ba56de63113b3763a95078ecf549878e3290aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 13:44:47 +0200 Subject: [PATCH 08/44] add ipdb to python breakpoint() --- pkgs/clan-cli/shell.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 336b2f0..f0ff0d7 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -6,6 +6,7 @@ let ++ clan-cli.devDependencies ++ [ ps.pip + ps.ipdb ] ); checkScript = writeScriptBin "check" '' @@ -23,6 +24,8 @@ mkShell { # evaluating the flake .# CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json; PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}"; + PYTHONBREAKPOINT = "ipdb.set_trace"; + shellHook = '' tmp_path=$(realpath ./.direnv) From 9b7b6996d669975aa57bc7557a6fc9d7e276fe30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 13:44:04 +0200 Subject: [PATCH 09/44] nix_eval: add --json by default --- pkgs/clan-cli/clan_cli/config/__init__.py | 2 -- pkgs/clan-cli/clan_cli/config/machine.py | 1 - pkgs/clan-cli/clan_cli/nix.py | 37 ++++++++++++----------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index f73a9dc..d81fbc2 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -100,7 +100,6 @@ def options_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict proc = subprocess.run( nix_eval( flags=[ - "--json", "--show-trace", "--impure", "--expr", @@ -138,7 +137,6 @@ def read_machine_option_value(machine_name: str, option: str) -> str: proc = subprocess.run( nix_eval( flags=[ - "--json", "--show-trace", "--extra-experimental-features", "nix-command flakes", diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index 0dec9c1..1e3b29a 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -45,7 +45,6 @@ def schema_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict: proc = subprocess.run( nix_eval( flags=[ - "--json", "--impure", "--show-trace", "--extra-experimental-features", diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 7a29f02..559b823 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -41,29 +41,30 @@ def nix_build_machine( def nix_eval(flags: list[str]) -> list[str]: - if os.environ.get("IN_NIX_SANDBOX"): - with tempfile.TemporaryDirectory() as nix_store: - return [ - "nix", - "eval", - "--show-trace", - "--extra-experimental-features", - "nix-command flakes", - "--override-input", - "nixpkgs", - str(nixpkgs_source()), - # --store is required to prevent this error: - # error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted - "--store", - nix_store, - ] + flags - return [ + default_flags = [ "nix", "eval", "--show-trace", + "--json", "--extra-experimental-features", "nix-command flakes", - ] + flags + ] + if os.environ.get("IN_NIX_SANDBOX"): + with tempfile.TemporaryDirectory() as nix_store: + return ( + default_flags + + [ + "--override-input", + "nixpkgs", + str(nixpkgs_source()), + # --store is required to prevent this error: + # error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted + "--store", + nix_store, + ] + + flags + ) + return default_flags + flags def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: From 3f6fa0eeca368e510f0f295b7ca18365efe2b158 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 6 Sep 2023 16:08:36 +0200 Subject: [PATCH 10/44] clanCore secrets: add secretStore option --- nixosModules/clanCore/secrets/default.nix | 7 +++++++ nixosModules/clanCore/secrets/sops.nix | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 48fef29..fa961dd 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -1,5 +1,12 @@ { config, lib, ... }: { + options.clanCore.secretStore = lib.mkOption { + type = lib.types.enum [ "sops" "password-store" "custom" ]; + default = "sops"; + description = '' + method to store secrets + ''; + }; options.clanCore.secrets = lib.mkOption { type = lib.types.attrsOf (lib.types.submodule (secret: { diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index ab97722..2371489 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -21,7 +21,7 @@ let secrets = filterDir containsMachineOrGroups secretsDir; in { - config = { + config = lib.mkIf (config.clanCore.secretStore == "sops") { system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' #!/bin/sh set -efu From 5285423479136a5a71f2246a1e9a27444d64c024 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 6 Sep 2023 16:09:43 +0200 Subject: [PATCH 11/44] secrets: add password-store implementation --- nixosModules/clanCore/secrets/default.nix | 3 +- .../clanCore/secrets/password-store.nix | 120 ++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 nixosModules/clanCore/secrets/password-store.nix diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index fa961dd..51c6aa3 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -68,6 +68,7 @@ })); }; imports = [ - ./sops.nix # for now we have only one implementation, thats why we import it here and not in clanModules + ./sops.nix + ./password-store.nix ]; } diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix new file mode 100644 index 0000000..7b559a5 --- /dev/null +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -0,0 +1,120 @@ +{ config, lib, pkgs, ... }: +let + passwordstoreDir = "$HOME/.password-store"; +in +{ + options.clan.password-store.targetDirectory = lib.mkOption { + type = lib.types.path; + default = "/etc/secrets"; + description = '' + The directory where the password store is deployed to. + ''; + }; + config = lib.mkIf (config.clanCore.secretStore == "password-store") { + system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' + #!/bin/sh + set -efu + set -x # remove for prod + + PATH=${lib.makeBinPath [ + pkgs.pass + ]}:$PATH + + # TODO maybe initialize password store if it doesn't exist yet + + ${lib.foldlAttrs (acc: n: v: '' + ${acc} + # ${n} + # if any of the secrets are missing, we regenerate all connected facts/secrets + (if ! ${lib.concatMapStringsSep " && " (x: "pass show machines/${config.clanCore.machineName}/${x.name} >/dev/null") (lib.attrValues v.secrets)}; then + + facts=$(mktemp -d) + trap "rm -rf $facts" EXIT + secrets=$(mktemp -d) + trap "rm -rf $secrets" EXIT + ${v.generator} + + ${lib.concatMapStrings (fact: '' + mkdir -p "$(dirname ${fact.path})" + cp "$facts"/${fact.name} ${fact.path} + '') (lib.attrValues v.facts)} + + ${lib.concatMapStrings (secret: '' + cat "$secrets"/${secret.name} | pass insert -m machines/${config.clanCore.machineName}/${secret.name} + '') (lib.attrValues v.secrets)} + fi) + '') "" config.clanCore.secrets} + ''; + system.clan.deploySecrets = pkgs.writeScript "deploy-secrets" '' + #!/bin/sh + set -efu + set -x # remove for prod + + target=$1 + + umask 0077 + + PATH=${lib.makeBinPath [ + pkgs.pass + pkgs.git + pkgs.findutils + pkgs.openssh + pkgs.rsync + ]}:$PATH + + if test -e ${passwordstoreDir}/.git; then + local_pass_info=$( + git -C ${passwordstoreDir} log -1 --format=%H machines/${config.clanCore.machineName} + # we append a hash for every symlink, otherwise we would miss updates on + # files where the symlink points to + find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type l \ + -exec realpath {} + | + sort | + xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H + ) + remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg '' + cat ${config.clan.password-store.targetDirectory}/.pass_info || : + ''}) + + if test "$local_pass_info" = "$remote_pass_info"; then + echo secrets already match + exit 0 + fi + fi + + tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX) + trap cleanup EXIT + cleanup() { + rm -fR "$tmp_dir" + } + + find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id | + while read -r gpg_path; do + + rel_name=''${gpg_path#${passwordstoreDir}} + rel_name=''${rel_name%.gpg} + + pass_date=$( + if test -e ${passwordstoreDir}/.git; then + git -C ${passwordstoreDir} log -1 --format=%aI "$gpg_path" + fi + ) + pass_name=$rel_name + tmp_path=$tmp_dir/$(basename $rel_name) + + mkdir -p "$(dirname "$tmp_path")" + pass show "$pass_name" > "$tmp_path" + if [ -n "$pass_date" ]; then + touch -d "$pass_date" "$tmp_path" + fi + done + + if test -n "''${local_pass_info-}"; then + echo "$local_pass_info" > "$tmp_dir"/.pass_info + fi + + rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/ + ''; + }; +} + From 798e85ee8a36fa10a654afec7a889d8e89f260c5 Mon Sep 17 00:00:00 2001 From: lassulus Date: Sat, 9 Sep 2023 15:38:28 +0200 Subject: [PATCH 12/44] clan secrets generate: use get_clan_flake_toplevel --- pkgs/clan-cli/clan_cli/secrets/generate.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 0b01a8c..2c0fb9a 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -4,8 +4,11 @@ import sys from clan_cli.errors import ClanError +from ..dirs import get_clan_flake_toplevel -def get_secret_script(machine: str) -> None: + +def generate_secrets(machine: str) -> None: + clan_flake = get_clan_flake_toplevel() proc = subprocess.run( [ "nix", @@ -13,12 +16,13 @@ def get_secret_script(machine: str) -> None: "--impure", "--print-out-paths", "--expr", - "let f = builtins.getFlake (toString ./.); in " - f"(f.nixosConfigurations.{machine}.extendModules " - "{ modules = [{ clanCore.clanDir = toString ./.; }]; })" - ".config.system.clan.generateSecrets", + f'let f = builtins.getFlake "{clan_flake}"; in ' + "(f.nixosConfigurations." + f"{machine}" + ".extendModules { modules = [{ clanCore.clanDir = " + f"{clan_flake}" + "; }]; }).config.system.clan.generateSecrets", ], - check=True, capture_output=True, text=True, ) @@ -30,7 +34,6 @@ def get_secret_script(machine: str) -> None: print(secret_generator_script) secret_generator = subprocess.run( [secret_generator_script], - check=True, ) if secret_generator.returncode != 0: @@ -40,7 +43,7 @@ def get_secret_script(machine: str) -> None: def generate_command(args: argparse.Namespace) -> None: - get_secret_script(args.machine) + generate_secrets(args.machine) def register_generate_parser(parser: argparse.ArgumentParser) -> None: From 285041026da56463d697c9ff8e4ca850b79e587a Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 11 Sep 2023 21:42:45 +0200 Subject: [PATCH 13/44] clanCore sops: add dummy deployScript --- nixosModules/clanCore/secrets/sops.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index 2371489..f209d53 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -64,6 +64,9 @@ in fi) '') "" config.clanCore.secrets} ''; + system.clan.deploySecrets = pkgs.writeScript "deploy-secrets" '' + echo deployment is not needed for sops secret store, since the secrets are part of the flake + ''; sops.secrets = builtins.mapAttrs (name: _: { sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; From ac13c5b76be4a4ef6a98eb61c9e345eea13edbfe Mon Sep 17 00:00:00 2001 From: lassulus Date: Mon, 11 Sep 2023 21:44:39 +0200 Subject: [PATCH 14/44] clan-cli secrets: add deploy subcommand --- pkgs/clan-cli/clan_cli/secrets/__init__.py | 4 ++ pkgs/clan-cli/clan_cli/secrets/deploy.py | 53 ++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 pkgs/clan-cli/clan_cli/secrets/deploy.py diff --git a/pkgs/clan-cli/clan_cli/secrets/__init__.py b/pkgs/clan-cli/clan_cli/secrets/__init__.py index 3c161d3..4515bcf 100644 --- a/pkgs/clan-cli/clan_cli/secrets/__init__.py +++ b/pkgs/clan-cli/clan_cli/secrets/__init__.py @@ -1,6 +1,7 @@ # !/usr/bin/env python3 import argparse +from .deploy import register_deploy_parser from .generate import register_generate_parser from .groups import register_groups_parser from .import_sops import register_import_sops_parser @@ -36,6 +37,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None: ) register_generate_parser(parser_generate) + parser_deploy = subparser.add_parser("deploy", help="deploy secrets for machines") + register_deploy_parser(parser_deploy) + parser_key = subparser.add_parser("key", help="create and show age keys") register_key_parser(parser_key) diff --git a/pkgs/clan-cli/clan_cli/secrets/deploy.py b/pkgs/clan-cli/clan_cli/secrets/deploy.py new file mode 100644 index 0000000..16c1a1c --- /dev/null +++ b/pkgs/clan-cli/clan_cli/secrets/deploy.py @@ -0,0 +1,53 @@ +import argparse +import subprocess +import sys + +from clan_cli.errors import ClanError + +from ..dirs import get_clan_flake_toplevel + + +def deploy_secrets(machine: str) -> None: + clan_flake = get_clan_flake_toplevel() + proc = subprocess.run( + [ + "nix", + "build", + "--impure", + "--print-out-paths", + "--expr", + f'let f = builtins.getFlake "{clan_flake}"; in ' + "(f.nixosConfigurations." + f"{machine}" + ".extendModules { modules = [{ clanCore.clanDir = " + f"{clan_flake}" + "; }]; }).config.system.clan.deploySecrets", + ], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(proc.stderr, file=sys.stderr) + raise ClanError(f"failed to deploy secrets:\n{proc.stderr}") + + secret_deploy_script = proc.stdout.strip() + secret_deploy = subprocess.run( + [secret_deploy_script], + ) + + if secret_deploy.returncode != 0: + raise ClanError("failed to deploy secrets") + else: + print("successfully deployed secrets") + + +def deploy_command(args: argparse.Namespace) -> None: + deploy_secrets(args.machine) + + +def register_deploy_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "machine", + help="The machine to deploy secrets to", + ) + parser.set_defaults(func=deploy_command) From 3b0701f275e9edc83e2305fec438d999075b4551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 13 Sep 2023 20:33:51 +0200 Subject: [PATCH 15/44] deploy: use nix-flake-archive instead of rsync to upload --- pkgs/clan-cli/clan_cli/machines/update.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index bed009f..e6f7dcf 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -1,5 +1,6 @@ import argparse import json +import os import subprocess from ..ssh import Host, HostGroup, HostKeyCheck @@ -13,11 +14,13 @@ def deploy_nixos(hosts: HostGroup) -> None: def deploy(h: Host) -> None: target = f"{h.user or 'root'}@{h.host}" ssh_arg = f"-p {h.port}" if h.port else "" + env = os.environ.copy() + env["NIX_SSHOPTS"] = ssh_arg res = h.run_local( ["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"], check=True, stdout=subprocess.PIPE, - extra_env=dict(NIX_SSHOPTS=ssh_arg), + extra_env=env ) data = json.loads(res.stdout) path = data["path"] From 6153a9ee71a3b1d53cb078865fac6323d1ff8f1c Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 13 Sep 2023 23:16:56 +0200 Subject: [PATCH 16/44] clanCore.secrets: set default and add generate/deploy composite --- nixosModules/clanCore/secrets/default.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 51c6aa3..6220976 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -1,4 +1,4 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: { options.clanCore.secretStore = lib.mkOption { type = lib.types.enum [ "sops" "password-store" "custom" ]; @@ -8,6 +8,7 @@ ''; }; options.clanCore.secrets = lib.mkOption { + default = { }; type = lib.types.attrsOf (lib.types.submodule (secret: { options = { @@ -67,6 +68,10 @@ }; })); }; + config.system.build.generateDeploySecrets = pkgs.writeScript "generate_deploy_secrets" '' + ${config.system.build.generateSecrets} + ${config.system.build.deploySecrets} + ''; imports = [ ./sops.nix ./password-store.nix From 23c979f8db79d1580b561faed209b31fe28ffc8c Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 13 Sep 2023 23:18:05 +0200 Subject: [PATCH 17/44] secrets deploy/generate: use nix_build_machine --- pkgs/clan-cli/clan_cli/secrets/deploy.py | 30 ++++++++++------------ pkgs/clan-cli/clan_cli/secrets/generate.py | 25 ++++++++---------- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/secrets/deploy.py b/pkgs/clan-cli/clan_cli/secrets/deploy.py index 16c1a1c..50fc984 100644 --- a/pkgs/clan-cli/clan_cli/secrets/deploy.py +++ b/pkgs/clan-cli/clan_cli/secrets/deploy.py @@ -4,25 +4,20 @@ import sys from clan_cli.errors import ClanError -from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build_machine def deploy_secrets(machine: str) -> None: - clan_flake = get_clan_flake_toplevel() proc = subprocess.run( - [ - "nix", - "build", - "--impure", - "--print-out-paths", - "--expr", - f'let f = builtins.getFlake "{clan_flake}"; in ' - "(f.nixosConfigurations." - f"{machine}" - ".extendModules { modules = [{ clanCore.clanDir = " - f"{clan_flake}" - "; }]; }).config.system.clan.deploySecrets", - ], + nix_build_machine( + machine=machine, + attr=[ + "config", + "system", + "clan", + "deploySecrets", + ], + ), capture_output=True, text=True, ) @@ -32,7 +27,10 @@ def deploy_secrets(machine: str) -> None: secret_deploy_script = proc.stdout.strip() secret_deploy = subprocess.run( - [secret_deploy_script], + [ + secret_deploy_script, + f"root@{machine}", + ], ) if secret_deploy.returncode != 0: diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 2c0fb9a..3093eee 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -4,25 +4,20 @@ import sys from clan_cli.errors import ClanError -from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build_machine def generate_secrets(machine: str) -> None: - clan_flake = get_clan_flake_toplevel() proc = subprocess.run( - [ - "nix", - "build", - "--impure", - "--print-out-paths", - "--expr", - f'let f = builtins.getFlake "{clan_flake}"; in ' - "(f.nixosConfigurations." - f"{machine}" - ".extendModules { modules = [{ clanCore.clanDir = " - f"{clan_flake}" - "; }]; }).config.system.clan.generateSecrets", - ], + nix_build_machine( + machine=machine, + attr=[ + "config", + "system", + "clan", + "generateSecrets", + ], + ), capture_output=True, text=True, ) From 0e3f8bb3f9fea30f03e3117ac62ff99d0de29b62 Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 13 Sep 2023 23:18:54 +0200 Subject: [PATCH 18/44] clan-cli nix_build_machine: cast flake_url to str --- pkgs/clan-cli/clan_cli/nix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 559b823..6d08a33 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -10,7 +10,7 @@ def nix_build_machine( machine: str, attr: list[str], flake_url: Path | None = None ) -> list[str]: if flake_url is None: - flake_url = get_clan_flake_toplevel() + flake_url = str(get_clan_flake_toplevel()) payload = json.dumps( dict( clan_flake=flake_url, From c487280ba9a095795bbc9bb1a1e564b9e61b63ec Mon Sep 17 00:00:00 2001 From: lassulus Date: Wed, 13 Sep 2023 23:19:39 +0200 Subject: [PATCH 19/44] clan-cli machines update: generate and deploy secrets --- pkgs/clan-cli/clan_cli/machines/update.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index e6f7dcf..f2d2df5 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -4,6 +4,8 @@ import os import subprocess from ..ssh import Host, HostGroup, HostKeyCheck +from ..secrets.deploy import deploy_secrets +from ..secrets.generate import generate_secrets def deploy_nixos(hosts: HostGroup) -> None: @@ -32,6 +34,9 @@ def deploy_nixos(hosts: HostGroup) -> None: ssh_arg += " -i " + h.key if h.key else "" + generate_secrets(h.host) + deploy_secrets(h.host) + flake_attr = h.meta.get("flake_attr", "") if flake_attr: flake_attr = "#" + flake_attr From acf1c0b87ae6da3f59a3d0453cd8a9eef0b801eb Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 10:37:16 +0200 Subject: [PATCH 20/44] lib jsonschema: add path --- lib/jsonschema/default.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix index 26c399e..90a8fb2 100644 --- a/lib/jsonschema/default.nix +++ b/lib/jsonschema/default.nix @@ -7,6 +7,7 @@ let float = "number"; int = "integer"; str = "string"; + path = "string"; # TODO add prober path checks }; # remove _module attribute from options @@ -103,6 +104,13 @@ rec { type = "string"; } + # parse string + else if option.type.name == "path" + # return jsonschema property definition for path + then default // description // { + type = "string"; + } + # parse enum else if option.type.name == "enum" # return jsonschema property definition for enum From 55fc055549028cecefa8280b1bae85b95b215da7 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 10:42:04 +0200 Subject: [PATCH 21/44] clan_cli/nix: convert path to string --- pkgs/clan-cli/clan_cli/nix.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 6d08a33..06b564f 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -10,10 +10,10 @@ def nix_build_machine( machine: str, attr: list[str], flake_url: Path | None = None ) -> list[str]: if flake_url is None: - flake_url = str(get_clan_flake_toplevel()) + flake_url = get_clan_flake_toplevel() payload = json.dumps( dict( - clan_flake=flake_url, + clan_flake=flake_url.as_posix(), machine=machine, attr=attr, ) From c5786614bf7e82c26d085bf4e0baef39cb4f080f Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 13:49:20 +0200 Subject: [PATCH 22/44] clan-cli secrets: deploy -> upload --- nixosModules/clanCore/secrets/default.nix | 6 +-- .../clanCore/secrets/password-store.nix | 4 +- nixosModules/clanCore/secrets/sops.nix | 4 +- pkgs/clan-cli/clan_cli/machines/update.py | 4 +- pkgs/clan-cli/clan_cli/secrets/__init__.py | 6 +-- pkgs/clan-cli/clan_cli/secrets/deploy.py | 51 ------------------- pkgs/clan-cli/clan_cli/secrets/upload.py | 51 +++++++++++++++++++ 7 files changed, 63 insertions(+), 63 deletions(-) delete mode 100644 pkgs/clan-cli/clan_cli/secrets/deploy.py create mode 100644 pkgs/clan-cli/clan_cli/secrets/upload.py diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 6220976..f3e7b5e 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -68,9 +68,9 @@ }; })); }; - config.system.build.generateDeploySecrets = pkgs.writeScript "generate_deploy_secrets" '' - ${config.system.build.generateSecrets} - ${config.system.build.deploySecrets} + config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" '' + ${config.system.clan.generateSecrets} + ${config.system.clan.uploadSecrets} ''; imports = [ ./sops.nix diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix index 7b559a5..bc11b9e 100644 --- a/nixosModules/clanCore/secrets/password-store.nix +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -7,7 +7,7 @@ in type = lib.types.path; default = "/etc/secrets"; description = '' - The directory where the password store is deployed to. + The directory where the password store is uploaded to. ''; }; config = lib.mkIf (config.clanCore.secretStore == "password-store") { @@ -45,7 +45,7 @@ in fi) '') "" config.clanCore.secrets} ''; - system.clan.deploySecrets = pkgs.writeScript "deploy-secrets" '' + system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" '' #!/bin/sh set -efu set -x # remove for prod diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index f209d53..5fe455d 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -64,8 +64,8 @@ in fi) '') "" config.clanCore.secrets} ''; - system.clan.deploySecrets = pkgs.writeScript "deploy-secrets" '' - echo deployment is not needed for sops secret store, since the secrets are part of the flake + system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" '' + echo upload is not needed for sops secret store, since the secrets are part of the flake ''; sops.secrets = builtins.mapAttrs (name: _: { diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index f2d2df5..30726b6 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -4,7 +4,7 @@ import os import subprocess from ..ssh import Host, HostGroup, HostKeyCheck -from ..secrets.deploy import deploy_secrets +from ..secrets.upload import upload_secrets from ..secrets.generate import generate_secrets @@ -35,7 +35,7 @@ def deploy_nixos(hosts: HostGroup) -> None: ssh_arg += " -i " + h.key if h.key else "" generate_secrets(h.host) - deploy_secrets(h.host) + upload_secrets(h.host) flake_attr = h.meta.get("flake_attr", "") if flake_attr: diff --git a/pkgs/clan-cli/clan_cli/secrets/__init__.py b/pkgs/clan-cli/clan_cli/secrets/__init__.py index 4515bcf..01ac958 100644 --- a/pkgs/clan-cli/clan_cli/secrets/__init__.py +++ b/pkgs/clan-cli/clan_cli/secrets/__init__.py @@ -1,13 +1,13 @@ # !/usr/bin/env python3 import argparse -from .deploy import register_deploy_parser from .generate import register_generate_parser from .groups import register_groups_parser from .import_sops import register_import_sops_parser from .key import register_key_parser from .machines import register_machines_parser from .secrets import register_secrets_parser +from .upload import register_upload_parser from .users import register_users_parser @@ -37,8 +37,8 @@ def register_parser(parser: argparse.ArgumentParser) -> None: ) register_generate_parser(parser_generate) - parser_deploy = subparser.add_parser("deploy", help="deploy secrets for machines") - register_deploy_parser(parser_deploy) + parser_upload = subparser.add_parser("upload", help="upload secrets for machines") + register_upload_parser(parser_upload) parser_key = subparser.add_parser("key", help="create and show age keys") register_key_parser(parser_key) diff --git a/pkgs/clan-cli/clan_cli/secrets/deploy.py b/pkgs/clan-cli/clan_cli/secrets/deploy.py deleted file mode 100644 index 50fc984..0000000 --- a/pkgs/clan-cli/clan_cli/secrets/deploy.py +++ /dev/null @@ -1,51 +0,0 @@ -import argparse -import subprocess -import sys - -from clan_cli.errors import ClanError - -from ..nix import nix_build_machine - - -def deploy_secrets(machine: str) -> None: - proc = subprocess.run( - nix_build_machine( - machine=machine, - attr=[ - "config", - "system", - "clan", - "deploySecrets", - ], - ), - capture_output=True, - text=True, - ) - if proc.returncode != 0: - print(proc.stderr, file=sys.stderr) - raise ClanError(f"failed to deploy secrets:\n{proc.stderr}") - - secret_deploy_script = proc.stdout.strip() - secret_deploy = subprocess.run( - [ - secret_deploy_script, - f"root@{machine}", - ], - ) - - if secret_deploy.returncode != 0: - raise ClanError("failed to deploy secrets") - else: - print("successfully deployed secrets") - - -def deploy_command(args: argparse.Namespace) -> None: - deploy_secrets(args.machine) - - -def register_deploy_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "machine", - help="The machine to deploy secrets to", - ) - parser.set_defaults(func=deploy_command) diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py new file mode 100644 index 0000000..344a006 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -0,0 +1,51 @@ +import argparse +import subprocess +import sys + +from clan_cli.errors import ClanError + +from ..nix import nix_build_machine + + +def upload_secrets(machine: str) -> None: + proc = subprocess.run( + nix_build_machine( + machine=machine, + attr=[ + "config", + "system", + "clan", + "uploadSecrets", + ], + ), + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(proc.stderr, file=sys.stderr) + raise ClanError(f"failed to upload secrets:\n{proc.stderr}") + + secret_upload_script = proc.stdout.strip() + secret_upload = subprocess.run( + [ + secret_upload_script, + f"root@{machine}", + ], + ) + + if secret_upload.returncode != 0: + raise ClanError("failed to upload secrets") + else: + print("successfully uploaded secrets") + + +def upload_command(args: argparse.Namespace) -> None: + upload_secrets(args.machine) + + +def register_upload_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "machine", + help="The machine to upload secrets to", + ) + parser.set_defaults(func=upload_command) From c5c2a848c7ab36809728b531a5e8cc25b70fdeae Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 13:49:57 +0200 Subject: [PATCH 23/44] secrets pass: append openssh to PATH so we use systems openssh first --- nixosModules/clanCore/secrets/password-store.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix index bc11b9e..0cb5873 100644 --- a/nixosModules/clanCore/secrets/password-store.nix +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -58,9 +58,8 @@ in pkgs.pass pkgs.git pkgs.findutils - pkgs.openssh pkgs.rsync - ]}:$PATH + ]}:$PATH:${lib.getBin pkgs.openssh} if test -e ${passwordstoreDir}/.git; then local_pass_info=$( From a59e8478fac17d9ca4c9c97dff506008793fc87c Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 13:51:13 +0200 Subject: [PATCH 24/44] clan-cli nix_build_machine: don't create result link --- pkgs/clan-cli/clan_cli/nix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 06b564f..212b55b 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -22,6 +22,7 @@ def nix_build_machine( return [ "nix", "build", + "--no-link", "--impure", "--print-out-paths", "--expr", From 6b7301cefbd406c0246804f160687f6f712d1e49 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 14:15:51 +0200 Subject: [PATCH 25/44] clanCore secrets: document custom store --- nixosModules/clanCore/secrets/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index f3e7b5e..e8c115e 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -5,6 +5,8 @@ default = "sops"; description = '' method to store secrets + custom can be used to define a custom secret store. + one would have to define system.clan.generateSecrets and system.clan.uploadSecrets ''; }; options.clanCore.secrets = lib.mkOption { From 0132abc547f432f4ba1ee7b92fa027c450e3dae2 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 15:22:13 +0200 Subject: [PATCH 26/44] secrets: use CLAN_DIR instead of clanCore.clanDir for fact storage --- nixosModules/clanCore/secrets/default.nix | 4 +-- .../clanCore/secrets/password-store.nix | 7 ++-- nixosModules/clanCore/secrets/sops.nix | 5 +-- pkgs/clan-cli/clan_cli/nix.py | 34 +++---------------- pkgs/clan-cli/clan_cli/secrets/generate.py | 21 +++++++----- pkgs/clan-cli/clan_cli/secrets/upload.py | 17 +++++----- 6 files changed, 32 insertions(+), 56 deletions(-) diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index e8c115e..700f5d0 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -59,10 +59,10 @@ description = '' path to a fact which is generated by the generator ''; - default = "${config.clanCore.clanDir}/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; + default = "machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; }; value = lib.mkOption { - default = builtins.readFile fact.config.path; + default = builtins.readFile "${config.clanCore.clanDir}/fact.config.path"; }; }; })); diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix index 0cb5873..3db20e0 100644 --- a/nixosModules/clanCore/secrets/password-store.nix +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -1,6 +1,6 @@ { config, lib, pkgs, ... }: let - passwordstoreDir = "$HOME/.password-store"; + passwordstoreDir = "\${PASSWORD_STORE_DIR:-$HOME/.password-store}"; in { options.clan.password-store.targetDirectory = lib.mkOption { @@ -14,8 +14,8 @@ in system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' #!/bin/sh set -efu - set -x # remove for prod + test -d "$CLAN_DIR" PATH=${lib.makeBinPath [ pkgs.pass ]}:$PATH @@ -36,7 +36,7 @@ in ${lib.concatMapStrings (fact: '' mkdir -p "$(dirname ${fact.path})" - cp "$facts"/${fact.name} ${fact.path} + cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path} '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' @@ -48,7 +48,6 @@ in system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" '' #!/bin/sh set -efu - set -x # remove for prod target=$1 diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index 5fe455d..25d0af6 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -25,7 +25,8 @@ in system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' #!/bin/sh set -efu - set -x # remove for prod + + test -d "$CLAN_DIR" PATH=$PATH:${lib.makeBinPath [ config.clanCore.clanPkgs.clan-cli @@ -55,7 +56,7 @@ in ${lib.concatMapStrings (fact: '' mkdir -p "$(dirname ${fact.path})" - cp "$facts"/${fact.name} ${fact.path} + cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path} '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 212b55b..0a34fac 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -1,44 +1,18 @@ -import json import os import tempfile -from pathlib import Path -from .dirs import get_clan_flake_toplevel, nixpkgs_flake, nixpkgs_source, unfree_nixpkgs +from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs -def nix_build_machine( - machine: str, attr: list[str], flake_url: Path | None = None +def nix_build( + flags: list[str], ) -> list[str]: - if flake_url is None: - flake_url = get_clan_flake_toplevel() - payload = json.dumps( - dict( - clan_flake=flake_url.as_posix(), - machine=machine, - attr=attr, - ) - ) - escaped_payload = json.dumps(payload) return [ "nix", "build", "--no-link", - "--impure", "--print-out-paths", - "--expr", - f"let args = builtins.fromJSON {escaped_payload}; in " - """ - let - flake = builtins.getFlake args.clan_flake; - config = flake.nixosConfigurations.${args.machine}.extendModules { - modules = [{ - clanCore.clanDir = args.clan_flake; - }]; - }; - in - flake.inputs.nixpkgs.lib.getAttrFromPath args.attr config - """, - ] + ] + flags def nix_eval(flags: list[str]) -> list[str]: diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 3093eee..782bb55 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -1,22 +1,24 @@ import argparse +import os import subprocess import sys from clan_cli.errors import ClanError -from ..nix import nix_build_machine +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build def generate_secrets(machine: str) -> None: + clan_dir = get_clan_flake_toplevel().as_posix().strip() + env = os.environ.copy() + env["CLAN_DIR"] = clan_dir + proc = subprocess.run( - nix_build_machine( - machine=machine, - attr=[ - "config", - "system", - "clan", - "generateSecrets", - ], + nix_build( + [ + f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets' + ] ), capture_output=True, text=True, @@ -29,6 +31,7 @@ def generate_secrets(machine: str) -> None: print(secret_generator_script) secret_generator = subprocess.run( [secret_generator_script], + env=env, ) if secret_generator.returncode != 0: diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index 344a006..8dc4afe 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -4,19 +4,18 @@ import sys from clan_cli.errors import ClanError -from ..nix import nix_build_machine +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build def upload_secrets(machine: str) -> None: + clan_dir = get_clan_flake_toplevel().as_posix() + proc = subprocess.run( - nix_build_machine( - machine=machine, - attr=[ - "config", - "system", - "clan", - "uploadSecrets", - ], + nix_build( + [ + f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets' + ] ), capture_output=True, text=True, From 8d29d0e69c4f3cd93ee2ca69d2d2d971ddb150b3 Mon Sep 17 00:00:00 2001 From: lassulus Date: Thu, 14 Sep 2023 16:57:38 +0200 Subject: [PATCH 27/44] clan-cli: get deploymentAddress from clan.networking --- nixosModules/clanCore/flake-module.nix | 1 + nixosModules/clanCore/networking.nix | 15 +++++++ pkgs/clan-cli/clan_cli/machines/update.py | 51 ++++++++++++++++------- pkgs/clan-cli/clan_cli/secrets/upload.py | 24 +++++++---- 4 files changed, 68 insertions(+), 23 deletions(-) create mode 100644 nixosModules/clanCore/networking.nix diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/clanCore/flake-module.nix index da174ff..868a523 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/clanCore/flake-module.nix @@ -3,6 +3,7 @@ imports = [ ./secrets ./zerotier.nix + ./networking.nix inputs.sops-nix.nixosModules.sops # just some example options. Can be removed later ./bloatware diff --git a/nixosModules/clanCore/networking.nix b/nixosModules/clanCore/networking.nix new file mode 100644 index 0000000..813939d --- /dev/null +++ b/nixosModules/clanCore/networking.nix @@ -0,0 +1,15 @@ +{ config, lib, ... }: +{ + options.clan.networking = { + deploymentAddress = lib.mkOption { + description = '' + The target SSH node for deployment. + + By default, the node's attribute name will be used. + If set to null, only local deployment will be supported. + ''; + type = lib.types.nullOr lib.types.str; + default = "root@${config.networking.hostName}"; + }; + }; +} diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 30726b6..963b0a1 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -2,10 +2,13 @@ import argparse import json import os import subprocess +from typing import Optional -from ..ssh import Host, HostGroup, HostKeyCheck -from ..secrets.upload import upload_secrets +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_eval from ..secrets.generate import generate_secrets +from ..secrets.upload import upload_secrets +from ..ssh import Host, HostGroup, HostKeyCheck def deploy_nixos(hosts: HostGroup) -> None: @@ -22,7 +25,7 @@ def deploy_nixos(hosts: HostGroup) -> None: ["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"], check=True, stdout=subprocess.PIPE, - extra_env=env + extra_env=env, ) data = json.loads(res.stdout) path = data["path"] @@ -75,20 +78,36 @@ def deploy_nixos(hosts: HostGroup) -> None: # FIXME: we want some kind of inventory here. def update(args: argparse.Namespace) -> None: - meta = {} - if args.flake_uri: - meta["flake_uri"] = args.flake_uri - if args.flake_attr: - meta["flake_attr"] = args.flake_attr - deploy_nixos(HostGroup([Host(args.host, user=args.user, meta=meta)])) + clan_dir = get_clan_flake_toplevel().as_posix() + host = json.loads( + subprocess.run( + nix_eval( + [ + f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress' + ] + ), + stdout=subprocess.PIPE, + check=True, + text=True, + ).stdout + ) + parts = host.split("@") + user: Optional[str] = None + if len(parts) > 1: + user = parts[0] + hostname = parts[1] + else: + hostname = parts[0] + maybe_port = hostname.split(":") + port = None + if len(maybe_port) > 1: + hostname = maybe_port[0] + port = int(maybe_port[1]) + print(f"deploying {host}") + deploy_nixos(HostGroup([Host(host=hostname, port=port, user=user)])) def register_update_parser(parser: argparse.ArgumentParser) -> None: - # TODO pass all args we don't parse into ssh_args, currently it fails if arg starts with - - parser.add_argument("--flake-uri", type=str, default=".#", help="nix flake uri") - parser.add_argument( - "--flake-attr", type=str, help="nixos configuration in the flake" - ) - parser.add_argument("--user", type=str, default="root") - parser.add_argument("host", type=str) + parser.add_argument("--target-host", type=str, default="root") + parser.add_argument("machine", type=str) parser.set_defaults(func=update) diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py index 8dc4afe..38a27b9 100644 --- a/pkgs/clan-cli/clan_cli/secrets/upload.py +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -1,11 +1,11 @@ import argparse +import json import subprocess -import sys from clan_cli.errors import ClanError from ..dirs import get_clan_flake_toplevel -from ..nix import nix_build +from ..nix import nix_build, nix_eval def upload_secrets(machine: str) -> None: @@ -17,18 +17,28 @@ def upload_secrets(machine: str) -> None: f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets' ] ), - capture_output=True, + stdout=subprocess.PIPE, text=True, + check=True, + ) + host = json.loads( + subprocess.run( + nix_eval( + [ + f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress' + ] + ), + stdout=subprocess.PIPE, + text=True, + check=True, + ).stdout ) - if proc.returncode != 0: - print(proc.stderr, file=sys.stderr) - raise ClanError(f"failed to upload secrets:\n{proc.stderr}") secret_upload_script = proc.stdout.strip() secret_upload = subprocess.run( [ secret_upload_script, - f"root@{machine}", + host, ], ) From 8a96254100cade13ae079247c6c3ea11056dbf4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 14:21:15 +0200 Subject: [PATCH 28/44] nix_build: add experimental flags --- pkgs/clan-cli/clan_cli/nix.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 0a34fac..e647903 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -12,6 +12,8 @@ def nix_build( "build", "--no-link", "--print-out-paths", + "--extra-experimental-features", + "nix-command flakes", ] + flags From a29f301f84d631a73882e5edaf6af7b60e773b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 14 Sep 2023 16:57:19 +0200 Subject: [PATCH 29/44] add vms --- nixosModules/clanCore/flake-module.nix | 1 + nixosModules/clanCore/vm.nix | 8 ++ pkgs/clan-cli/clan_cli/webui/app.py | 4 +- pkgs/clan-cli/clan_cli/webui/routers/vms.py | 113 ++++++++++++++++++ pkgs/clan-cli/clan_cli/webui/schemas.py | 13 ++ pkgs/clan-cli/default.nix | 3 +- .../tests/test_flake_with_core/.clan-flake | 0 .../tests/test_flake_with_core/flake.nix | 15 +++ pkgs/clan-cli/tests/test_vms_api.py | 33 +++++ 9 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 nixosModules/clanCore/vm.nix create mode 100644 pkgs/clan-cli/clan_cli/webui/routers/vms.py create mode 100644 pkgs/clan-cli/tests/test_flake_with_core/.clan-flake create mode 100644 pkgs/clan-cli/tests/test_flake_with_core/flake.nix create mode 100644 pkgs/clan-cli/tests/test_vms_api.py diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/clanCore/flake-module.nix index 868a523..0fcf4cd 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/clanCore/flake-module.nix @@ -7,6 +7,7 @@ inputs.sops-nix.nixosModules.sops # just some example options. Can be removed later ./bloatware + ./vm.nix ]; options.clanSchema = lib.mkOption { type = lib.types.attrs; diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix new file mode 100644 index 0000000..4382c8c --- /dev/null +++ b/nixosModules/clanCore/vm.nix @@ -0,0 +1,8 @@ +{ config, options, lib, ... }: { + system.clan.vm.config = { + enabled = options.virtualisation ? cores; + } // (lib.optionalAttrs (options.virtualisation ? cores) { + inherit (config.virtualisation) cores graphics; + memory_size = config.virtualisation.memorySize; + }); +} diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index e839654..76f5809 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -3,7 +3,7 @@ from fastapi.routing import APIRoute from fastapi.staticfiles import StaticFiles from .assets import asset_path -from .routers import health, machines, root +from .routers import health, machines, root, vms def setup_app() -> FastAPI: @@ -11,6 +11,8 @@ def setup_app() -> FastAPI: app.include_router(health.router) app.include_router(machines.router) app.include_router(root.router) + app.include_router(vms.router) + app.add_exception_handler(vms.NixBuildException, vms.nix_build_exception_handler) app.mount("/static", StaticFiles(directory=asset_path()), name="static") diff --git a/pkgs/clan-cli/clan_cli/webui/routers/vms.py b/pkgs/clan-cli/clan_cli/webui/routers/vms.py new file mode 100644 index 0000000..18c8c74 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/webui/routers/vms.py @@ -0,0 +1,113 @@ +import asyncio +import json +import shlex +from typing import Annotated, AsyncIterator + +from fastapi import APIRouter, Body, HTTPException, Request, status +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse, StreamingResponse + +from ...nix import nix_build, nix_eval +from ..schemas import VmConfig, VmInspectResponse + +router = APIRouter() + + +class NixBuildException(HTTPException): + def __init__(self, msg: str, loc: list = ["body", "flake_attr"]): + detail = [ + { + "loc": loc, + "msg": msg, + "type": "value_error", + } + ] + super().__init__( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail + ) + + +def nix_build_exception_handler( + request: Request, exc: NixBuildException +) -> JSONResponse: + return JSONResponse( + status_code=exc.status_code, + content=jsonable_encoder(dict(detail=exc.detail)), + ) + + +def nix_inspect_vm(machine: str, flake_url: str) -> list[str]: + return nix_eval( + [ + f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.clan.vm.config" + ] + ) + + +def nix_build_vm(machine: str, flake_url: str) -> list[str]: + return nix_build( + [ + f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.build.vm" + ] + ) + + +@router.post("/api/vms/inspect") +async def inspect_vm( + flake_url: Annotated[str, Body()], flake_attr: Annotated[str, Body()] +) -> VmInspectResponse: + cmd = nix_inspect_vm(flake_attr, flake_url=flake_url) + proc = await asyncio.create_subprocess_exec( + cmd[0], + *cmd[1:], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + + if proc.returncode != 0: + raise NixBuildException( + f""" +Failed to evaluate vm from '{flake_url}#{flake_attr}'. +command: {shlex.join(cmd)} +exit code: {proc.returncode} +command output: +{stderr.decode("utf-8")} +""" + ) + data = json.loads(stdout) + return VmInspectResponse( + config=VmConfig(flake_url=flake_url, flake_attr=flake_attr, **data) + ) + + +async def vm_build(vm: VmConfig) -> AsyncIterator[str]: + cmd = nix_build_vm(vm.flake_attr, flake_url=vm.flake_url) + proc = await asyncio.create_subprocess_exec( + cmd[0], + *cmd[1:], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert proc.stdout is not None and proc.stderr is not None + async for line in proc.stdout: + yield line.decode("utf-8", "ignore") + stderr = "" + async for line in proc.stderr: + stderr += line.decode("utf-8", "ignore") + res = await proc.wait() + if res != 0: + raise NixBuildException( + f""" +Failed to build vm from '{vm.flake_url}#{vm.flake_attr}'. +command: {shlex.join(cmd)} +exit code: {res} +command output: +{stderr} + """ + ) + + +@router.post("/api/vms/create") +async def create_vm(vm: Annotated[VmConfig, Body()]) -> StreamingResponse: + return StreamingResponse(vm_build(vm)) diff --git a/pkgs/clan-cli/clan_cli/webui/schemas.py b/pkgs/clan-cli/clan_cli/webui/schemas.py index 90c5437..dc6ea3f 100644 --- a/pkgs/clan-cli/clan_cli/webui/schemas.py +++ b/pkgs/clan-cli/clan_cli/webui/schemas.py @@ -32,3 +32,16 @@ class ConfigResponse(BaseModel): class SchemaResponse(BaseModel): schema_: dict = Field(alias="schema") + + +class VmConfig(BaseModel): + flake_url: str + flake_attr: str + + cores: int + memory_size: int + graphics: bool + + +class VmInspectResponse(BaseModel): + config: VmConfig diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index c73810b..1f43eb4 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -98,7 +98,8 @@ python3.pkgs.buildPythonPackage { chmod +w -R ./src cd ./src - NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 ${checkPython}/bin/python -m pytest -s ./tests + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests touch $out ''; passthru.clan-openapi = runCommand "clan-openapi" { } '' diff --git a/pkgs/clan-cli/tests/test_flake_with_core/.clan-flake b/pkgs/clan-cli/tests/test_flake_with_core/.clan-flake new file mode 100644 index 0000000..e69de29 diff --git a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix new file mode 100644 index 0000000..0a36a13 --- /dev/null +++ b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix @@ -0,0 +1,15 @@ +{ + # this placeholder is replaced by the path to nixpkgs + inputs.clan-core.url = "__CLAN_CORE__"; + + outputs = { self, clan-core }: { + nixosConfigurations = clan-core.lib.buildClan { + directory = self; + machines = { + vm1 = { modulesPath, ... }: { + imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ]; + }; + }; + }; + }; +} diff --git a/pkgs/clan-cli/tests/test_vms_api.py b/pkgs/clan-cli/tests/test_vms_api.py new file mode 100644 index 0000000..8935e6c --- /dev/null +++ b/pkgs/clan-cli/tests/test_vms_api.py @@ -0,0 +1,33 @@ +from pathlib import Path + +import pytest +from api import TestClient + + +@pytest.mark.impure +def test_inspect(api: TestClient, test_flake_with_core: Path) -> None: + response = api.post( + "/api/vms/inspect", + json=dict(flake_url=str(test_flake_with_core), flake_attr="vm1"), + ) + assert response.status_code == 200, "Failed to inspect vm" + config = response.json()["config"] + assert config.get("flake_attr") == "vm1" + assert config.get("cores") == 1 + assert config.get("memory_size") == 1024 + assert config.get("graphics") is True + + +@pytest.mark.impure +def test_create(api: TestClient, test_flake_with_core: Path) -> None: + response = api.post( + "/api/vms/create", + json=dict( + flake_url=str(test_flake_with_core), + flake_attr="vm1", + cores=1, + memory_size=1024, + graphics=True, + ), + ) + assert response.status_code == 200, "Failed to inspect vm" From 32b2e4c084bec221ad14fdcbc0aa30169a843158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 16 Sep 2023 10:56:39 +0200 Subject: [PATCH 30/44] extend dev CORS flag to include methods --- pkgs/clan-cli/clan_cli/webui/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 7d915a7..ede2fbf 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -65,7 +65,8 @@ def start_server(args: argparse.Namespace) -> None: ( "Access-Control-Allow-Origin", f"http://{host}:{args.dev_port}", - ) + ), + ("Access-Control-Allow-Methods", "HEAD, POST, GET, OPTIONS"), ] else: open_url = f"http://[{args.host}]:{args.port}" From 1bcff16b664cb67fced4fc3cee84e47adc502fbb Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 16 Sep 2023 16:21:33 +0200 Subject: [PATCH 31/44] add join clan page --- pkgs/clan-cli/clan_cli/webui/app.py | 11 ++ pkgs/clan-cli/clan_cli/webui/server.py | 18 +- .../tests/test_flake_with_core/flake.nix | 3 + pkgs/ui/nix/pdefs.nix | 23 ++- pkgs/ui/orval.config.ts | 2 +- pkgs/ui/package-lock.json | 12 ++ pkgs/ui/package.json | 1 + pkgs/ui/src/app/join/page.tsx | 184 ++++++++++++++++++ .../createMachineForm/customConfig.tsx | 4 +- pkgs/ui/src/components/hooks/useVms.tsx | 50 +++++ 10 files changed, 299 insertions(+), 9 deletions(-) create mode 100644 pkgs/ui/src/app/join/page.tsx create mode 100644 pkgs/ui/src/components/hooks/useVms.tsx diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index 76f5809..e7469d5 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -1,13 +1,24 @@ from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.staticfiles import StaticFiles +from fastapi.middleware.cors import CORSMiddleware from .assets import asset_path from .routers import health, machines, root, vms +origins = [ + "http://localhost:3000", +] def setup_app() -> FastAPI: app = FastAPI() + app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) app.include_router(health.router) app.include_router(machines.router) app.include_router(root.router) diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 7d915a7..213b276 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -55,17 +55,25 @@ def start_server(args: argparse.Namespace) -> None: with ExitStack() as stack: headers: list[tuple[str, str]] = [] if args.dev: - stack.enter_context(spawn_node_dev_server(args.dev_host, args.dev_port)) + # stack.enter_context(spawn_node_dev_server(args.dev_host, args.dev_port)) open_url = f"http://{args.dev_host}:{args.dev_port}" host = args.dev_host if ":" in host: host = f"[{host}]" headers = [ - ( - "Access-Control-Allow-Origin", - f"http://{host}:{args.dev_port}", - ) + # ( + # "Access-Control-Allow-Origin", + # f"http://{host}:{args.dev_port}", + # ), + # ( + # "Access-Control-Allow-Methods", + # "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT" + # ), + # ( + # "Allow", + # "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT" + # ) ] else: open_url = f"http://[{args.host}]:{args.port}" diff --git a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix index 0a36a13..fab76b5 100644 --- a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix +++ b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix @@ -1,4 +1,7 @@ { + # Use this path to our repo root e.g. for UI test + # inputs.clan-core.url = "../../../../."; + # this placeholder is replaced by the path to nixpkgs inputs.clan-core.url = "__CLAN_CORE__"; diff --git a/pkgs/ui/nix/pdefs.nix b/pkgs/ui/nix/pdefs.nix index 5224db1..4e79bc9 100644 --- a/pkgs/ui/nix/pdefs.nix +++ b/pkgs/ui/nix/pdefs.nix @@ -10986,6 +10986,11 @@ descriptor = "^0.4.1"; pin = "0.4.1"; }; + pretty-bytes = { + descriptor = "^6.1.1"; + pin = "6.1.1"; + runtime = true; + }; react = { descriptor = "18.2.0"; pin = "18.2.0"; @@ -13086,6 +13091,9 @@ dev = true; key = "prettier-plugin-tailwindcss/0.4.1"; }; + "node_modules/pretty-bytes" = { + key = "pretty-bytes/6.1.1"; + }; "node_modules/printable-characters" = { dev = true; key = "printable-characters/1.0.42"; @@ -15195,6 +15203,19 @@ version = "0.4.1"; }; }; + pretty-bytes = { + "6.1.1" = { + fetchInfo = { + narHash = "sha256-ERXqMD/9tkPebbHVL3n/9EQRz7mFs5VYO6k/wo5JDzQ="; + type = "tarball"; + url = "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz"; + }; + ident = "pretty-bytes"; + ltype = "file"; + treeInfo = { }; + version = "6.1.1"; + }; + }; printable-characters = { "1.0.42" = { fetchInfo = { @@ -18237,4 +18258,4 @@ }; }; }; -} +} \ No newline at end of file diff --git a/pkgs/ui/orval.config.ts b/pkgs/ui/orval.config.ts index db48112..b92bd3d 100644 --- a/pkgs/ui/orval.config.ts +++ b/pkgs/ui/orval.config.ts @@ -1,5 +1,5 @@ const config = { - petstore: { + clan: { output: { mode: "tags-split", target: "src/api", diff --git a/pkgs/ui/package-lock.json b/pkgs/ui/package-lock.json index 5462475..fc249db 100644 --- a/pkgs/ui/package-lock.json +++ b/pkgs/ui/package-lock.json @@ -22,6 +22,7 @@ "hex-rgb": "^5.0.0", "next": "13.4.12", "postcss": "8.4.27", + "pretty-bytes": "^6.1.1", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.45.4", @@ -6810,6 +6811,17 @@ } } }, + "node_modules/pretty-bytes": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz", + "integrity": "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ==", + "engines": { + "node": "^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/printable-characters": { "version": "1.0.42", "resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz", diff --git a/pkgs/ui/package.json b/pkgs/ui/package.json index 6ae1b83..02e28cd 100644 --- a/pkgs/ui/package.json +++ b/pkgs/ui/package.json @@ -26,6 +26,7 @@ "hex-rgb": "^5.0.0", "next": "13.4.12", "postcss": "8.4.27", + "pretty-bytes": "^6.1.1", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.45.4", diff --git a/pkgs/ui/src/app/join/page.tsx b/pkgs/ui/src/app/join/page.tsx new file mode 100644 index 0000000..660c407 --- /dev/null +++ b/pkgs/ui/src/app/join/page.tsx @@ -0,0 +1,184 @@ +"use client"; +import React, { useState } from "react"; +import { VmConfig } from "@/api/model"; +import { useVms } from "@/components/hooks/useVms"; +import prettyBytes from "pretty-bytes"; + +import { + Alert, + AlertTitle, + Button, + Chip, + LinearProgress, + ListSubheader, + Switch, + Typography, +} from "@mui/material"; +import { useSearchParams } from "next/navigation"; +import { toast } from "react-hot-toast"; +import { Error, Numbers } from "@mui/icons-material"; +import { createVm, inspectVm } from "@/api/default/default"; + +interface FlakeBadgeProps { + flakeUrl: string; + flakeAttr: string; +} +const FlakeBadge = (props: FlakeBadgeProps) => ( + +); + +interface VmPropLabelProps { + children: React.ReactNode; +} +const VmPropLabel = (props: VmPropLabelProps) => ( +
+ {props.children} +
+); + +interface VmPropContentProps { + children: React.ReactNode; +} +const VmPropContent = (props: VmPropContentProps) => ( +
{props.children}
+); + +interface VmDetailsProps { + vmConfig: VmConfig; +} + +const VmDetails = (props: VmDetailsProps) => { + const { vmConfig } = props; + const { cores, flake_attr, flake_url, graphics, memory_size } = vmConfig; + const [isStarting, setStarting] = useState(false); + const handleStartVm = async () => { + setStarting(true); + const response = await createVm(vmConfig); + setStarting(false); + if (response.statusText === "OK") { + toast.success(("VM created @ " + response?.data) as string); + } else { + toast.error("Could not create VM"); + } + }; + return ( +
+
+ General +
+ + Flake + + + + + Machine + {flake_attr} + +
+ VM +
+ CPU Cores + + + {cores} + + + Graphics + + + + + Memory Size + {prettyBytes(memory_size * 1024 * 1024)} + +
+ {isStarting && } + +
+
+ ); +}; + +interface ErrorLogOptions { + lines: string[]; +} +const ErrorLog = (props: ErrorLogOptions) => { + const { lines } = props; + return ( +
+
Log
+ {lines.map((item, idx) => ( + + {item} +
+ + ))} +
+ ); +}; + +export default function Page() { + const queryParams = useSearchParams(); + const flakeUrl = queryParams.get("flake") || ""; + const flakeAttribute = queryParams.get("attr") || "default"; + + const { config, error, isLoading } = useVms({ + url: flakeUrl, + attr: flakeAttribute, + }); + const clanName = "Lassul.us"; + return ( +
+ + Join{" "} + + {clanName} + + {"' "} + Clan + + {error && ( + + Error + An Error occurred - See details below + + )} +
+ {isLoading && ( +
+ Loading Flake + +
+ +
+ + +
+ )} + {(!flakeUrl || !flakeAttribute) &&
Invalid URL
} + {config && } + {error && ( + err.msg.split("\n")) + ?.flat() + .filter(Boolean) || [] + } + /> + )} +
+
+ ); +} diff --git a/pkgs/ui/src/components/createMachineForm/customConfig.tsx b/pkgs/ui/src/components/createMachineForm/customConfig.tsx index fd005fb..d7805ec 100644 --- a/pkgs/ui/src/components/createMachineForm/customConfig.tsx +++ b/pkgs/ui/src/components/createMachineForm/customConfig.tsx @@ -54,7 +54,7 @@ export function CustomConfig(props: FormStepContentProps) { } return acc; }, {}), - [schema], + [schema] ); return isLoading ? ( @@ -124,7 +124,7 @@ function PureCustomConfig(props: PureCustomConfigProps) { message: "invalid config", }); toast.error( - "Configuration is invalid. Please check the highlighted fields for details.", + "Configuration is invalid. Please check the highlighted fields for details." ); } else { formHooks.clearErrors("config"); diff --git a/pkgs/ui/src/components/hooks/useVms.tsx b/pkgs/ui/src/components/hooks/useVms.tsx new file mode 100644 index 0000000..3901a25 --- /dev/null +++ b/pkgs/ui/src/components/hooks/useVms.tsx @@ -0,0 +1,50 @@ +import { inspectVm } from "@/api/default/default"; +import { HTTPValidationError, VmConfig } from "@/api/model"; +import { AxiosError } from "axios"; +import { useEffect, useState } from "react"; +import { toast } from "react-hot-toast"; + +interface UseVmsOptions { + url: string; + attr: string; +} +export const useVms = (options: UseVmsOptions) => { + const { url, attr } = options; + const [isLoading, setIsLoading] = useState(true); + const [config, setConfig] = useState(); + const [error, setError] = useState>(); + + useEffect(() => { + const getVmInfo = async (url: string, attr: string) => { + if (url === "") { + toast.error("Flake url is missing", { id: "missing.flake.url" }); + return undefined; + } + try { + const response = await inspectVm({ + flake_attr: attr, + flake_url: url, + }); + const { + data: { config }, + } = response; + setError(undefined); + return config; + } catch (e) { + const err = e as AxiosError; + setError(err); + toast.error(err.message); + return undefined; + } finally { + setIsLoading(false); + } + }; + getVmInfo(url, attr).then((c) => setConfig(c)); + }, [url, attr]); + + return { + error, + isLoading, + config, + }; +}; From 18714d665775147a60a2052d7b1f73649ed93e3e Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Sat, 16 Sep 2023 16:28:19 +0200 Subject: [PATCH 32/44] reformat --- pkgs/clan-cli/clan_cli/webui/app.py | 3 ++- pkgs/clan-cli/clan_cli/webui/server.py | 2 +- pkgs/ui/nix/pdefs.nix | 2 +- pkgs/ui/src/components/createMachineForm/customConfig.tsx | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index e7469d5..834f337 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -1,7 +1,7 @@ from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware from fastapi.routing import APIRoute from fastapi.staticfiles import StaticFiles -from fastapi.middleware.cors import CORSMiddleware from .assets import asset_path from .routers import health, machines, root, vms @@ -10,6 +10,7 @@ origins = [ "http://localhost:3000", ] + def setup_app() -> FastAPI: app = FastAPI() app.add_middleware( diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 213b276..1b7164f 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -55,7 +55,7 @@ def start_server(args: argparse.Namespace) -> None: with ExitStack() as stack: headers: list[tuple[str, str]] = [] if args.dev: - # stack.enter_context(spawn_node_dev_server(args.dev_host, args.dev_port)) + stack.enter_context(spawn_node_dev_server(args.dev_host, args.dev_port)) open_url = f"http://{args.dev_host}:{args.dev_port}" host = args.dev_host diff --git a/pkgs/ui/nix/pdefs.nix b/pkgs/ui/nix/pdefs.nix index 4e79bc9..863553a 100644 --- a/pkgs/ui/nix/pdefs.nix +++ b/pkgs/ui/nix/pdefs.nix @@ -18258,4 +18258,4 @@ }; }; }; -} \ No newline at end of file +} diff --git a/pkgs/ui/src/components/createMachineForm/customConfig.tsx b/pkgs/ui/src/components/createMachineForm/customConfig.tsx index d7805ec..fd005fb 100644 --- a/pkgs/ui/src/components/createMachineForm/customConfig.tsx +++ b/pkgs/ui/src/components/createMachineForm/customConfig.tsx @@ -54,7 +54,7 @@ export function CustomConfig(props: FormStepContentProps) { } return acc; }, {}), - [schema] + [schema], ); return isLoading ? ( @@ -124,7 +124,7 @@ function PureCustomConfig(props: PureCustomConfigProps) { message: "invalid config", }); toast.error( - "Configuration is invalid. Please check the highlighted fields for details." + "Configuration is invalid. Please check the highlighted fields for details.", ); } else { formHooks.clearErrors("config"); From a65413c98e8f5f5a62b20b164ca62db95a482f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:22:05 +0200 Subject: [PATCH 33/44] use experimental flags everywhere --- pkgs/clan-cli/clan_cli/admin.py | 20 ++++--- pkgs/clan-cli/clan_cli/config/parsing.py | 11 ++-- pkgs/clan-cli/clan_cli/machines/update.py | 4 +- pkgs/clan-cli/clan_cli/nix.py | 69 ++++++++++++----------- pkgs/clan-cli/tests/test_ssh_cli.py | 6 +- 5 files changed, 58 insertions(+), 52 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/admin.py index 2efe081..9b47aaa 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/admin.py @@ -3,18 +3,22 @@ import argparse import os import subprocess +from .nix import nix_command + def create(args: argparse.Namespace) -> None: os.makedirs(args.folder, exist_ok=True) # TODO create clan template in flake subprocess.run( - [ - "nix", - "flake", - "init", - "-t", - "git+https://git.clan.lol/clan/clan-core#new-clan", - ] + nix_command( + [ + "flake", + "init", + "-t", + "git+https://git.clan.lol/clan/clan-core#new-clan", + ] + ), + check=True, ) @@ -24,7 +28,7 @@ def register_parser(parser: argparse.ArgumentParser) -> None: "-f", "--folder", help="the folder where the clan is defined, default to the current folder", - default=os.environ["PWD"], + default=os.getcwd(), ) subparser = parser.add_subparsers( title="command", diff --git a/pkgs/clan-cli/clan_cli/config/parsing.py b/pkgs/clan-cli/clan_cli/config/parsing.py index 92b476b..3178224 100644 --- a/pkgs/clan-cli/clan_cli/config/parsing.py +++ b/pkgs/clan-cli/clan_cli/config/parsing.py @@ -3,7 +3,8 @@ import subprocess from pathlib import Path from typing import Any, Optional, Type, Union -from clan_cli.errors import ClanError +from ..errors import ClanError +from ..nix import nix_eval script_dir = Path(__file__).parent @@ -30,11 +31,9 @@ def schema_from_module_file( slib.parseModule {absolute_path} """ # run the nix expression and parse the output as json - return json.loads( - subprocess.check_output( - ["nix", "eval", "--impure", "--json", "--expr", nix_expr] - ) - ) + cmd = nix_eval(["--expr", nix_expr]) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + return json.loads(proc.stdout) def subtype_from_schema(schema: dict[str, Any]) -> Type: diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 963b0a1..875f7fd 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -5,7 +5,7 @@ import subprocess from typing import Optional from ..dirs import get_clan_flake_toplevel -from ..nix import nix_eval +from ..nix import nix_command, nix_eval from ..secrets.generate import generate_secrets from ..secrets.upload import upload_secrets from ..ssh import Host, HostGroup, HostKeyCheck @@ -22,7 +22,7 @@ def deploy_nixos(hosts: HostGroup) -> None: env = os.environ.copy() env["NIX_SSHOPTS"] = ssh_arg res = h.run_local( - ["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"], + nix_command(["flake", "archive", "--to", f"ssh://{target}", "--json"]), check=True, stdout=subprocess.PIPE, extra_env=env, diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index e647903..72dbced 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -4,28 +4,35 @@ import tempfile from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs +def nix_command(flags: list[str]) -> list[str]: + return ["nix", "--experimental-features", "nix-command flakes"] + flags + + def nix_build( flags: list[str], ) -> list[str]: - return [ - "nix", - "build", - "--no-link", - "--print-out-paths", - "--extra-experimental-features", - "nix-command flakes", - ] + flags + return ( + nix_command( + [ + "build", + "--no-link", + "--print-out-paths", + "--extra-experimental-features", + "nix-command flakes", + ] + ) + + flags + ) def nix_eval(flags: list[str]) -> list[str]: - default_flags = [ - "nix", - "eval", - "--show-trace", - "--json", - "--extra-experimental-features", - "nix-command flakes", - ] + default_flags = nix_command( + [ + "eval", + "--show-trace", + "--json", + ] + ) if os.environ.get("IN_NIX_SANDBOX"): with tempfile.TemporaryDirectory() as nix_store: return ( @@ -51,14 +58,13 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: return cmd wrapped_packages = [f"nixpkgs#{p}" for p in packages] return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "--inputs-from", - f"{str(nixpkgs_flake())}", - ] + nix_command( + [ + "shell", + "--inputs-from", + f"{str(nixpkgs_flake())}", + ] + ) + wrapped_packages + ["-c"] + cmd @@ -69,14 +75,13 @@ def unfree_nix_shell(packages: list[str], cmd: list[str]) -> list[str]: if os.environ.get("IN_NIX_SANDBOX"): return cmd return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "-f", - str(unfree_nixpkgs()), - ] + nix_command( + [ + "shell", + "-f", + str(unfree_nixpkgs()), + ] + ) + packages + ["-c"] + cmd diff --git a/pkgs/clan-cli/tests/test_ssh_cli.py b/pkgs/clan-cli/tests/test_ssh_cli.py index 11b839f..8a7f43d 100644 --- a/pkgs/clan-cli/tests/test_ssh_cli.py +++ b/pkgs/clan-cli/tests/test_ssh_cli.py @@ -30,9 +30,8 @@ def test_ssh_no_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", @@ -61,9 +60,8 @@ def test_ssh_with_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", From d41cca2e2874c828fbf92464307f34188722615b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:34:16 +0200 Subject: [PATCH 34/44] add clan_core path helper function --- pkgs/clan-cli/tests/root.py | 11 ++++++++++- pkgs/clan-cli/tests/test_flake.py | 7 +++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkgs/clan-cli/tests/root.py b/pkgs/clan-cli/tests/root.py index c881ce0..0105e82 100644 --- a/pkgs/clan-cli/tests/root.py +++ b/pkgs/clan-cli/tests/root.py @@ -4,12 +4,13 @@ import pytest TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent +CLAN_CORE = PROJECT_ROOT.parent.parent @pytest.fixture(scope="session") def project_root() -> Path: """ - Root directory of the tests + Root directory the clan-cli """ return PROJECT_ROOT @@ -20,3 +21,11 @@ def test_root() -> Path: Root directory of the tests """ return TEST_ROOT + + +@pytest.fixture(scope="session") +def clan_core() -> Path: + """ + Directory of the clan-core flake + """ + return CLAN_CORE diff --git a/pkgs/clan-cli/tests/test_flake.py b/pkgs/clan-cli/tests/test_flake.py index cff8caf..66083b1 100644 --- a/pkgs/clan-cli/tests/test_flake.py +++ b/pkgs/clan-cli/tests/test_flake.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Iterator import pytest -from root import PROJECT_ROOT +from root import CLAN_CORE from clan_cli.dirs import nixpkgs_source @@ -46,9 +46,8 @@ def test_flake(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: @pytest.fixture(scope="module") def test_flake_with_core(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: - clan_core_flake = PROJECT_ROOT.parent.parent - if not (clan_core_flake / "flake.nix").exists(): + if not (CLAN_CORE / "flake.nix").exists(): raise Exception( "clan-core flake not found. This test requires the clan-core flake to be present" ) - yield from create_flake(monkeymodule, "test_flake_with_core", clan_core_flake) + yield from create_flake(monkeymodule, "test_flake_with_core", CLAN_CORE) From 164f95723dbbff40fe6bf41043cfbcc29e734b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:42:58 +0200 Subject: [PATCH 35/44] pytest: add impure marker --- pkgs/clan-cli/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 127d6cf..1c16c93 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -18,6 +18,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"] faulthandler_timeout = 30 addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5" norecursedirs = "tests/helpers" +markers = [ "impure" ] [tool.mypy] python_version = "3.10" From f6a7e42e38c4bd2e0c776a0fcbf0771c084517b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 15 Sep 2023 16:36:09 +0200 Subject: [PATCH 36/44] cli: rename admin create to create --- pkgs/clan-cli/clan_cli/__init__.py | 6 +++--- pkgs/clan-cli/clan_cli/{admin.py => create.py} | 18 +----------------- pkgs/clan-cli/tests/test_admin_cli.py | 14 -------------- pkgs/clan-cli/tests/test_clan_template.py | 12 ++++++++++++ 4 files changed, 16 insertions(+), 34 deletions(-) rename pkgs/clan-cli/clan_cli/{admin.py => create.py} (50%) delete mode 100644 pkgs/clan-cli/tests/test_admin_cli.py create mode 100644 pkgs/clan-cli/tests/test_clan_template.py diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index ff67eea..0eb6dd5 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -4,7 +4,7 @@ import sys from types import ModuleType from typing import Optional -from . import admin, config, machines, secrets, webui, zerotier +from . import config, create, machines, secrets, webui, zerotier from .errors import ClanError from .ssh import cli as ssh_cli @@ -19,8 +19,8 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog=prog, description="cLAN tool") subparsers = parser.add_subparsers() - parser_admin = subparsers.add_parser("admin", help="administrate a clan") - admin.register_parser(parser_admin) + parser_create = subparsers.add_parser("create", help="create a clan flake") + create.register_parser(parser_create) # DISABLED: this currently crashes if a flake does not define .#clanOptions if os.environ.get("CLAN_OPTIONS_FILE") is not None: diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/create.py similarity index 50% rename from pkgs/clan-cli/clan_cli/admin.py rename to pkgs/clan-cli/clan_cli/create.py index 9b47aaa..2a6b3ce 100644 --- a/pkgs/clan-cli/clan_cli/admin.py +++ b/pkgs/clan-cli/clan_cli/create.py @@ -1,13 +1,11 @@ # !/usr/bin/env python3 import argparse -import os import subprocess from .nix import nix_command def create(args: argparse.Namespace) -> None: - os.makedirs(args.folder, exist_ok=True) # TODO create clan template in flake subprocess.run( nix_command( @@ -24,18 +22,4 @@ def create(args: argparse.Namespace) -> None: # takes a (sub)parser and configures it def register_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "-f", - "--folder", - help="the folder where the clan is defined, default to the current folder", - default=os.getcwd(), - ) - subparser = parser.add_subparsers( - title="command", - description="the command to run", - help="the command to run", - required=True, - ) - - parser_create = subparser.add_parser("create", help="create a new clan") - parser_create.set_defaults(func=create) + parser.set_defaults(func=create) diff --git a/pkgs/clan-cli/tests/test_admin_cli.py b/pkgs/clan-cli/tests/test_admin_cli.py deleted file mode 100644 index dba43d1..0000000 --- a/pkgs/clan-cli/tests/test_admin_cli.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Union - -import pytest_subprocess.fake_process -from cli import Cli -from pytest_subprocess import utils - - -# using fp fixture from pytest-subprocess -def test_create(fp: pytest_subprocess.fake_process.FakeProcess) -> None: - cmd: list[Union[str, utils.Any]] = ["nix", "flake", "init", "-t", fp.any()] - fp.register(cmd) - cli = Cli() - cli.run(["admin", "--folder", "./my-clan", "create"]) - assert fp.call_count(cmd) == 1 diff --git a/pkgs/clan-cli/tests/test_clan_template.py b/pkgs/clan-cli/tests/test_clan_template.py new file mode 100644 index 0000000..10b4a9b --- /dev/null +++ b/pkgs/clan-cli/tests/test_clan_template.py @@ -0,0 +1,12 @@ +from pathlib import Path + +import pytest +from cli import Cli + + +@pytest.mark.impure +def test_template(monkeypatch: pytest.MonkeyPatch, temporary_dir: Path) -> None: + monkeypatch.chdir(temporary_dir) + cli = Cli() + cli.run(["create"]) + assert (temporary_dir / ".clan-flake").exists() From e753b9a87f2d381ae18468560ac4b38156891e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Sep 2023 13:02:11 +0200 Subject: [PATCH 37/44] try to fix ci by pinning nix-fast-build --- .gitea/workflows/checks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/checks.yaml b/.gitea/workflows/checks.yaml index 7f1fda2..6705af0 100644 --- a/.gitea/workflows/checks.yaml +++ b/.gitea/workflows/checks.yaml @@ -8,4 +8,4 @@ jobs: runs-on: nix steps: - uses: actions/checkout@v3 - - run: nix run --refresh github:Mic92/nix-ci-build + - run: nix run --refresh github:Mic92/nix-fast-build/ae50c356c2f9e790f3d9d8e00bfa9f4b54f49bdd From 81bc0d7ea4772ff5e6e973b47729433f5002aa6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Sep 2023 13:19:24 +0200 Subject: [PATCH 38/44] devShell: use rm -f to clean up old pre-commit file --- devShell.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devShell.nix b/devShell.nix index 62daa71..b5b963c 100644 --- a/devShell.nix +++ b/devShell.nix @@ -15,7 +15,7 @@ ]; shellHook = '' # no longer used - rm "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ''; }; }; From e66d61bb69ec87c184333136c84aee3b1a99c3e6 Mon Sep 17 00:00:00 2001 From: Clan Merge Bot Date: Mon, 18 Sep 2023 00:00:13 +0000 Subject: [PATCH 39/44] update flake lock - 2023-09-18T00:00+00:00 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'disko': 'github:nix-community/disko/be98cffef02e5ebf438ea80b34b86e669c48eff1' (2023-09-12) → 'github:nix-community/disko/9ab96378f8cf602d5f3ce5a32f2c339509288d8e' (2023-09-17) • Updated input 'floco': 'github:aakropotkin/floco/1e84b4b16bba5746e1195fa3a4d8addaaf2d9ef4' (2023-08-03) → 'github:aakropotkin/floco/d16bd444ab9d29a6640f52ee4e43a66528e07515' (2023-09-16) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/3a2786eea085f040a66ecde1bc3ddc7099f6dbeb' (2023-09-11) → 'github:NixOS/nixpkgs/ace5093e36ab1e95cb9463863491bee90d5a4183' (2023-09-15) • Updated input 'treefmt-nix': 'github:numtide/treefmt-nix/b8d3a059f5487d6767d07c3716386753e3132d9f' (2023-09-04) → 'github:numtide/treefmt-nix/7a49c388d7a6b63bb551b1ddedfa4efab8f400d8' (2023-09-12) --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index d8ddfc2..3f8987b 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1694511957, - "narHash": "sha256-teCLY68npc0nuyOHYJURLuJSOME0yotJI29WXcpF1E4=", + "lastModified": 1694925805, + "narHash": "sha256-UNMivSc89undITtNoy6o6bf3Dck4v75rzGiMEXAPEB0=", "owner": "nix-community", "repo": "disko", - "rev": "be98cffef02e5ebf438ea80b34b86e669c48eff1", + "rev": "9ab96378f8cf602d5f3ce5a32f2c339509288d8e", "type": "github" }, "original": { @@ -47,11 +47,11 @@ ] }, "locked": { - "lastModified": 1691024356, - "narHash": "sha256-uGLyhkwew6ORO6nAz0Y7KHdiQrDJVI2n6rl4gl7mWzk=", + "lastModified": 1694873346, + "narHash": "sha256-Uvh03bg0a6ZnNWiX1Gb8g+m343wSJ/wb8ryUASt0loc=", "owner": "aakropotkin", "repo": "floco", - "rev": "1e84b4b16bba5746e1195fa3a4d8addaaf2d9ef4", + "rev": "d16bd444ab9d29a6640f52ee4e43a66528e07515", "type": "github" }, "original": { @@ -98,11 +98,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694422566, - "narHash": "sha256-lHJ+A9esOz9vln/3CJG23FV6Wd2OoOFbDeEs4cMGMqc=", + "lastModified": 1694767346, + "narHash": "sha256-5uH27SiVFUwsTsqC5rs3kS7pBoNhtoy9QfTP9BmknGk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a2786eea085f040a66ecde1bc3ddc7099f6dbeb", + "rev": "ace5093e36ab1e95cb9463863491bee90d5a4183", "type": "github" }, "original": { @@ -151,11 +151,11 @@ ] }, "locked": { - "lastModified": 1693817438, - "narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=", + "lastModified": 1694528738, + "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "b8d3a059f5487d6767d07c3716386753e3132d9f", + "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8", "type": "github" }, "original": { From 2931b08b462a87b1bbf12e246773e547f8c52975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 19 Sep 2023 14:15:15 +0200 Subject: [PATCH 40/44] clan-cli/update: use correct flake attr when deploying --- pkgs/clan-cli/clan_cli/machines/update.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index 875f7fd..9f5a170 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -104,10 +104,20 @@ def update(args: argparse.Namespace) -> None: hostname = maybe_port[0] port = int(maybe_port[1]) print(f"deploying {host}") - deploy_nixos(HostGroup([Host(host=hostname, port=port, user=user)])) + deploy_nixos( + HostGroup( + [ + Host( + host=hostname, + port=port, + user=user, + meta=dict(flake_attr=args.machine), + ) + ] + ) + ) def register_update_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument("--target-host", type=str, default="root") parser.add_argument("machine", type=str) parser.set_defaults(func=update) From dfa3b2acaaa334bfb13fd17ece330e30e7631587 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 19 Sep 2023 11:04:07 +0200 Subject: [PATCH 41/44] quickstart: fixes --- docs/quickstart.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/quickstart.md b/docs/quickstart.md index a23bbbc..5efea07 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -43,6 +43,7 @@ Absolutely, let's break down the migration step by step, explaining each action ```nix inputs.clan-core = { url = "git+https://git.clan.lol/clan/clan-core"; + # Don't do this if your machines are on nixpkgs stable. inputs.nixpkgs.follows = "nixpkgs"; }; ``` @@ -75,7 +76,8 @@ Absolutely, let's break down the migration step by step, explaining each action ```nix nixosConfigurations = clan-core.lib.buildClan { - directory = ./.; + # this needs to point at the repository root + directory = self; specialArgs = {}; machines = { example-desktop = { From 76ee9893f212e9b62f49351338504a9555d7707c Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 19 Sep 2023 11:04:33 +0200 Subject: [PATCH 42/44] docs/secrets-management: machine -> machines --- docs/secrets-management.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/secrets-management.md b/docs/secrets-management.md index 321d218..2328f82 100644 --- a/docs/secrets-management.md +++ b/docs/secrets-management.md @@ -86,7 +86,7 @@ $ clan secrets machines list For existing machines, add their keys: ```console -$ clan secrets machine add +$ clan secrets machines add ``` To fetch an age key from an SSH host key: From 9b6fafcb2d3e44fea56a9fca6248cbc9abc4bd37 Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 19 Sep 2023 11:29:59 +0200 Subject: [PATCH 43/44] clan config: re-enable clan config --- flake.nix | 2 -- flakeModules/clan-config.nix | 42 ----------------------- nixosModules/clanCore/secrets/default.nix | 3 +- pkgs/clan-cli/clan_cli/__init__.py | 7 ++-- pkgs/clan-cli/clan_cli/config/__init__.py | 2 +- pkgs/clan-cli/default.nix | 14 +++----- pkgs/clan-cli/shell.nix | 3 -- 7 files changed, 9 insertions(+), 64 deletions(-) delete mode 100644 flakeModules/clan-config.nix diff --git a/flake.nix b/flake.nix index 79119ed..7bd18b6 100644 --- a/flake.nix +++ b/flake.nix @@ -31,8 +31,6 @@ ./formatter.nix ./templates/flake-module.nix - ./flakeModules/clan-config.nix - ./pkgs/flake-module.nix ./lib/flake-module.nix diff --git a/flakeModules/clan-config.nix b/flakeModules/clan-config.nix deleted file mode 100644 index 236d225..0000000 --- a/flakeModules/clan-config.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ ... } @ clanCore: { - flake.flakeModules.clan-config = { self, inputs, ... }: - let - - # take the default nixos configuration - options = self.nixosConfigurations.default.options; - - # this is actually system independent as it uses toFile - docs = inputs.nixpkgs.legacyPackages.x86_64-linux.nixosOptionsDoc { - inherit options; - }; - - optionsJSONFile = docs.optionsJSON.options; - - warnIfNoDefaultConfig = return: - if ! self ? nixosConfigurations.default - then - builtins.trace - "WARNING: .#nixosConfigurations.default could not be found. Please define it." - return - else return; - - in - { - flake.clanOptions = warnIfNoDefaultConfig optionsJSONFile; - - flake.clanSettings = self + /clan-settings.json; - - perSystem = { pkgs, ... }: { - devShells.clan-config = pkgs.mkShell { - packages = [ - clanCore.config.flake.packages.${pkgs.system}.clan-cli - ]; - shellHook = '' - export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions) - export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" - ''; - }; - }; - }; -} diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 700f5d0..44fa757 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -62,7 +62,8 @@ default = "machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; }; value = lib.mkOption { - default = builtins.readFile "${config.clanCore.clanDir}/fact.config.path"; + defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}"; + default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"; }; }; })); diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index 0eb6dd5..f67bee8 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -1,5 +1,4 @@ import argparse -import os import sys from types import ModuleType from typing import Optional @@ -22,10 +21,8 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: parser_create = subparsers.add_parser("create", help="create a clan flake") create.register_parser(parser_create) - # DISABLED: this currently crashes if a flake does not define .#clanOptions - if os.environ.get("CLAN_OPTIONS_FILE") is not None: - parser_config = subparsers.add_parser("config", help="set nixos configuration") - config.register_parser(parser_config) + parser_config = subparsers.add_parser("config", help="set nixos configuration") + config.register_parser(parser_config) parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine") ssh_cli.register_parser(parser_ssh) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index d81fbc2..a383855 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -286,7 +286,7 @@ def register_parser( # add single positional argument for the option (e.g. "foo.bar") parser.add_argument( "option", - help="Option to configure", + help="Option to read or set", type=str, ) diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index 1f43eb4..f41152f 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -22,9 +22,6 @@ , ui-assets }: let - # This provides dummy options for testing clan config and prevents it from - # evaluating the flake .# - CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json; dependencies = [ argcomplete # optional dependency: if not enabled, shell completion will not work @@ -54,9 +51,9 @@ let ''; nixpkgs = runCommand "nixpkgs" { nativeBuildInputs = [ pkgs.nix ]; } '' mkdir $out - mkdir -p $out/unfree - cat > $out/unfree/default.nix < $out/unfree/default.nix < $out/flake.nix << EOF { @@ -81,8 +78,6 @@ python3.pkgs.buildPythonPackage { src = source; format = "pyproject"; - inherit CLAN_OPTIONS_FILE; - nativeBuildInputs = [ setuptools installShellFiles @@ -93,12 +88,11 @@ python3.pkgs.buildPythonPackage { { nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ]; } '' - export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}" cp -r ${source} ./src chmod +w -R ./src cd ./src - export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests touch $out ''; diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index f0ff0d7..f95340c 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -20,9 +20,6 @@ mkShell { pythonWithDeps ]; # sets up an editable install and add enty points to $PATH - # This provides dummy options for testing clan config and prevents it from - # evaluating the flake .# - CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json; PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}"; PYTHONBREAKPOINT = "ipdb.set_trace"; From c11f73c4d0d36b9f083cc954fd86a18ba319bf8c Mon Sep 17 00:00:00 2001 From: DavHau Date: Tue, 19 Sep 2023 11:53:10 +0200 Subject: [PATCH 44/44] clan config: fix settings file location --- pkgs/clan-cli/clan_cli/config/__init__.py | 6 +++--- pkgs/clan-cli/clan_cli/config/machine.py | 4 ++-- pkgs/clan-cli/tests/test_flake/flake.nix | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index a383855..1a3c0fb 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -9,6 +9,7 @@ from typing import Any, Optional, Type from clan_cli.dirs import get_clan_flake_toplevel from clan_cli.errors import ClanError +from clan_cli.machines.folders import machine_settings_file from clan_cli.nix import nix_eval script_dir = Path(__file__).parent @@ -166,7 +167,6 @@ def get_or_set_option(args: argparse.Namespace) -> None: print(read_machine_option_value(args.machine, args.option)) else: # load options - print(args.options_file) if args.options_file is None: options = options_for_machine(machine_name=args.machine) else: @@ -174,8 +174,8 @@ def get_or_set_option(args: argparse.Namespace) -> None: options = json.load(f) # compute settings json file location if args.settings_file is None: - flake = get_clan_flake_toplevel() - settings_file = flake / "machines" / f"{args.machine}.json" + get_clan_flake_toplevel() + settings_file = machine_settings_file(args.machine) else: settings_file = args.settings_file # set the option with the given value diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index 1e3b29a..cad6b50 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -12,7 +12,7 @@ from clan_cli.nix import nix_eval def config_for_machine(machine_name: str) -> dict: - # read the config from a json file located at {flake}/machines/{machine_name}.json + # read the config from a json file located at {flake}/machines/{machine_name}/settings.json if not machine_folder(machine_name).exists(): raise HTTPException( status_code=404, @@ -26,7 +26,7 @@ def config_for_machine(machine_name: str) -> dict: def set_config_for_machine(machine_name: str, config: dict) -> None: - # write the config to a json file located at {flake}/machines/{machine_name}.json + # write the config to a json file located at {flake}/machines/{machine_name}/settings.json if not machine_folder(machine_name).exists(): raise HTTPException( status_code=404, diff --git a/pkgs/clan-cli/tests/test_flake/flake.nix b/pkgs/clan-cli/tests/test_flake/flake.nix index 3ba6f3f..23d2a67 100644 --- a/pkgs/clan-cli/tests/test_flake/flake.nix +++ b/pkgs/clan-cli/tests/test_flake/flake.nix @@ -6,8 +6,8 @@ nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { modules = [ ./nixosModules/machine1.nix - (if builtins.pathExists ./machines/machine1.json - then builtins.fromJSON (builtins.readFile ./machines/machine1.json) + (if builtins.pathExists ./machines/machine1/settings.json + then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) else { }) { nixpkgs.hostPlatform = "x86_64-linux";