diff --git a/.gitea/workflows/checks.yaml b/.gitea/workflows/checks.yaml index 7f1fda2..6705af0 100644 --- a/.gitea/workflows/checks.yaml +++ b/.gitea/workflows/checks.yaml @@ -8,4 +8,4 @@ jobs: runs-on: nix steps: - uses: actions/checkout@v3 - - run: nix run --refresh github:Mic92/nix-ci-build + - run: nix run --refresh github:Mic92/nix-fast-build/ae50c356c2f9e790f3d9d8e00bfa9f4b54f49bdd diff --git a/.gitignore b/.gitignore index 39e97f9..42dff4f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ .direnv result* -pkgs/clan-cli/clan_cli/nixpkgs -pkgs/clan-cli/clan_cli/webui/assets -machines +/pkgs/clan-cli/clan_cli/nixpkgs +/pkgs/clan-cli/clan_cli/webui/assets +/machines # python __pycache__ diff --git a/devShell.nix b/devShell.nix index 62daa71..b5b963c 100644 --- a/devShell.nix +++ b/devShell.nix @@ -15,7 +15,7 @@ ]; shellHook = '' # no longer used - rm "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" + rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ''; }; }; diff --git a/docs/quickstart.md b/docs/quickstart.md index a23bbbc..5efea07 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -43,6 +43,7 @@ Absolutely, let's break down the migration step by step, explaining each action ```nix inputs.clan-core = { url = "git+https://git.clan.lol/clan/clan-core"; + # Don't do this if your machines are on nixpkgs stable. inputs.nixpkgs.follows = "nixpkgs"; }; ``` @@ -75,7 +76,8 @@ Absolutely, let's break down the migration step by step, explaining each action ```nix nixosConfigurations = clan-core.lib.buildClan { - directory = ./.; + # this needs to point at the repository root + directory = self; specialArgs = {}; machines = { example-desktop = { diff --git a/docs/secrets-management.md b/docs/secrets-management.md index 321d218..2328f82 100644 --- a/docs/secrets-management.md +++ b/docs/secrets-management.md @@ -86,7 +86,7 @@ $ clan secrets machines list For existing machines, add their keys: ```console -$ clan secrets machine add +$ clan secrets machines add ``` To fetch an age key from an SSH host key: diff --git a/flake.lock b/flake.lock index d8ddfc2..3f8987b 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1694511957, - "narHash": "sha256-teCLY68npc0nuyOHYJURLuJSOME0yotJI29WXcpF1E4=", + "lastModified": 1694925805, + "narHash": "sha256-UNMivSc89undITtNoy6o6bf3Dck4v75rzGiMEXAPEB0=", "owner": "nix-community", "repo": "disko", - "rev": "be98cffef02e5ebf438ea80b34b86e669c48eff1", + "rev": "9ab96378f8cf602d5f3ce5a32f2c339509288d8e", "type": "github" }, "original": { @@ -47,11 +47,11 @@ ] }, "locked": { - "lastModified": 1691024356, - "narHash": "sha256-uGLyhkwew6ORO6nAz0Y7KHdiQrDJVI2n6rl4gl7mWzk=", + "lastModified": 1694873346, + "narHash": "sha256-Uvh03bg0a6ZnNWiX1Gb8g+m343wSJ/wb8ryUASt0loc=", "owner": "aakropotkin", "repo": "floco", - "rev": "1e84b4b16bba5746e1195fa3a4d8addaaf2d9ef4", + "rev": "d16bd444ab9d29a6640f52ee4e43a66528e07515", "type": "github" }, "original": { @@ -98,11 +98,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1694422566, - "narHash": "sha256-lHJ+A9esOz9vln/3CJG23FV6Wd2OoOFbDeEs4cMGMqc=", + "lastModified": 1694767346, + "narHash": "sha256-5uH27SiVFUwsTsqC5rs3kS7pBoNhtoy9QfTP9BmknGk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3a2786eea085f040a66ecde1bc3ddc7099f6dbeb", + "rev": "ace5093e36ab1e95cb9463863491bee90d5a4183", "type": "github" }, "original": { @@ -151,11 +151,11 @@ ] }, "locked": { - "lastModified": 1693817438, - "narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=", + "lastModified": 1694528738, + "narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=", "owner": "numtide", "repo": "treefmt-nix", - "rev": "b8d3a059f5487d6767d07c3716386753e3132d9f", + "rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 79119ed..7bd18b6 100644 --- a/flake.nix +++ b/flake.nix @@ -31,8 +31,6 @@ ./formatter.nix ./templates/flake-module.nix - ./flakeModules/clan-config.nix - ./pkgs/flake-module.nix ./lib/flake-module.nix diff --git a/flakeModules/clan-config.nix b/flakeModules/clan-config.nix deleted file mode 100644 index 236d225..0000000 --- a/flakeModules/clan-config.nix +++ /dev/null @@ -1,42 +0,0 @@ -{ ... } @ clanCore: { - flake.flakeModules.clan-config = { self, inputs, ... }: - let - - # take the default nixos configuration - options = self.nixosConfigurations.default.options; - - # this is actually system independent as it uses toFile - docs = inputs.nixpkgs.legacyPackages.x86_64-linux.nixosOptionsDoc { - inherit options; - }; - - optionsJSONFile = docs.optionsJSON.options; - - warnIfNoDefaultConfig = return: - if ! self ? nixosConfigurations.default - then - builtins.trace - "WARNING: .#nixosConfigurations.default could not be found. Please define it." - return - else return; - - in - { - flake.clanOptions = warnIfNoDefaultConfig optionsJSONFile; - - flake.clanSettings = self + /clan-settings.json; - - perSystem = { pkgs, ... }: { - devShells.clan-config = pkgs.mkShell { - packages = [ - clanCore.config.flake.packages.${pkgs.system}.clan-cli - ]; - shellHook = '' - export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions) - export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" - export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}" - ''; - }; - }; - }; -} diff --git a/lib/jsonschema/default.nix b/lib/jsonschema/default.nix index 26c399e..90a8fb2 100644 --- a/lib/jsonschema/default.nix +++ b/lib/jsonschema/default.nix @@ -7,6 +7,7 @@ let float = "number"; int = "integer"; str = "string"; + path = "string"; # TODO add prober path checks }; # remove _module attribute from options @@ -103,6 +104,13 @@ rec { type = "string"; } + # parse string + else if option.type.name == "path" + # return jsonschema property definition for path + then default // description // { + type = "string"; + } + # parse enum else if option.type.name == "enum" # return jsonschema property definition for enum diff --git a/nixosModules/clanCore/flake-module.nix b/nixosModules/clanCore/flake-module.nix index da174ff..0fcf4cd 100644 --- a/nixosModules/clanCore/flake-module.nix +++ b/nixosModules/clanCore/flake-module.nix @@ -3,9 +3,11 @@ imports = [ ./secrets ./zerotier.nix + ./networking.nix inputs.sops-nix.nixosModules.sops # just some example options. Can be removed later ./bloatware + ./vm.nix ]; options.clanSchema = lib.mkOption { type = lib.types.attrs; diff --git a/nixosModules/clanCore/networking.nix b/nixosModules/clanCore/networking.nix new file mode 100644 index 0000000..813939d --- /dev/null +++ b/nixosModules/clanCore/networking.nix @@ -0,0 +1,15 @@ +{ config, lib, ... }: +{ + options.clan.networking = { + deploymentAddress = lib.mkOption { + description = '' + The target SSH node for deployment. + + By default, the node's attribute name will be used. + If set to null, only local deployment will be supported. + ''; + type = lib.types.nullOr lib.types.str; + default = "root@${config.networking.hostName}"; + }; + }; +} diff --git a/nixosModules/clanCore/secrets/default.nix b/nixosModules/clanCore/secrets/default.nix index 48fef29..44fa757 100644 --- a/nixosModules/clanCore/secrets/default.nix +++ b/nixosModules/clanCore/secrets/default.nix @@ -1,6 +1,16 @@ -{ config, lib, ... }: +{ config, lib, pkgs, ... }: { + options.clanCore.secretStore = lib.mkOption { + type = lib.types.enum [ "sops" "password-store" "custom" ]; + default = "sops"; + description = '' + method to store secrets + custom can be used to define a custom secret store. + one would have to define system.clan.generateSecrets and system.clan.uploadSecrets + ''; + }; options.clanCore.secrets = lib.mkOption { + default = { }; type = lib.types.attrsOf (lib.types.submodule (secret: { options = { @@ -49,10 +59,11 @@ description = '' path to a fact which is generated by the generator ''; - default = "${config.clanCore.clanDir}/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; + default = "machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}"; }; value = lib.mkOption { - default = builtins.readFile fact.config.path; + defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}"; + default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}"; }; }; })); @@ -60,7 +71,12 @@ }; })); }; + config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" '' + ${config.system.clan.generateSecrets} + ${config.system.clan.uploadSecrets} + ''; imports = [ - ./sops.nix # for now we have only one implementation, thats why we import it here and not in clanModules + ./sops.nix + ./password-store.nix ]; } diff --git a/nixosModules/clanCore/secrets/password-store.nix b/nixosModules/clanCore/secrets/password-store.nix new file mode 100644 index 0000000..3db20e0 --- /dev/null +++ b/nixosModules/clanCore/secrets/password-store.nix @@ -0,0 +1,118 @@ +{ config, lib, pkgs, ... }: +let + passwordstoreDir = "\${PASSWORD_STORE_DIR:-$HOME/.password-store}"; +in +{ + options.clan.password-store.targetDirectory = lib.mkOption { + type = lib.types.path; + default = "/etc/secrets"; + description = '' + The directory where the password store is uploaded to. + ''; + }; + config = lib.mkIf (config.clanCore.secretStore == "password-store") { + system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' + #!/bin/sh + set -efu + + test -d "$CLAN_DIR" + PATH=${lib.makeBinPath [ + pkgs.pass + ]}:$PATH + + # TODO maybe initialize password store if it doesn't exist yet + + ${lib.foldlAttrs (acc: n: v: '' + ${acc} + # ${n} + # if any of the secrets are missing, we regenerate all connected facts/secrets + (if ! ${lib.concatMapStringsSep " && " (x: "pass show machines/${config.clanCore.machineName}/${x.name} >/dev/null") (lib.attrValues v.secrets)}; then + + facts=$(mktemp -d) + trap "rm -rf $facts" EXIT + secrets=$(mktemp -d) + trap "rm -rf $secrets" EXIT + ${v.generator} + + ${lib.concatMapStrings (fact: '' + mkdir -p "$(dirname ${fact.path})" + cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path} + '') (lib.attrValues v.facts)} + + ${lib.concatMapStrings (secret: '' + cat "$secrets"/${secret.name} | pass insert -m machines/${config.clanCore.machineName}/${secret.name} + '') (lib.attrValues v.secrets)} + fi) + '') "" config.clanCore.secrets} + ''; + system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" '' + #!/bin/sh + set -efu + + target=$1 + + umask 0077 + + PATH=${lib.makeBinPath [ + pkgs.pass + pkgs.git + pkgs.findutils + pkgs.rsync + ]}:$PATH:${lib.getBin pkgs.openssh} + + if test -e ${passwordstoreDir}/.git; then + local_pass_info=$( + git -C ${passwordstoreDir} log -1 --format=%H machines/${config.clanCore.machineName} + # we append a hash for every symlink, otherwise we would miss updates on + # files where the symlink points to + find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type l \ + -exec realpath {} + | + sort | + xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H + ) + remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg '' + cat ${config.clan.password-store.targetDirectory}/.pass_info || : + ''}) + + if test "$local_pass_info" = "$remote_pass_info"; then + echo secrets already match + exit 0 + fi + fi + + tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX) + trap cleanup EXIT + cleanup() { + rm -fR "$tmp_dir" + } + + find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id | + while read -r gpg_path; do + + rel_name=''${gpg_path#${passwordstoreDir}} + rel_name=''${rel_name%.gpg} + + pass_date=$( + if test -e ${passwordstoreDir}/.git; then + git -C ${passwordstoreDir} log -1 --format=%aI "$gpg_path" + fi + ) + pass_name=$rel_name + tmp_path=$tmp_dir/$(basename $rel_name) + + mkdir -p "$(dirname "$tmp_path")" + pass show "$pass_name" > "$tmp_path" + if [ -n "$pass_date" ]; then + touch -d "$pass_date" "$tmp_path" + fi + done + + if test -n "''${local_pass_info-}"; then + echo "$local_pass_info" > "$tmp_dir"/.pass_info + fi + + rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/ + ''; + }; +} + diff --git a/nixosModules/clanCore/secrets/sops.nix b/nixosModules/clanCore/secrets/sops.nix index ab97722..25d0af6 100644 --- a/nixosModules/clanCore/secrets/sops.nix +++ b/nixosModules/clanCore/secrets/sops.nix @@ -21,11 +21,12 @@ let secrets = filterDir containsMachineOrGroups secretsDir; in { - config = { + config = lib.mkIf (config.clanCore.secretStore == "sops") { system.clan.generateSecrets = pkgs.writeScript "generate-secrets" '' #!/bin/sh set -efu - set -x # remove for prod + + test -d "$CLAN_DIR" PATH=$PATH:${lib.makeBinPath [ config.clanCore.clanPkgs.clan-cli @@ -55,7 +56,7 @@ in ${lib.concatMapStrings (fact: '' mkdir -p "$(dirname ${fact.path})" - cp "$facts"/${fact.name} ${fact.path} + cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path} '') (lib.attrValues v.facts)} ${lib.concatMapStrings (secret: '' @@ -64,6 +65,9 @@ in fi) '') "" config.clanCore.secrets} ''; + system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" '' + echo upload is not needed for sops secret store, since the secrets are part of the flake + ''; sops.secrets = builtins.mapAttrs (name: _: { sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret"; diff --git a/nixosModules/clanCore/vm.nix b/nixosModules/clanCore/vm.nix new file mode 100644 index 0000000..4382c8c --- /dev/null +++ b/nixosModules/clanCore/vm.nix @@ -0,0 +1,8 @@ +{ config, options, lib, ... }: { + system.clan.vm.config = { + enabled = options.virtualisation ? cores; + } // (lib.optionalAttrs (options.virtualisation ? cores) { + inherit (config.virtualisation) cores graphics; + memory_size = config.virtualisation.memorySize; + }); +} diff --git a/pkgs/clan-cli/clan_cli/__init__.py b/pkgs/clan-cli/clan_cli/__init__.py index ff67eea..f67bee8 100644 --- a/pkgs/clan-cli/clan_cli/__init__.py +++ b/pkgs/clan-cli/clan_cli/__init__.py @@ -1,10 +1,9 @@ import argparse -import os import sys from types import ModuleType from typing import Optional -from . import admin, config, machines, secrets, webui, zerotier +from . import config, create, machines, secrets, webui, zerotier from .errors import ClanError from .ssh import cli as ssh_cli @@ -19,13 +18,11 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog=prog, description="cLAN tool") subparsers = parser.add_subparsers() - parser_admin = subparsers.add_parser("admin", help="administrate a clan") - admin.register_parser(parser_admin) + parser_create = subparsers.add_parser("create", help="create a clan flake") + create.register_parser(parser_create) - # DISABLED: this currently crashes if a flake does not define .#clanOptions - if os.environ.get("CLAN_OPTIONS_FILE") is not None: - parser_config = subparsers.add_parser("config", help="set nixos configuration") - config.register_parser(parser_config) + parser_config = subparsers.add_parser("config", help="set nixos configuration") + config.register_parser(parser_config) parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine") ssh_cli.register_parser(parser_ssh) diff --git a/pkgs/clan-cli/clan_cli/admin.py b/pkgs/clan-cli/clan_cli/admin.py deleted file mode 100644 index 2efe081..0000000 --- a/pkgs/clan-cli/clan_cli/admin.py +++ /dev/null @@ -1,37 +0,0 @@ -# !/usr/bin/env python3 -import argparse -import os -import subprocess - - -def create(args: argparse.Namespace) -> None: - os.makedirs(args.folder, exist_ok=True) - # TODO create clan template in flake - subprocess.run( - [ - "nix", - "flake", - "init", - "-t", - "git+https://git.clan.lol/clan/clan-core#new-clan", - ] - ) - - -# takes a (sub)parser and configures it -def register_parser(parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "-f", - "--folder", - help="the folder where the clan is defined, default to the current folder", - default=os.environ["PWD"], - ) - subparser = parser.add_subparsers( - title="command", - description="the command to run", - help="the command to run", - required=True, - ) - - parser_create = subparser.add_parser("create", help="create a new clan") - parser_create.set_defaults(func=create) diff --git a/pkgs/clan-cli/clan_cli/config/__init__.py b/pkgs/clan-cli/clan_cli/config/__init__.py index f73a9dc..1a3c0fb 100644 --- a/pkgs/clan-cli/clan_cli/config/__init__.py +++ b/pkgs/clan-cli/clan_cli/config/__init__.py @@ -9,6 +9,7 @@ from typing import Any, Optional, Type from clan_cli.dirs import get_clan_flake_toplevel from clan_cli.errors import ClanError +from clan_cli.machines.folders import machine_settings_file from clan_cli.nix import nix_eval script_dir = Path(__file__).parent @@ -100,7 +101,6 @@ def options_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict proc = subprocess.run( nix_eval( flags=[ - "--json", "--show-trace", "--impure", "--expr", @@ -138,7 +138,6 @@ def read_machine_option_value(machine_name: str, option: str) -> str: proc = subprocess.run( nix_eval( flags=[ - "--json", "--show-trace", "--extra-experimental-features", "nix-command flakes", @@ -168,7 +167,6 @@ def get_or_set_option(args: argparse.Namespace) -> None: print(read_machine_option_value(args.machine, args.option)) else: # load options - print(args.options_file) if args.options_file is None: options = options_for_machine(machine_name=args.machine) else: @@ -176,8 +174,8 @@ def get_or_set_option(args: argparse.Namespace) -> None: options = json.load(f) # compute settings json file location if args.settings_file is None: - flake = get_clan_flake_toplevel() - settings_file = flake / "machines" / f"{args.machine}.json" + get_clan_flake_toplevel() + settings_file = machine_settings_file(args.machine) else: settings_file = args.settings_file # set the option with the given value @@ -288,7 +286,7 @@ def register_parser( # add single positional argument for the option (e.g. "foo.bar") parser.add_argument( "option", - help="Option to configure", + help="Option to read or set", type=str, ) diff --git a/pkgs/clan-cli/clan_cli/config/machine.py b/pkgs/clan-cli/clan_cli/config/machine.py index 0dec9c1..cad6b50 100644 --- a/pkgs/clan-cli/clan_cli/config/machine.py +++ b/pkgs/clan-cli/clan_cli/config/machine.py @@ -12,7 +12,7 @@ from clan_cli.nix import nix_eval def config_for_machine(machine_name: str) -> dict: - # read the config from a json file located at {flake}/machines/{machine_name}.json + # read the config from a json file located at {flake}/machines/{machine_name}/settings.json if not machine_folder(machine_name).exists(): raise HTTPException( status_code=404, @@ -26,7 +26,7 @@ def config_for_machine(machine_name: str) -> dict: def set_config_for_machine(machine_name: str, config: dict) -> None: - # write the config to a json file located at {flake}/machines/{machine_name}.json + # write the config to a json file located at {flake}/machines/{machine_name}/settings.json if not machine_folder(machine_name).exists(): raise HTTPException( status_code=404, @@ -45,7 +45,6 @@ def schema_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict: proc = subprocess.run( nix_eval( flags=[ - "--json", "--impure", "--show-trace", "--extra-experimental-features", diff --git a/pkgs/clan-cli/clan_cli/config/parsing.py b/pkgs/clan-cli/clan_cli/config/parsing.py index 92b476b..3178224 100644 --- a/pkgs/clan-cli/clan_cli/config/parsing.py +++ b/pkgs/clan-cli/clan_cli/config/parsing.py @@ -3,7 +3,8 @@ import subprocess from pathlib import Path from typing import Any, Optional, Type, Union -from clan_cli.errors import ClanError +from ..errors import ClanError +from ..nix import nix_eval script_dir = Path(__file__).parent @@ -30,11 +31,9 @@ def schema_from_module_file( slib.parseModule {absolute_path} """ # run the nix expression and parse the output as json - return json.loads( - subprocess.check_output( - ["nix", "eval", "--impure", "--json", "--expr", nix_expr] - ) - ) + cmd = nix_eval(["--expr", nix_expr]) + proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True) + return json.loads(proc.stdout) def subtype_from_schema(schema: dict[str, Any]) -> Type: diff --git a/pkgs/clan-cli/clan_cli/create.py b/pkgs/clan-cli/clan_cli/create.py new file mode 100644 index 0000000..2a6b3ce --- /dev/null +++ b/pkgs/clan-cli/clan_cli/create.py @@ -0,0 +1,25 @@ +# !/usr/bin/env python3 +import argparse +import subprocess + +from .nix import nix_command + + +def create(args: argparse.Namespace) -> None: + # TODO create clan template in flake + subprocess.run( + nix_command( + [ + "flake", + "init", + "-t", + "git+https://git.clan.lol/clan/clan-core#new-clan", + ] + ), + check=True, + ) + + +# takes a (sub)parser and configures it +def register_parser(parser: argparse.ArgumentParser) -> None: + parser.set_defaults(func=create) diff --git a/pkgs/clan-cli/clan_cli/machines/update.py b/pkgs/clan-cli/clan_cli/machines/update.py index bed009f..9f5a170 100644 --- a/pkgs/clan-cli/clan_cli/machines/update.py +++ b/pkgs/clan-cli/clan_cli/machines/update.py @@ -1,7 +1,13 @@ import argparse import json +import os import subprocess +from typing import Optional +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_command, nix_eval +from ..secrets.generate import generate_secrets +from ..secrets.upload import upload_secrets from ..ssh import Host, HostGroup, HostKeyCheck @@ -13,11 +19,13 @@ def deploy_nixos(hosts: HostGroup) -> None: def deploy(h: Host) -> None: target = f"{h.user or 'root'}@{h.host}" ssh_arg = f"-p {h.port}" if h.port else "" + env = os.environ.copy() + env["NIX_SSHOPTS"] = ssh_arg res = h.run_local( - ["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"], + nix_command(["flake", "archive", "--to", f"ssh://{target}", "--json"]), check=True, stdout=subprocess.PIPE, - extra_env=dict(NIX_SSHOPTS=ssh_arg), + extra_env=env, ) data = json.loads(res.stdout) path = data["path"] @@ -29,6 +37,9 @@ def deploy_nixos(hosts: HostGroup) -> None: ssh_arg += " -i " + h.key if h.key else "" + generate_secrets(h.host) + upload_secrets(h.host) + flake_attr = h.meta.get("flake_attr", "") if flake_attr: flake_attr = "#" + flake_attr @@ -67,20 +78,46 @@ def deploy_nixos(hosts: HostGroup) -> None: # FIXME: we want some kind of inventory here. def update(args: argparse.Namespace) -> None: - meta = {} - if args.flake_uri: - meta["flake_uri"] = args.flake_uri - if args.flake_attr: - meta["flake_attr"] = args.flake_attr - deploy_nixos(HostGroup([Host(args.host, user=args.user, meta=meta)])) + clan_dir = get_clan_flake_toplevel().as_posix() + host = json.loads( + subprocess.run( + nix_eval( + [ + f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress' + ] + ), + stdout=subprocess.PIPE, + check=True, + text=True, + ).stdout + ) + parts = host.split("@") + user: Optional[str] = None + if len(parts) > 1: + user = parts[0] + hostname = parts[1] + else: + hostname = parts[0] + maybe_port = hostname.split(":") + port = None + if len(maybe_port) > 1: + hostname = maybe_port[0] + port = int(maybe_port[1]) + print(f"deploying {host}") + deploy_nixos( + HostGroup( + [ + Host( + host=hostname, + port=port, + user=user, + meta=dict(flake_attr=args.machine), + ) + ] + ) + ) def register_update_parser(parser: argparse.ArgumentParser) -> None: - # TODO pass all args we don't parse into ssh_args, currently it fails if arg starts with - - parser.add_argument("--flake-uri", type=str, default=".#", help="nix flake uri") - parser.add_argument( - "--flake-attr", type=str, help="nixos configuration in the flake" - ) - parser.add_argument("--user", type=str, default="root") - parser.add_argument("host", type=str) + parser.add_argument("machine", type=str) parser.set_defaults(func=update) diff --git a/pkgs/clan-cli/clan_cli/nix.py b/pkgs/clan-cli/clan_cli/nix.py index 7a29f02..72dbced 100644 --- a/pkgs/clan-cli/clan_cli/nix.py +++ b/pkgs/clan-cli/clan_cli/nix.py @@ -1,69 +1,54 @@ -import json import os import tempfile -from pathlib import Path -from .dirs import get_clan_flake_toplevel, nixpkgs_flake, nixpkgs_source, unfree_nixpkgs +from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs -def nix_build_machine( - machine: str, attr: list[str], flake_url: Path | None = None +def nix_command(flags: list[str]) -> list[str]: + return ["nix", "--experimental-features", "nix-command flakes"] + flags + + +def nix_build( + flags: list[str], ) -> list[str]: - if flake_url is None: - flake_url = get_clan_flake_toplevel() - payload = json.dumps( - dict( - clan_flake=flake_url, - machine=machine, - attr=attr, + return ( + nix_command( + [ + "build", + "--no-link", + "--print-out-paths", + "--extra-experimental-features", + "nix-command flakes", + ] ) + + flags ) - escaped_payload = json.dumps(payload) - return [ - "nix", - "build", - "--impure", - "--print-out-paths", - "--expr", - f"let args = builtins.fromJSON {escaped_payload}; in " - """ - let - flake = builtins.getFlake args.clan_flake; - config = flake.nixosConfigurations.${args.machine}.extendModules { - modules = [{ - clanCore.clanDir = args.clan_flake; - }]; - }; - in - flake.inputs.nixpkgs.lib.getAttrFromPath args.attr config - """, - ] def nix_eval(flags: list[str]) -> list[str]: + default_flags = nix_command( + [ + "eval", + "--show-trace", + "--json", + ] + ) if os.environ.get("IN_NIX_SANDBOX"): with tempfile.TemporaryDirectory() as nix_store: - return [ - "nix", - "eval", - "--show-trace", - "--extra-experimental-features", - "nix-command flakes", - "--override-input", - "nixpkgs", - str(nixpkgs_source()), - # --store is required to prevent this error: - # error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted - "--store", - nix_store, - ] + flags - return [ - "nix", - "eval", - "--show-trace", - "--extra-experimental-features", - "nix-command flakes", - ] + flags + return ( + default_flags + + [ + "--override-input", + "nixpkgs", + str(nixpkgs_source()), + # --store is required to prevent this error: + # error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted + "--store", + nix_store, + ] + + flags + ) + return default_flags + flags def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: @@ -73,14 +58,13 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]: return cmd wrapped_packages = [f"nixpkgs#{p}" for p in packages] return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "--inputs-from", - f"{str(nixpkgs_flake())}", - ] + nix_command( + [ + "shell", + "--inputs-from", + f"{str(nixpkgs_flake())}", + ] + ) + wrapped_packages + ["-c"] + cmd @@ -91,14 +75,13 @@ def unfree_nix_shell(packages: list[str], cmd: list[str]) -> list[str]: if os.environ.get("IN_NIX_SANDBOX"): return cmd return ( - [ - "nix", - "shell", - "--extra-experimental-features", - "nix-command flakes", - "-f", - str(unfree_nixpkgs()), - ] + nix_command( + [ + "shell", + "-f", + str(unfree_nixpkgs()), + ] + ) + packages + ["-c"] + cmd diff --git a/pkgs/clan-cli/clan_cli/secrets/__init__.py b/pkgs/clan-cli/clan_cli/secrets/__init__.py index 3c161d3..01ac958 100644 --- a/pkgs/clan-cli/clan_cli/secrets/__init__.py +++ b/pkgs/clan-cli/clan_cli/secrets/__init__.py @@ -7,6 +7,7 @@ from .import_sops import register_import_sops_parser from .key import register_key_parser from .machines import register_machines_parser from .secrets import register_secrets_parser +from .upload import register_upload_parser from .users import register_users_parser @@ -36,6 +37,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None: ) register_generate_parser(parser_generate) + parser_upload = subparser.add_parser("upload", help="upload secrets for machines") + register_upload_parser(parser_upload) + parser_key = subparser.add_parser("key", help="create and show age keys") register_key_parser(parser_key) diff --git a/pkgs/clan-cli/clan_cli/secrets/generate.py b/pkgs/clan-cli/clan_cli/secrets/generate.py index 0b01a8c..782bb55 100644 --- a/pkgs/clan-cli/clan_cli/secrets/generate.py +++ b/pkgs/clan-cli/clan_cli/secrets/generate.py @@ -1,24 +1,25 @@ import argparse +import os import subprocess import sys from clan_cli.errors import ClanError +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build + + +def generate_secrets(machine: str) -> None: + clan_dir = get_clan_flake_toplevel().as_posix().strip() + env = os.environ.copy() + env["CLAN_DIR"] = clan_dir -def get_secret_script(machine: str) -> None: proc = subprocess.run( - [ - "nix", - "build", - "--impure", - "--print-out-paths", - "--expr", - "let f = builtins.getFlake (toString ./.); in " - f"(f.nixosConfigurations.{machine}.extendModules " - "{ modules = [{ clanCore.clanDir = toString ./.; }]; })" - ".config.system.clan.generateSecrets", - ], - check=True, + nix_build( + [ + f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets' + ] + ), capture_output=True, text=True, ) @@ -30,7 +31,7 @@ def get_secret_script(machine: str) -> None: print(secret_generator_script) secret_generator = subprocess.run( [secret_generator_script], - check=True, + env=env, ) if secret_generator.returncode != 0: @@ -40,7 +41,7 @@ def get_secret_script(machine: str) -> None: def generate_command(args: argparse.Namespace) -> None: - get_secret_script(args.machine) + generate_secrets(args.machine) def register_generate_parser(parser: argparse.ArgumentParser) -> None: diff --git a/pkgs/clan-cli/clan_cli/secrets/upload.py b/pkgs/clan-cli/clan_cli/secrets/upload.py new file mode 100644 index 0000000..38a27b9 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/secrets/upload.py @@ -0,0 +1,60 @@ +import argparse +import json +import subprocess + +from clan_cli.errors import ClanError + +from ..dirs import get_clan_flake_toplevel +from ..nix import nix_build, nix_eval + + +def upload_secrets(machine: str) -> None: + clan_dir = get_clan_flake_toplevel().as_posix() + + proc = subprocess.run( + nix_build( + [ + f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets' + ] + ), + stdout=subprocess.PIPE, + text=True, + check=True, + ) + host = json.loads( + subprocess.run( + nix_eval( + [ + f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress' + ] + ), + stdout=subprocess.PIPE, + text=True, + check=True, + ).stdout + ) + + secret_upload_script = proc.stdout.strip() + secret_upload = subprocess.run( + [ + secret_upload_script, + host, + ], + ) + + if secret_upload.returncode != 0: + raise ClanError("failed to upload secrets") + else: + print("successfully uploaded secrets") + + +def upload_command(args: argparse.Namespace) -> None: + upload_secrets(args.machine) + + +def register_upload_parser(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "machine", + help="The machine to upload secrets to", + ) + parser.set_defaults(func=upload_command) diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index e839654..834f337 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -1,16 +1,30 @@ from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware from fastapi.routing import APIRoute from fastapi.staticfiles import StaticFiles from .assets import asset_path -from .routers import health, machines, root +from .routers import health, machines, root, vms + +origins = [ + "http://localhost:3000", +] def setup_app() -> FastAPI: app = FastAPI() + app.add_middleware( + CORSMiddleware, + allow_origins=origins, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) app.include_router(health.router) app.include_router(machines.router) app.include_router(root.router) + app.include_router(vms.router) + app.add_exception_handler(vms.NixBuildException, vms.nix_build_exception_handler) app.mount("/static", StaticFiles(directory=asset_path()), name="static") diff --git a/pkgs/clan-cli/clan_cli/webui/routers/vms.py b/pkgs/clan-cli/clan_cli/webui/routers/vms.py new file mode 100644 index 0000000..18c8c74 --- /dev/null +++ b/pkgs/clan-cli/clan_cli/webui/routers/vms.py @@ -0,0 +1,113 @@ +import asyncio +import json +import shlex +from typing import Annotated, AsyncIterator + +from fastapi import APIRouter, Body, HTTPException, Request, status +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse, StreamingResponse + +from ...nix import nix_build, nix_eval +from ..schemas import VmConfig, VmInspectResponse + +router = APIRouter() + + +class NixBuildException(HTTPException): + def __init__(self, msg: str, loc: list = ["body", "flake_attr"]): + detail = [ + { + "loc": loc, + "msg": msg, + "type": "value_error", + } + ] + super().__init__( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail + ) + + +def nix_build_exception_handler( + request: Request, exc: NixBuildException +) -> JSONResponse: + return JSONResponse( + status_code=exc.status_code, + content=jsonable_encoder(dict(detail=exc.detail)), + ) + + +def nix_inspect_vm(machine: str, flake_url: str) -> list[str]: + return nix_eval( + [ + f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.clan.vm.config" + ] + ) + + +def nix_build_vm(machine: str, flake_url: str) -> list[str]: + return nix_build( + [ + f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.build.vm" + ] + ) + + +@router.post("/api/vms/inspect") +async def inspect_vm( + flake_url: Annotated[str, Body()], flake_attr: Annotated[str, Body()] +) -> VmInspectResponse: + cmd = nix_inspect_vm(flake_attr, flake_url=flake_url) + proc = await asyncio.create_subprocess_exec( + cmd[0], + *cmd[1:], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + + if proc.returncode != 0: + raise NixBuildException( + f""" +Failed to evaluate vm from '{flake_url}#{flake_attr}'. +command: {shlex.join(cmd)} +exit code: {proc.returncode} +command output: +{stderr.decode("utf-8")} +""" + ) + data = json.loads(stdout) + return VmInspectResponse( + config=VmConfig(flake_url=flake_url, flake_attr=flake_attr, **data) + ) + + +async def vm_build(vm: VmConfig) -> AsyncIterator[str]: + cmd = nix_build_vm(vm.flake_attr, flake_url=vm.flake_url) + proc = await asyncio.create_subprocess_exec( + cmd[0], + *cmd[1:], + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + assert proc.stdout is not None and proc.stderr is not None + async for line in proc.stdout: + yield line.decode("utf-8", "ignore") + stderr = "" + async for line in proc.stderr: + stderr += line.decode("utf-8", "ignore") + res = await proc.wait() + if res != 0: + raise NixBuildException( + f""" +Failed to build vm from '{vm.flake_url}#{vm.flake_attr}'. +command: {shlex.join(cmd)} +exit code: {res} +command output: +{stderr} + """ + ) + + +@router.post("/api/vms/create") +async def create_vm(vm: Annotated[VmConfig, Body()]) -> StreamingResponse: + return StreamingResponse(vm_build(vm)) diff --git a/pkgs/clan-cli/clan_cli/webui/schemas.py b/pkgs/clan-cli/clan_cli/webui/schemas.py index 90c5437..dc6ea3f 100644 --- a/pkgs/clan-cli/clan_cli/webui/schemas.py +++ b/pkgs/clan-cli/clan_cli/webui/schemas.py @@ -32,3 +32,16 @@ class ConfigResponse(BaseModel): class SchemaResponse(BaseModel): schema_: dict = Field(alias="schema") + + +class VmConfig(BaseModel): + flake_url: str + flake_attr: str + + cores: int + memory_size: int + graphics: bool + + +class VmInspectResponse(BaseModel): + config: VmConfig diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 7d915a7..1b7164f 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -62,10 +62,18 @@ def start_server(args: argparse.Namespace) -> None: if ":" in host: host = f"[{host}]" headers = [ - ( - "Access-Control-Allow-Origin", - f"http://{host}:{args.dev_port}", - ) + # ( + # "Access-Control-Allow-Origin", + # f"http://{host}:{args.dev_port}", + # ), + # ( + # "Access-Control-Allow-Methods", + # "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT" + # ), + # ( + # "Allow", + # "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT" + # ) ] else: open_url = f"http://[{args.host}]:{args.port}" diff --git a/pkgs/clan-cli/default.nix b/pkgs/clan-cli/default.nix index c73810b..f41152f 100644 --- a/pkgs/clan-cli/default.nix +++ b/pkgs/clan-cli/default.nix @@ -22,9 +22,6 @@ , ui-assets }: let - # This provides dummy options for testing clan config and prevents it from - # evaluating the flake .# - CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json; dependencies = [ argcomplete # optional dependency: if not enabled, shell completion will not work @@ -54,9 +51,9 @@ let ''; nixpkgs = runCommand "nixpkgs" { nativeBuildInputs = [ pkgs.nix ]; } '' mkdir $out - mkdir -p $out/unfree - cat > $out/unfree/default.nix < $out/unfree/default.nix < $out/flake.nix << EOF { @@ -81,8 +78,6 @@ python3.pkgs.buildPythonPackage { src = source; format = "pyproject"; - inherit CLAN_OPTIONS_FILE; - nativeBuildInputs = [ setuptools installShellFiles @@ -93,12 +88,12 @@ python3.pkgs.buildPythonPackage { { nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ]; } '' - export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}" cp -r ${source} ./src chmod +w -R ./src cd ./src - NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 ${checkPython}/bin/python -m pytest -s ./tests + export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 + ${checkPython}/bin/python -m pytest -m "not impure" -s ./tests touch $out ''; passthru.clan-openapi = runCommand "clan-openapi" { } '' diff --git a/pkgs/clan-cli/pyproject.toml b/pkgs/clan-cli/pyproject.toml index 127d6cf..1c16c93 100644 --- a/pkgs/clan-cli/pyproject.toml +++ b/pkgs/clan-cli/pyproject.toml @@ -18,6 +18,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"] faulthandler_timeout = 30 addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5" norecursedirs = "tests/helpers" +markers = [ "impure" ] [tool.mypy] python_version = "3.10" diff --git a/pkgs/clan-cli/shell.nix b/pkgs/clan-cli/shell.nix index 336b2f0..f95340c 100644 --- a/pkgs/clan-cli/shell.nix +++ b/pkgs/clan-cli/shell.nix @@ -6,6 +6,7 @@ let ++ clan-cli.devDependencies ++ [ ps.pip + ps.ipdb ] ); checkScript = writeScriptBin "check" '' @@ -19,10 +20,9 @@ mkShell { pythonWithDeps ]; # sets up an editable install and add enty points to $PATH - # This provides dummy options for testing clan config and prevents it from - # evaluating the flake .# - CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json; PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}"; + PYTHONBREAKPOINT = "ipdb.set_trace"; + shellHook = '' tmp_path=$(realpath ./.direnv) diff --git a/pkgs/clan-cli/tests/clan_flake.py b/pkgs/clan-cli/tests/clan_flake.py deleted file mode 100644 index 23d38b4..0000000 --- a/pkgs/clan-cli/tests/clan_flake.py +++ /dev/null @@ -1,23 +0,0 @@ -from pathlib import Path -from typing import Iterator - -import pytest - - -@pytest.fixture -def clan_flake(temporary_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]: - flake = temporary_dir / "clan-flake" - flake.mkdir() - (flake / ".clan-flake").touch() - (flake / "flake.nix").write_text( - """ -{ - description = "A flake for testing clan"; - inputs = {}; - outputs = { self }: {}; -} -""" - ) - monkeypatch.chdir(flake) - monkeypatch.setenv("HOME", str(temporary_dir)) - yield flake diff --git a/pkgs/clan-cli/tests/conftest.py b/pkgs/clan-cli/tests/conftest.py index 70ebd82..c502c73 100644 --- a/pkgs/clan-cli/tests/conftest.py +++ b/pkgs/clan-cli/tests/conftest.py @@ -1,51 +1,16 @@ import os import sys -import tempfile -from pathlib import Path -from typing import Generator - -import pytest - -from clan_cli.dirs import nixpkgs_source sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) pytest_plugins = [ "api", "temporary_dir", - "clan_flake", "root", "age_keys", "sshd", "command", "ports", "host_group", + "test_flake", ] - - -@pytest.fixture(scope="module") -def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]: - with pytest.MonkeyPatch.context() as mp: - yield mp - - -@pytest.fixture(scope="module") -def machine_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]: - template = Path(__file__).parent / "machine_flake" - # copy the template to a new temporary location - with tempfile.TemporaryDirectory() as tmpdir_: - flake = Path(tmpdir_) - for path in template.glob("**/*"): - if path.is_dir(): - (flake / path.relative_to(template)).mkdir() - else: - (flake / path.relative_to(template)).write_text(path.read_text()) - # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake - # provided by get_clan_flake_toplevel - flake_nix = flake / "flake.nix" - flake_nix.write_text( - flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source())) - ) - # check that an empty config is returned if no json file exists - monkeymodule.chdir(flake) - yield flake diff --git a/pkgs/clan-cli/tests/root.py b/pkgs/clan-cli/tests/root.py index c881ce0..0105e82 100644 --- a/pkgs/clan-cli/tests/root.py +++ b/pkgs/clan-cli/tests/root.py @@ -4,12 +4,13 @@ import pytest TEST_ROOT = Path(__file__).parent.resolve() PROJECT_ROOT = TEST_ROOT.parent +CLAN_CORE = PROJECT_ROOT.parent.parent @pytest.fixture(scope="session") def project_root() -> Path: """ - Root directory of the tests + Root directory the clan-cli """ return PROJECT_ROOT @@ -20,3 +21,11 @@ def test_root() -> Path: Root directory of the tests """ return TEST_ROOT + + +@pytest.fixture(scope="session") +def clan_core() -> Path: + """ + Directory of the clan-core flake + """ + return CLAN_CORE diff --git a/pkgs/clan-cli/tests/test_admin_cli.py b/pkgs/clan-cli/tests/test_admin_cli.py deleted file mode 100644 index dba43d1..0000000 --- a/pkgs/clan-cli/tests/test_admin_cli.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import Union - -import pytest_subprocess.fake_process -from cli import Cli -from pytest_subprocess import utils - - -# using fp fixture from pytest-subprocess -def test_create(fp: pytest_subprocess.fake_process.FakeProcess) -> None: - cmd: list[Union[str, utils.Any]] = ["nix", "flake", "init", "-t", fp.any()] - fp.register(cmd) - cli = Cli() - cli.run(["admin", "--folder", "./my-clan", "create"]) - assert fp.call_count(cmd) == 1 diff --git a/pkgs/clan-cli/tests/test_clan_template.py b/pkgs/clan-cli/tests/test_clan_template.py new file mode 100644 index 0000000..10b4a9b --- /dev/null +++ b/pkgs/clan-cli/tests/test_clan_template.py @@ -0,0 +1,12 @@ +from pathlib import Path + +import pytest +from cli import Cli + + +@pytest.mark.impure +def test_template(monkeypatch: pytest.MonkeyPatch, temporary_dir: Path) -> None: + monkeypatch.chdir(temporary_dir) + cli = Cli() + cli.run(["create"]) + assert (temporary_dir / ".clan-flake").exists() diff --git a/pkgs/clan-cli/tests/test_config.py b/pkgs/clan-cli/tests/test_config.py index 79fc2d8..c65ef11 100644 --- a/pkgs/clan-cli/tests/test_config.py +++ b/pkgs/clan-cli/tests/test_config.py @@ -50,7 +50,7 @@ def test_set_some_option( def test_configure_machine( - machine_flake: Path, + test_flake: Path, temporary_dir: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, diff --git a/pkgs/clan-cli/tests/test_flake.py b/pkgs/clan-cli/tests/test_flake.py new file mode 100644 index 0000000..66083b1 --- /dev/null +++ b/pkgs/clan-cli/tests/test_flake.py @@ -0,0 +1,53 @@ +import fileinput +import shutil +import tempfile +from pathlib import Path +from typing import Iterator + +import pytest +from root import CLAN_CORE + +from clan_cli.dirs import nixpkgs_source + + +@pytest.fixture(scope="module") +def monkeymodule() -> Iterator[pytest.MonkeyPatch]: + with pytest.MonkeyPatch.context() as mp: + yield mp + + +def create_flake( + monkeymodule: pytest.MonkeyPatch, name: str, clan_core_flake: Path | None = None +) -> Iterator[Path]: + template = Path(__file__).parent / name + # copy the template to a new temporary location + with tempfile.TemporaryDirectory() as tmpdir_: + home = Path(tmpdir_) + flake = home / name + shutil.copytree(template, flake) + # in the flake.nix file replace the string __CLAN_URL__ with the the clan flake + # provided by get_test_flake_toplevel + flake_nix = flake / "flake.nix" + for line in fileinput.input(flake_nix, inplace=True): + line = line.replace("__NIXPKGS__", str(nixpkgs_source())) + if clan_core_flake: + line = line.replace("__CLAN_CORE__", str(clan_core_flake)) + print(line) + # check that an empty config is returned if no json file exists + monkeymodule.chdir(flake) + monkeymodule.setenv("HOME", str(home)) + yield flake + + +@pytest.fixture(scope="module") +def test_flake(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: + yield from create_flake(monkeymodule, "test_flake") + + +@pytest.fixture(scope="module") +def test_flake_with_core(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]: + if not (CLAN_CORE / "flake.nix").exists(): + raise Exception( + "clan-core flake not found. This test requires the clan-core flake to be present" + ) + yield from create_flake(monkeymodule, "test_flake_with_core", CLAN_CORE) diff --git a/pkgs/clan-cli/tests/machine_flake/.clan-flake b/pkgs/clan-cli/tests/test_flake/.clan-flake similarity index 100% rename from pkgs/clan-cli/tests/machine_flake/.clan-flake rename to pkgs/clan-cli/tests/test_flake/.clan-flake diff --git a/pkgs/clan-cli/tests/machine_flake/flake.nix b/pkgs/clan-cli/tests/test_flake/flake.nix similarity index 73% rename from pkgs/clan-cli/tests/machine_flake/flake.nix rename to pkgs/clan-cli/tests/test_flake/flake.nix index ff220a5..23d2a67 100644 --- a/pkgs/clan-cli/tests/machine_flake/flake.nix +++ b/pkgs/clan-cli/tests/test_flake/flake.nix @@ -1,15 +1,13 @@ { - inputs = { - # this placeholder is replaced by the path to nixpkgs - nixpkgs.url = "__NIXPKGS__"; - }; + # this placeholder is replaced by the path to nixpkgs + inputs.nixpkgs.url = "__NIXPKGS__"; outputs = inputs: { nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem { modules = [ ./nixosModules/machine1.nix - (if builtins.pathExists ./machines/machine1.json - then builtins.fromJSON (builtins.readFile ./machines/machine1.json) + (if builtins.pathExists ./machines/machine1/settings.json + then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json) else { }) { nixpkgs.hostPlatform = "x86_64-linux"; diff --git a/pkgs/clan-cli/tests/machine_flake/nixosModules/machine1.nix b/pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix similarity index 100% rename from pkgs/clan-cli/tests/machine_flake/nixosModules/machine1.nix rename to pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix diff --git a/pkgs/clan-cli/tests/test_flake_with_core/.clan-flake b/pkgs/clan-cli/tests/test_flake_with_core/.clan-flake new file mode 100644 index 0000000..e69de29 diff --git a/pkgs/clan-cli/tests/test_flake_with_core/flake.nix b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix new file mode 100644 index 0000000..fab76b5 --- /dev/null +++ b/pkgs/clan-cli/tests/test_flake_with_core/flake.nix @@ -0,0 +1,18 @@ +{ + # Use this path to our repo root e.g. for UI test + # inputs.clan-core.url = "../../../../."; + + # this placeholder is replaced by the path to nixpkgs + inputs.clan-core.url = "__CLAN_CORE__"; + + outputs = { self, clan-core }: { + nixosConfigurations = clan-core.lib.buildClan { + directory = self; + machines = { + vm1 = { modulesPath, ... }: { + imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ]; + }; + }; + }; + }; +} diff --git a/pkgs/clan-cli/tests/test_import_sops_cli.py b/pkgs/clan-cli/tests/test_import_sops_cli.py index 6ea6a85..64f68c9 100644 --- a/pkgs/clan-cli/tests/test_import_sops_cli.py +++ b/pkgs/clan-cli/tests/test_import_sops_cli.py @@ -10,7 +10,7 @@ if TYPE_CHECKING: def test_import_sops( test_root: Path, - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], diff --git a/pkgs/clan-cli/tests/test_machines_api.py b/pkgs/clan-cli/tests/test_machines_api.py index 2551fbf..2fba075 100644 --- a/pkgs/clan-cli/tests/test_machines_api.py +++ b/pkgs/clan-cli/tests/test_machines_api.py @@ -3,7 +3,7 @@ from pathlib import Path from api import TestClient -def test_machines(api: TestClient, clan_flake: Path) -> None: +def test_machines(api: TestClient, test_flake: Path) -> None: response = api.get("/api/machines") assert response.status_code == 200 assert response.json() == {"machines": []} @@ -21,7 +21,7 @@ def test_machines(api: TestClient, clan_flake: Path) -> None: assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]} -def test_configure_machine(api: TestClient, machine_flake: Path) -> None: +def test_configure_machine(api: TestClient, test_flake: Path) -> None: # ensure error 404 if machine does not exist when accessing the config response = api.get("/api/machines/machine1/config") assert response.status_code == 404 diff --git a/pkgs/clan-cli/tests/test_machines_cli.py b/pkgs/clan-cli/tests/test_machines_cli.py index a2e3fa8..6b92645 100644 --- a/pkgs/clan-cli/tests/test_machines_cli.py +++ b/pkgs/clan-cli/tests/test_machines_cli.py @@ -4,7 +4,7 @@ import pytest from cli import Cli -def test_machine_subcommands(clan_flake: Path, capsys: pytest.CaptureFixture) -> None: +def test_machine_subcommands(test_flake: Path, capsys: pytest.CaptureFixture) -> None: cli = Cli() cli.run(["machines", "create", "machine1"]) diff --git a/pkgs/clan-cli/tests/test_machines_config.py b/pkgs/clan-cli/tests/test_machines_config.py index 2484435..4d3a0a7 100644 --- a/pkgs/clan-cli/tests/test_machines_config.py +++ b/pkgs/clan-cli/tests/test_machines_config.py @@ -3,6 +3,6 @@ from pathlib import Path from clan_cli.config import machine -def test_schema_for_machine(machine_flake: Path) -> None: - schema = machine.schema_for_machine("machine1", machine_flake) +def test_schema_for_machine(test_flake: Path) -> None: + schema = machine.schema_for_machine("machine1", test_flake) assert "properties" in schema diff --git a/pkgs/clan-cli/tests/test_machines_update_cli.py b/pkgs/clan-cli/tests/test_machines_update_cli.py deleted file mode 100644 index 62650f3..0000000 --- a/pkgs/clan-cli/tests/test_machines_update_cli.py +++ /dev/null @@ -1,35 +0,0 @@ -import os -import shutil -from pathlib import Path -from tempfile import TemporaryDirectory - -import pytest -from host_group import HostGroup - - -def test_update( - clan_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch -) -> None: - assert len(host_group.hosts) == 1 - host = host_group.hosts[0] - - with TemporaryDirectory() as tmpdir: - host.meta["flake_uri"] = clan_flake - host.meta["flake_path"] = str(Path(tmpdir) / "rsync-target") - host.ssh_options["SendEnv"] = "REALPATH" - bin = Path(tmpdir).joinpath("bin") - bin.mkdir() - nixos_rebuild = bin.joinpath("nixos-rebuild") - bash = shutil.which("bash") - assert bash is not None - nixos_rebuild.write_text( - f"""#!{bash} -exit 0 -""" - ) - nixos_rebuild.chmod(0o755) - f"{tmpdir}/bin:{os.environ['PATH']}" - nix_state_dir = Path(tmpdir).joinpath("nix") - nix_state_dir.mkdir() - - monkeypatch.setenv("REALPATH", str(nix_state_dir)) diff --git a/pkgs/clan-cli/tests/test_secrets_cli.py b/pkgs/clan-cli/tests/test_secrets_cli.py index af604a8..7f3c816 100644 --- a/pkgs/clan-cli/tests/test_secrets_cli.py +++ b/pkgs/clan-cli/tests/test_secrets_cli.py @@ -14,12 +14,12 @@ if TYPE_CHECKING: def _test_identities( what: str, - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"], ) -> None: cli = Cli() - sops_folder = clan_flake / "sops" + sops_folder = test_flake / "sops" cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey]) assert (sops_folder / what / "foo" / "key.json").exists() @@ -60,19 +60,19 @@ def _test_identities( def test_users( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("users", clan_flake, capsys, age_keys) + _test_identities("users", test_flake, capsys, age_keys) def test_machines( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: - _test_identities("machines", clan_flake, capsys, age_keys) + _test_identities("machines", test_flake, capsys, age_keys) def test_groups( - clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] + test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"] ) -> None: cli = Cli() capsys.readouterr() # empty the buffer @@ -100,7 +100,7 @@ def test_groups( cli.run(["secrets", "groups", "remove-user", "group1", "user1"]) cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"]) - groups = os.listdir(clan_flake / "sops" / "groups") + groups = os.listdir(test_flake / "sops" / "groups") assert len(groups) == 0 @@ -114,7 +114,7 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: def test_secrets( - clan_flake: Path, + test_flake: Path, capsys: pytest.CaptureFixture, monkeypatch: pytest.MonkeyPatch, age_keys: list["KeyPair"], @@ -125,7 +125,7 @@ def test_secrets( assert capsys.readouterr().out == "" monkeypatch.setenv("SOPS_NIX_SECRET", "foo") - monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(clan_flake / ".." / "age.key")) + monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake / ".." / "age.key")) cli.run(["secrets", "key", "generate"]) capsys.readouterr() # empty the buffer cli.run(["secrets", "key", "show"]) diff --git a/pkgs/clan-cli/tests/test_ssh_cli.py b/pkgs/clan-cli/tests/test_ssh_cli.py index 11b839f..8a7f43d 100644 --- a/pkgs/clan-cli/tests/test_ssh_cli.py +++ b/pkgs/clan-cli/tests/test_ssh_cli.py @@ -30,9 +30,8 @@ def test_ssh_no_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", @@ -61,9 +60,8 @@ def test_ssh_with_pass( monkeypatch.delenv("IN_NIX_SANDBOX") cmd: list[Union[str, utils.Any]] = [ "nix", + fp.any(), "shell", - "--extra-experimental-features", - "nix-command flakes", fp.any(), "-c", "torify", diff --git a/pkgs/clan-cli/tests/test_vms_api.py b/pkgs/clan-cli/tests/test_vms_api.py new file mode 100644 index 0000000..8935e6c --- /dev/null +++ b/pkgs/clan-cli/tests/test_vms_api.py @@ -0,0 +1,33 @@ +from pathlib import Path + +import pytest +from api import TestClient + + +@pytest.mark.impure +def test_inspect(api: TestClient, test_flake_with_core: Path) -> None: + response = api.post( + "/api/vms/inspect", + json=dict(flake_url=str(test_flake_with_core), flake_attr="vm1"), + ) + assert response.status_code == 200, "Failed to inspect vm" + config = response.json()["config"] + assert config.get("flake_attr") == "vm1" + assert config.get("cores") == 1 + assert config.get("memory_size") == 1024 + assert config.get("graphics") is True + + +@pytest.mark.impure +def test_create(api: TestClient, test_flake_with_core: Path) -> None: + response = api.post( + "/api/vms/create", + json=dict( + flake_url=str(test_flake_with_core), + flake_attr="vm1", + cores=1, + memory_size=1024, + graphics=True, + ), + ) + assert response.status_code == 200, "Failed to inspect vm" diff --git a/pkgs/ui/nix/pdefs.nix b/pkgs/ui/nix/pdefs.nix index 5224db1..863553a 100644 --- a/pkgs/ui/nix/pdefs.nix +++ b/pkgs/ui/nix/pdefs.nix @@ -10986,6 +10986,11 @@ descriptor = "^0.4.1"; pin = "0.4.1"; }; + pretty-bytes = { + descriptor = "^6.1.1"; + pin = "6.1.1"; + runtime = true; + }; react = { descriptor = "18.2.0"; pin = "18.2.0"; @@ -13086,6 +13091,9 @@ dev = true; key = "prettier-plugin-tailwindcss/0.4.1"; }; + "node_modules/pretty-bytes" = { + key = "pretty-bytes/6.1.1"; + }; "node_modules/printable-characters" = { dev = true; key = "printable-characters/1.0.42"; @@ -15195,6 +15203,19 @@ version = "0.4.1"; }; }; + pretty-bytes = { + "6.1.1" = { + fetchInfo = { + narHash = "sha256-ERXqMD/9tkPebbHVL3n/9EQRz7mFs5VYO6k/wo5JDzQ="; + type = "tarball"; + url = "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz"; + }; + ident = "pretty-bytes"; + ltype = "file"; + treeInfo = { }; + version = "6.1.1"; + }; + }; printable-characters = { "1.0.42" = { fetchInfo = { diff --git a/pkgs/ui/orval.config.ts b/pkgs/ui/orval.config.ts index db48112..b92bd3d 100644 --- a/pkgs/ui/orval.config.ts +++ b/pkgs/ui/orval.config.ts @@ -1,5 +1,5 @@ const config = { - petstore: { + clan: { output: { mode: "tags-split", target: "src/api", diff --git a/pkgs/ui/package-lock.json b/pkgs/ui/package-lock.json index 5462475..fc249db 100644 --- a/pkgs/ui/package-lock.json +++ b/pkgs/ui/package-lock.json @@ -22,6 +22,7 @@ "hex-rgb": "^5.0.0", "next": "13.4.12", "postcss": "8.4.27", + "pretty-bytes": "^6.1.1", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.45.4", @@ -6810,6 +6811,17 @@ } } }, + "node_modules/pretty-bytes": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz", + "integrity": "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ==", + "engines": { + "node": "^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/printable-characters": { "version": "1.0.42", "resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz", diff --git a/pkgs/ui/package.json b/pkgs/ui/package.json index 6ae1b83..02e28cd 100644 --- a/pkgs/ui/package.json +++ b/pkgs/ui/package.json @@ -26,6 +26,7 @@ "hex-rgb": "^5.0.0", "next": "13.4.12", "postcss": "8.4.27", + "pretty-bytes": "^6.1.1", "react": "18.2.0", "react-dom": "18.2.0", "react-hook-form": "^7.45.4", diff --git a/pkgs/ui/src/app/join/page.tsx b/pkgs/ui/src/app/join/page.tsx new file mode 100644 index 0000000..660c407 --- /dev/null +++ b/pkgs/ui/src/app/join/page.tsx @@ -0,0 +1,184 @@ +"use client"; +import React, { useState } from "react"; +import { VmConfig } from "@/api/model"; +import { useVms } from "@/components/hooks/useVms"; +import prettyBytes from "pretty-bytes"; + +import { + Alert, + AlertTitle, + Button, + Chip, + LinearProgress, + ListSubheader, + Switch, + Typography, +} from "@mui/material"; +import { useSearchParams } from "next/navigation"; +import { toast } from "react-hot-toast"; +import { Error, Numbers } from "@mui/icons-material"; +import { createVm, inspectVm } from "@/api/default/default"; + +interface FlakeBadgeProps { + flakeUrl: string; + flakeAttr: string; +} +const FlakeBadge = (props: FlakeBadgeProps) => ( + +); + +interface VmPropLabelProps { + children: React.ReactNode; +} +const VmPropLabel = (props: VmPropLabelProps) => ( +
+ {props.children} +
+); + +interface VmPropContentProps { + children: React.ReactNode; +} +const VmPropContent = (props: VmPropContentProps) => ( +
{props.children}
+); + +interface VmDetailsProps { + vmConfig: VmConfig; +} + +const VmDetails = (props: VmDetailsProps) => { + const { vmConfig } = props; + const { cores, flake_attr, flake_url, graphics, memory_size } = vmConfig; + const [isStarting, setStarting] = useState(false); + const handleStartVm = async () => { + setStarting(true); + const response = await createVm(vmConfig); + setStarting(false); + if (response.statusText === "OK") { + toast.success(("VM created @ " + response?.data) as string); + } else { + toast.error("Could not create VM"); + } + }; + return ( +
+
+ General +
+ + Flake + + + + + Machine + {flake_attr} + +
+ VM +
+ CPU Cores + + + {cores} + + + Graphics + + + + + Memory Size + {prettyBytes(memory_size * 1024 * 1024)} + +
+ {isStarting && } + +
+
+ ); +}; + +interface ErrorLogOptions { + lines: string[]; +} +const ErrorLog = (props: ErrorLogOptions) => { + const { lines } = props; + return ( +
+
Log
+ {lines.map((item, idx) => ( + + {item} +
+ + ))} +
+ ); +}; + +export default function Page() { + const queryParams = useSearchParams(); + const flakeUrl = queryParams.get("flake") || ""; + const flakeAttribute = queryParams.get("attr") || "default"; + + const { config, error, isLoading } = useVms({ + url: flakeUrl, + attr: flakeAttribute, + }); + const clanName = "Lassul.us"; + return ( +
+ + Join{" "} + + {clanName} + + {"' "} + Clan + + {error && ( + + Error + An Error occurred - See details below + + )} +
+ {isLoading && ( +
+ Loading Flake + +
+ +
+ + +
+ )} + {(!flakeUrl || !flakeAttribute) &&
Invalid URL
} + {config && } + {error && ( + err.msg.split("\n")) + ?.flat() + .filter(Boolean) || [] + } + /> + )} +
+
+ ); +} diff --git a/pkgs/ui/src/components/hooks/useVms.tsx b/pkgs/ui/src/components/hooks/useVms.tsx new file mode 100644 index 0000000..3901a25 --- /dev/null +++ b/pkgs/ui/src/components/hooks/useVms.tsx @@ -0,0 +1,50 @@ +import { inspectVm } from "@/api/default/default"; +import { HTTPValidationError, VmConfig } from "@/api/model"; +import { AxiosError } from "axios"; +import { useEffect, useState } from "react"; +import { toast } from "react-hot-toast"; + +interface UseVmsOptions { + url: string; + attr: string; +} +export const useVms = (options: UseVmsOptions) => { + const { url, attr } = options; + const [isLoading, setIsLoading] = useState(true); + const [config, setConfig] = useState(); + const [error, setError] = useState>(); + + useEffect(() => { + const getVmInfo = async (url: string, attr: string) => { + if (url === "") { + toast.error("Flake url is missing", { id: "missing.flake.url" }); + return undefined; + } + try { + const response = await inspectVm({ + flake_attr: attr, + flake_url: url, + }); + const { + data: { config }, + } = response; + setError(undefined); + return config; + } catch (e) { + const err = e as AxiosError; + setError(err); + toast.error(err.message); + return undefined; + } finally { + setIsLoading(false); + } + }; + getVmInfo(url, attr).then((c) => setConfig(c)); + }, [url, attr]); + + return { + error, + isLoading, + config, + }; +};