Luis/nextjs-python-web-template
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' of git.clan.lol:clan/clan-core into Qubasa-main

Browse Source
This commit is contained in:
Luis-Hebendanz
2023-09-19 15:30:25 +02:00
parent 2c80f6feac 07d5b2bf1c
commit 63b6b14109
59 changed files with 1005 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitea/workflows/checks.yaml
View File

@@ -8,4 +8,4 @@ jobs:
runs-on: nix
steps:
- uses: actions/checkout@v3
- run: nix run --refresh github:Mic92/nix-ci-build
- run: nix run --refresh github:Mic92/nix-fast-build/ae50c356c2f9e790f3d9d8e00bfa9f4b54f49bdd

6
.gitignore vendored
View File

@@ -1,8 +1,8 @@
.direnv
result*
pkgs/clan-cli/clan_cli/nixpkgs
pkgs/clan-cli/clan_cli/webui/assets
machines
/pkgs/clan-cli/clan_cli/nixpkgs
/pkgs/clan-cli/clan_cli/webui/assets
/machines
# python
__pycache__

2
devShell.nix
View File

@@ -15,7 +15,7 @@
];
shellHook = ''
# no longer used
rm "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
rm -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit"
'';
};
};

4
docs/quickstart.md
View File

@@ -43,6 +43,7 @@ Absolutely, let's break down the migration step by step, explaining each action
```nix
inputs.clan-core = {
url = "git+https://git.clan.lol/clan/clan-core";
# Don't do this if your machines are on nixpkgs stable.
inputs.nixpkgs.follows = "nixpkgs";
};
```
@@ -75,7 +76,8 @@ Absolutely, let's break down the migration step by step, explaining each action
```nix
nixosConfigurations = clan-core.lib.buildClan {
directory = ./.;
# this needs to point at the repository root
directory = self;
specialArgs = {};
machines = {
example-desktop = {

2
docs/secrets-management.md
View File

@@ -86,7 +86,7 @@ $ clan secrets machines list
For existing machines, add their keys:
```console
$ clan secrets machine add <machine_name> <age_key>
$ clan secrets machines add <machine_name> <age_key>
```
To fetch an age key from an SSH host key:

24
flake.lock generated
View File

@@ -7,11 +7,11 @@
]
},
"locked": {
"lastModified": 1694511957,
"narHash": "sha256-teCLY68npc0nuyOHYJURLuJSOME0yotJI29WXcpF1E4=",
"lastModified": 1694925805,
"narHash": "sha256-UNMivSc89undITtNoy6o6bf3Dck4v75rzGiMEXAPEB0=",
"owner": "nix-community",
"repo": "disko",
"rev": "be98cffef02e5ebf438ea80b34b86e669c48eff1",
"rev": "9ab96378f8cf602d5f3ce5a32f2c339509288d8e",
"type": "github"
},
"original": {
@@ -47,11 +47,11 @@
]
},
"locked": {
"lastModified": 1691024356,
"narHash": "sha256-uGLyhkwew6ORO6nAz0Y7KHdiQrDJVI2n6rl4gl7mWzk=",
"lastModified": 1694873346,
"narHash": "sha256-Uvh03bg0a6ZnNWiX1Gb8g+m343wSJ/wb8ryUASt0loc=",
"owner": "aakropotkin",
"repo": "floco",
"rev": "1e84b4b16bba5746e1195fa3a4d8addaaf2d9ef4",
"rev": "d16bd444ab9d29a6640f52ee4e43a66528e07515",
"type": "github"
},
"original": {
@@ -98,11 +98,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1694422566,
"narHash": "sha256-lHJ+A9esOz9vln/3CJG23FV6Wd2OoOFbDeEs4cMGMqc=",
"lastModified": 1694767346,
"narHash": "sha256-5uH27SiVFUwsTsqC5rs3kS7pBoNhtoy9QfTP9BmknGk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "3a2786eea085f040a66ecde1bc3ddc7099f6dbeb",
"rev": "ace5093e36ab1e95cb9463863491bee90d5a4183",
"type": "github"
},
"original": {
@@ -151,11 +151,11 @@
]
},
"locked": {
"lastModified": 1693817438,
"narHash": "sha256-fg3+n4Ky1gCzDtPm0MomMTFw0YkH05Y8ojy5t7bkfHg=",
"lastModified": 1694528738,
"narHash": "sha256-aWMEjib5oTqEzF9f3WXffC1cwICo6v/4dYKjwNktV8k=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "b8d3a059f5487d6767d07c3716386753e3132d9f",
"rev": "7a49c388d7a6b63bb551b1ddedfa4efab8f400d8",
"type": "github"
},
"original": {

2
flake.nix
View File

@@ -31,8 +31,6 @@
./formatter.nix
./templates/flake-module.nix
./flakeModules/clan-config.nix
./pkgs/flake-module.nix
./lib/flake-module.nix

42
flakeModules/clan-config.nix
View File

@@ -1,42 +0,0 @@
{ ... } @ clanCore: {
flake.flakeModules.clan-config = { self, inputs, ... }:
let
# take the default nixos configuration
options = self.nixosConfigurations.default.options;
# this is actually system independent as it uses toFile
docs = inputs.nixpkgs.legacyPackages.x86_64-linux.nixosOptionsDoc {
inherit options;
};
optionsJSONFile = docs.optionsJSON.options;
warnIfNoDefaultConfig = return:
if ! self ? nixosConfigurations.default
then
builtins.trace
"WARNING: .#nixosConfigurations.default could not be found. Please define it."
return
else return;
in
{
flake.clanOptions = warnIfNoDefaultConfig optionsJSONFile;
flake.clanSettings = self + /clan-settings.json;
perSystem = { pkgs, ... }: {
devShells.clan-config = pkgs.mkShell {
packages = [
clanCore.config.flake.packages.${pkgs.system}.clan-cli
];
shellHook = ''
export CLAN_OPTIONS_FILE=$(nix eval --raw .#clanOptions)
export XDG_DATA_DIRS="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="${clanCore.config.flake.packages.${pkgs.system}.clan-cli}/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
'';
};
};
};
}

8
lib/jsonschema/default.nix
View File

@@ -7,6 +7,7 @@ let
float = "number";
int = "integer";
str = "string";
path = "string"; # TODO add prober path checks
};
# remove _module attribute from options
@@ -103,6 +104,13 @@ rec {
type = "string";
}
# parse string
else if option.type.name == "path"
# return jsonschema property definition for path
then default // description // {
type = "string";
}
# parse enum
else if option.type.name == "enum"
# return jsonschema property definition for enum

2
nixosModules/clanCore/flake-module.nix
View File

@@ -3,9 +3,11 @@
imports = [
./secrets
./zerotier.nix
./networking.nix
inputs.sops-nix.nixosModules.sops
# just some example options. Can be removed later
./bloatware
./vm.nix
];
options.clanSchema = lib.mkOption {
type = lib.types.attrs;

15
nixosModules/clanCore/networking.nix Normal file
View File

@@ -0,0 +1,15 @@
{ config, lib, ... }:
{
options.clan.networking = {
deploymentAddress = lib.mkOption {
description = ''
The target SSH node for deployment.
By default, the node's attribute name will be used.
If set to null, only local deployment will be supported.
'';
type = lib.types.nullOr lib.types.str;
default = "root@${config.networking.hostName}";
};
};
}

24
nixosModules/clanCore/secrets/default.nix
View File

@@ -1,6 +1,16 @@
{ config, lib, ... }:
{ config, lib, pkgs, ... }:
{
options.clanCore.secretStore = lib.mkOption {
type = lib.types.enum [ "sops" "password-store" "custom" ];
default = "sops";
description = ''
method to store secrets
custom can be used to define a custom secret store.
one would have to define system.clan.generateSecrets and system.clan.uploadSecrets
'';
};
options.clanCore.secrets = lib.mkOption {
default = { };
type = lib.types.attrsOf
(lib.types.submodule (secret: {
options = {
@@ -49,10 +59,11 @@
description = ''
path to a fact which is generated by the generator
'';
default = "${config.clanCore.clanDir}/machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}";
default = "machines/${config.clanCore.machineName}/facts/${fact.config._module.args.name}";
};
value = lib.mkOption {
default = builtins.readFile fact.config.path;
defaultText = lib.literalExpression "\${config.clanCore.clanDir}/\${fact.config.path}";
default = builtins.readFile "${config.clanCore.clanDir}/${fact.config.path}";
};
};
}));
@@ -60,7 +71,12 @@
};
}));
};
config.system.build.generateUploadSecrets = pkgs.writeScript "generate_upload_secrets" ''
${config.system.clan.generateSecrets}
${config.system.clan.uploadSecrets}
'';
imports = [
./sops.nix # for now we have only one implementation, thats why we import it here and not in clanModules
./sops.nix
./password-store.nix
];
}

118
nixosModules/clanCore/secrets/password-store.nix Normal file
View File

@@ -0,0 +1,118 @@
{ config, lib, pkgs, ... }:
let
passwordstoreDir = "\${PASSWORD_STORE_DIR:-$HOME/.password-store}";
in
{
options.clan.password-store.targetDirectory = lib.mkOption {
type = lib.types.path;
default = "/etc/secrets";
description = ''
The directory where the password store is uploaded to.
'';
};
config = lib.mkIf (config.clanCore.secretStore == "password-store") {
system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
#!/bin/sh
set -efu
test -d "$CLAN_DIR"
PATH=${lib.makeBinPath [
pkgs.pass
]}:$PATH
# TODO maybe initialize password store if it doesn't exist yet
${lib.foldlAttrs (acc: n: v: ''
${acc}
# ${n}
# if any of the secrets are missing, we regenerate all connected facts/secrets
(if ! ${lib.concatMapStringsSep " && " (x: "pass show machines/${config.clanCore.machineName}/${x.name} >/dev/null") (lib.attrValues v.secrets)}; then
facts=$(mktemp -d)
trap "rm -rf $facts" EXIT
secrets=$(mktemp -d)
trap "rm -rf $secrets" EXIT
${v.generator}
${lib.concatMapStrings (fact: ''
mkdir -p "$(dirname ${fact.path})"
cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
'') (lib.attrValues v.facts)}
${lib.concatMapStrings (secret: ''
cat "$secrets"/${secret.name} | pass insert -m machines/${config.clanCore.machineName}/${secret.name}
'') (lib.attrValues v.secrets)}
fi)
'') "" config.clanCore.secrets}
'';
system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
#!/bin/sh
set -efu
target=$1
umask 0077
PATH=${lib.makeBinPath [
pkgs.pass
pkgs.git
pkgs.findutils
pkgs.rsync
]}:$PATH:${lib.getBin pkgs.openssh}
if test -e ${passwordstoreDir}/.git; then
local_pass_info=$(
git -C ${passwordstoreDir} log -1 --format=%H machines/${config.clanCore.machineName}
# we append a hash for every symlink, otherwise we would miss updates on
# files where the symlink points to
find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type l \
-exec realpath {} + |
sort |
xargs -r -n 1 git -C ${passwordstoreDir} log -1 --format=%H
)
remote_pass_info=$(ssh "$target" -- ${lib.escapeShellArg ''
cat ${config.clan.password-store.targetDirectory}/.pass_info || :
''})
if test "$local_pass_info" = "$remote_pass_info"; then
echo secrets already match
exit 0
fi
fi
tmp_dir=$(mktemp -dt populate-pass.XXXXXXXX)
trap cleanup EXIT
cleanup() {
rm -fR "$tmp_dir"
}
find ${passwordstoreDir}/machines/${config.clanCore.machineName} -type f -follow ! -name .gpg-id |
while read -r gpg_path; do
rel_name=''${gpg_path#${passwordstoreDir}}
rel_name=''${rel_name%.gpg}
pass_date=$(
if test -e ${passwordstoreDir}/.git; then
git -C ${passwordstoreDir} log -1 --format=%aI "$gpg_path"
fi
)
pass_name=$rel_name
tmp_path=$tmp_dir/$(basename $rel_name)
mkdir -p "$(dirname "$tmp_path")"
pass show "$pass_name" > "$tmp_path"
if [ -n "$pass_date" ]; then
touch -d "$pass_date" "$tmp_path"
fi
done
if test -n "''${local_pass_info-}"; then
echo "$local_pass_info" > "$tmp_dir"/.pass_info
fi
rsync --mkpath --delete -a "$tmp_dir"/ "$target":${config.clan.password-store.targetDirectory}/
'';
};
}

10
nixosModules/clanCore/secrets/sops.nix
View File

@@ -21,11 +21,12 @@ let
secrets = filterDir containsMachineOrGroups secretsDir;
in
{
config = {
config = lib.mkIf (config.clanCore.secretStore == "sops") {
system.clan.generateSecrets = pkgs.writeScript "generate-secrets" ''
#!/bin/sh
set -efu
set -x # remove for prod
test -d "$CLAN_DIR"
PATH=$PATH:${lib.makeBinPath [
config.clanCore.clanPkgs.clan-cli
@@ -55,7 +56,7 @@ in
${lib.concatMapStrings (fact: ''
mkdir -p "$(dirname ${fact.path})"
cp "$facts"/${fact.name} ${fact.path}
cp "$facts"/${fact.name} "$CLAN_DIR"/${fact.path}
'') (lib.attrValues v.facts)}
${lib.concatMapStrings (secret: ''
@@ -64,6 +65,9 @@ in
fi)
'') "" config.clanCore.secrets}
'';
system.clan.uploadSecrets = pkgs.writeScript "upload-secrets" ''
echo upload is not needed for sops secret store, since the secrets are part of the flake
'';
sops.secrets = builtins.mapAttrs
(name: _: {
sopsFile = config.clanCore.clanDir + "/sops/secrets/${name}/secret";

8
nixosModules/clanCore/vm.nix Normal file
View File

@@ -0,0 +1,8 @@
{ config, options, lib, ... }: {
system.clan.vm.config = {
enabled = options.virtualisation ? cores;
} // (lib.optionalAttrs (options.virtualisation ? cores) {
inherit (config.virtualisation) cores graphics;
memory_size = config.virtualisation.memorySize;
});
}

13
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -1,10 +1,9 @@
import argparse
import os
import sys
from types import ModuleType
from typing import Optional
from . import admin, config, machines, secrets, webui, zerotier
from . import config, create, machines, secrets, webui, zerotier
from .errors import ClanError
from .ssh import cli as ssh_cli
@@ -19,13 +18,11 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
subparsers = parser.add_subparsers()
parser_admin = subparsers.add_parser("admin", help="administrate a clan")
admin.register_parser(parser_admin)
parser_create = subparsers.add_parser("create", help="create a clan flake")
create.register_parser(parser_create)
# DISABLED: this currently crashes if a flake does not define .#clanOptions
if os.environ.get("CLAN_OPTIONS_FILE") is not None:
parser_config = subparsers.add_parser("config", help="set nixos configuration")
config.register_parser(parser_config)
parser_config = subparsers.add_parser("config", help="set nixos configuration")
config.register_parser(parser_config)
parser_ssh = subparsers.add_parser("ssh", help="ssh to a remote machine")
ssh_cli.register_parser(parser_ssh)

37
pkgs/clan-cli/clan_cli/admin.py
View File

@@ -1,37 +0,0 @@
# !/usr/bin/env python3
import argparse
import os
import subprocess
def create(args: argparse.Namespace) -> None:
os.makedirs(args.folder, exist_ok=True)
# TODO create clan template in flake
subprocess.run(
[
"nix",
"flake",
"init",
"-t",
"git+https://git.clan.lol/clan/clan-core#new-clan",
]
)
# takes a (sub)parser and configures it
def register_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"-f",
"--folder",
help="the folder where the clan is defined, default to the current folder",
default=os.environ["PWD"],
)
subparser = parser.add_subparsers(
title="command",
description="the command to run",
help="the command to run",
required=True,
)
parser_create = subparser.add_parser("create", help="create a new clan")
parser_create.set_defaults(func=create)

10
pkgs/clan-cli/clan_cli/config/__init__.py
View File

@@ -9,6 +9,7 @@ from typing import Any, Optional, Type
from clan_cli.dirs import get_clan_flake_toplevel
from clan_cli.errors import ClanError
from clan_cli.machines.folders import machine_settings_file
from clan_cli.nix import nix_eval
script_dir = Path(__file__).parent
@@ -100,7 +101,6 @@ def options_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict
proc = subprocess.run(
nix_eval(
flags=[
"--json",
"--show-trace",
"--impure",
"--expr",
@@ -138,7 +138,6 @@ def read_machine_option_value(machine_name: str, option: str) -> str:
proc = subprocess.run(
nix_eval(
flags=[
"--json",
"--show-trace",
"--extra-experimental-features",
"nix-command flakes",
@@ -168,7 +167,6 @@ def get_or_set_option(args: argparse.Namespace) -> None:
print(read_machine_option_value(args.machine, args.option))
else:
# load options
print(args.options_file)
if args.options_file is None:
options = options_for_machine(machine_name=args.machine)
else:
@@ -176,8 +174,8 @@ def get_or_set_option(args: argparse.Namespace) -> None:
options = json.load(f)
# compute settings json file location
if args.settings_file is None:
flake = get_clan_flake_toplevel()
settings_file = flake / "machines" / f"{args.machine}.json"
get_clan_flake_toplevel()
settings_file = machine_settings_file(args.machine)
else:
settings_file = args.settings_file
# set the option with the given value
@@ -288,7 +286,7 @@ def register_parser(
# add single positional argument for the option (e.g. "foo.bar")
parser.add_argument(
"option",
help="Option to configure",
help="Option to read or set",
type=str,
)

5
pkgs/clan-cli/clan_cli/config/machine.py
View File

@@ -12,7 +12,7 @@ from clan_cli.nix import nix_eval
def config_for_machine(machine_name: str) -> dict:
# read the config from a json file located at {flake}/machines/{machine_name}.json
# read the config from a json file located at {flake}/machines/{machine_name}/settings.json
if not machine_folder(machine_name).exists():
raise HTTPException(
status_code=404,
@@ -26,7 +26,7 @@ def config_for_machine(machine_name: str) -> dict:
def set_config_for_machine(machine_name: str, config: dict) -> None:
# write the config to a json file located at {flake}/machines/{machine_name}.json
# write the config to a json file located at {flake}/machines/{machine_name}/settings.json
if not machine_folder(machine_name).exists():
raise HTTPException(
status_code=404,
@@ -45,7 +45,6 @@ def schema_for_machine(machine_name: str, flake: Optional[Path] = None) -> dict:
proc = subprocess.run(
nix_eval(
flags=[
"--json",
"--impure",
"--show-trace",
"--extra-experimental-features",

11
pkgs/clan-cli/clan_cli/config/parsing.py
View File

@@ -3,7 +3,8 @@ import subprocess
from pathlib import Path
from typing import Any, Optional, Type, Union
from clan_cli.errors import ClanError
from ..errors import ClanError
from ..nix import nix_eval
script_dir = Path(__file__).parent
@@ -30,11 +31,9 @@ def schema_from_module_file(
slib.parseModule {absolute_path}
"""
# run the nix expression and parse the output as json
return json.loads(
subprocess.check_output(
["nix", "eval", "--impure", "--json", "--expr", nix_expr]
)
)
cmd = nix_eval(["--expr", nix_expr])
proc = subprocess.run(cmd, stdout=subprocess.PIPE, check=True)
return json.loads(proc.stdout)
def subtype_from_schema(schema: dict[str, Any]) -> Type:

25
pkgs/clan-cli/clan_cli/create.py Normal file
View File

@@ -0,0 +1,25 @@
# !/usr/bin/env python3
import argparse
import subprocess
from .nix import nix_command
def create(args: argparse.Namespace) -> None:
# TODO create clan template in flake
subprocess.run(
nix_command(
[
"flake",
"init",
"-t",
"git+https://git.clan.lol/clan/clan-core#new-clan",
]
),
check=True,
)
# takes a (sub)parser and configures it
def register_parser(parser: argparse.ArgumentParser) -> None:
parser.set_defaults(func=create)

67
pkgs/clan-cli/clan_cli/machines/update.py
View File

@@ -1,7 +1,13 @@
import argparse
import json
import os
import subprocess
from typing import Optional
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_command, nix_eval
from ..secrets.generate import generate_secrets
from ..secrets.upload import upload_secrets
from ..ssh import Host, HostGroup, HostKeyCheck
@@ -13,11 +19,13 @@ def deploy_nixos(hosts: HostGroup) -> None:
def deploy(h: Host) -> None:
target = f"{h.user or 'root'}@{h.host}"
ssh_arg = f"-p {h.port}" if h.port else ""
env = os.environ.copy()
env["NIX_SSHOPTS"] = ssh_arg
res = h.run_local(
["nix", "flake", "archive", "--to", f"ssh://{target}", "--json"],
nix_command(["flake", "archive", "--to", f"ssh://{target}", "--json"]),
check=True,
stdout=subprocess.PIPE,
extra_env=dict(NIX_SSHOPTS=ssh_arg),
extra_env=env,
)
data = json.loads(res.stdout)
path = data["path"]
@@ -29,6 +37,9 @@ def deploy_nixos(hosts: HostGroup) -> None:
ssh_arg += " -i " + h.key if h.key else ""
generate_secrets(h.host)
upload_secrets(h.host)
flake_attr = h.meta.get("flake_attr", "")
if flake_attr:
flake_attr = "#" + flake_attr
@@ -67,20 +78,46 @@ def deploy_nixos(hosts: HostGroup) -> None:
# FIXME: we want some kind of inventory here.
def update(args: argparse.Namespace) -> None:
meta = {}
if args.flake_uri:
meta["flake_uri"] = args.flake_uri
if args.flake_attr:
meta["flake_attr"] = args.flake_attr
deploy_nixos(HostGroup([Host(args.host, user=args.user, meta=meta)]))
clan_dir = get_clan_flake_toplevel().as_posix()
host = json.loads(
subprocess.run(
nix_eval(
[
f'{clan_dir}#nixosConfigurations."{args.machine}".config.clan.networking.deploymentAddress'
]
),
stdout=subprocess.PIPE,
check=True,
text=True,
).stdout
)
parts = host.split("@")
user: Optional[str] = None
if len(parts) > 1:
user = parts[0]
hostname = parts[1]
else:
hostname = parts[0]
maybe_port = hostname.split(":")
port = None
if len(maybe_port) > 1:
hostname = maybe_port[0]
port = int(maybe_port[1])
print(f"deploying {host}")
deploy_nixos(
HostGroup(
[
Host(
host=hostname,
port=port,
user=user,
meta=dict(flake_attr=args.machine),
)
]
)
)
def register_update_parser(parser: argparse.ArgumentParser) -> None:
# TODO pass all args we don't parse into ssh_args, currently it fails if arg starts with -
parser.add_argument("--flake-uri", type=str, default=".#", help="nix flake uri")
parser.add_argument(
"--flake-attr", type=str, help="nixos configuration in the flake"
)
parser.add_argument("--user", type=str, default="root")
parser.add_argument("host", type=str)
parser.add_argument("machine", type=str)
parser.set_defaults(func=update)

121
pkgs/clan-cli/clan_cli/nix.py
View File

@@ -1,69 +1,54 @@
import json
import os
import tempfile
from pathlib import Path
from .dirs import get_clan_flake_toplevel, nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
from .dirs import nixpkgs_flake, nixpkgs_source, unfree_nixpkgs
def nix_build_machine(
machine: str, attr: list[str], flake_url: Path | None = None
def nix_command(flags: list[str]) -> list[str]:
return ["nix", "--experimental-features", "nix-command flakes"] + flags
def nix_build(
flags: list[str],
) -> list[str]:
if flake_url is None:
flake_url = get_clan_flake_toplevel()
payload = json.dumps(
dict(
clan_flake=flake_url,
machine=machine,
attr=attr,
return (
nix_command(
[
"build",
"--no-link",
"--print-out-paths",
"--extra-experimental-features",
"nix-command flakes",
]
)
+ flags
)
escaped_payload = json.dumps(payload)
return [
"nix",
"build",
"--impure",
"--print-out-paths",
"--expr",
f"let args = builtins.fromJSON {escaped_payload}; in "
"""
let
flake = builtins.getFlake args.clan_flake;
config = flake.nixosConfigurations.${args.machine}.extendModules {
modules = [{
clanCore.clanDir = args.clan_flake;
}];
};
in
flake.inputs.nixpkgs.lib.getAttrFromPath args.attr config
""",
]
def nix_eval(flags: list[str]) -> list[str]:
default_flags = nix_command(
[
"eval",
"--show-trace",
"--json",
]
)
if os.environ.get("IN_NIX_SANDBOX"):
with tempfile.TemporaryDirectory() as nix_store:
return [
"nix",
"eval",
"--show-trace",
"--extra-experimental-features",
"nix-command flakes",
"--override-input",
"nixpkgs",
str(nixpkgs_source()),
# --store is required to prevent this error:
# error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted
"--store",
nix_store,
] + flags
return [
"nix",
"eval",
"--show-trace",
"--extra-experimental-features",
"nix-command flakes",
] + flags
return (
default_flags
+ [
"--override-input",
"nixpkgs",
str(nixpkgs_source()),
# --store is required to prevent this error:
# error: cannot unlink '/nix/store/6xg259477c90a229xwmb53pdfkn6ig3g-default-builder.sh': Operation not permitted
"--store",
nix_store,
]
+ flags
)
return default_flags + flags
def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
@@ -73,14 +58,13 @@ def nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
return cmd
wrapped_packages = [f"nixpkgs#{p}" for p in packages]
return (
[
"nix",
"shell",
"--extra-experimental-features",
"nix-command flakes",
"--inputs-from",
f"{str(nixpkgs_flake())}",
]
nix_command(
[
"shell",
"--inputs-from",
f"{str(nixpkgs_flake())}",
]
)
+ wrapped_packages
+ ["-c"]
+ cmd
@@ -91,14 +75,13 @@ def unfree_nix_shell(packages: list[str], cmd: list[str]) -> list[str]:
if os.environ.get("IN_NIX_SANDBOX"):
return cmd
return (
[
"nix",
"shell",
"--extra-experimental-features",
"nix-command flakes",
"-f",
str(unfree_nixpkgs()),
]
nix_command(
[
"shell",
"-f",
str(unfree_nixpkgs()),
]
)
+ packages
+ ["-c"]
+ cmd

4
pkgs/clan-cli/clan_cli/secrets/__init__.py
View File

@@ -7,6 +7,7 @@ from .import_sops import register_import_sops_parser
from .key import register_key_parser
from .machines import register_machines_parser
from .secrets import register_secrets_parser
from .upload import register_upload_parser
from .users import register_users_parser
@@ -36,6 +37,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
)
register_generate_parser(parser_generate)
parser_upload = subparser.add_parser("upload", help="upload secrets for machines")
register_upload_parser(parser_upload)
parser_key = subparser.add_parser("key", help="create and show age keys")
register_key_parser(parser_key)

31
pkgs/clan-cli/clan_cli/secrets/generate.py
View File

@@ -1,24 +1,25 @@
import argparse
import os
import subprocess
import sys
from clan_cli.errors import ClanError
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_build
def generate_secrets(machine: str) -> None:
clan_dir = get_clan_flake_toplevel().as_posix().strip()
env = os.environ.copy()
env["CLAN_DIR"] = clan_dir
def get_secret_script(machine: str) -> None:
proc = subprocess.run(
[
"nix",
"build",
"--impure",
"--print-out-paths",
"--expr",
"let f = builtins.getFlake (toString ./.); in "
f"(f.nixosConfigurations.{machine}.extendModules "
"{ modules = [{ clanCore.clanDir = toString ./.; }]; })"
".config.system.clan.generateSecrets",
],
check=True,
nix_build(
[
f'path:{clan_dir}#nixosConfigurations."{machine}".config.system.clan.generateSecrets'
]
),
capture_output=True,
text=True,
)
@@ -30,7 +31,7 @@ def get_secret_script(machine: str) -> None:
print(secret_generator_script)
secret_generator = subprocess.run(
[secret_generator_script],
check=True,
env=env,
)
if secret_generator.returncode != 0:
@@ -40,7 +41,7 @@ def get_secret_script(machine: str) -> None:
def generate_command(args: argparse.Namespace) -> None:
get_secret_script(args.machine)
generate_secrets(args.machine)
def register_generate_parser(parser: argparse.ArgumentParser) -> None:

60
pkgs/clan-cli/clan_cli/secrets/upload.py Normal file
View File

@@ -0,0 +1,60 @@
import argparse
import json
import subprocess
from clan_cli.errors import ClanError
from ..dirs import get_clan_flake_toplevel
from ..nix import nix_build, nix_eval
def upload_secrets(machine: str) -> None:
clan_dir = get_clan_flake_toplevel().as_posix()
proc = subprocess.run(
nix_build(
[
f'{clan_dir}#nixosConfigurations."{machine}".config.system.clan.uploadSecrets'
]
),
stdout=subprocess.PIPE,
text=True,
check=True,
)
host = json.loads(
subprocess.run(
nix_eval(
[
f'{clan_dir}#nixosConfigurations."{machine}".config.clan.networking.deploymentAddress'
]
),
stdout=subprocess.PIPE,
text=True,
check=True,
).stdout
)
secret_upload_script = proc.stdout.strip()
secret_upload = subprocess.run(
[
secret_upload_script,
host,
],
)
if secret_upload.returncode != 0:
raise ClanError("failed to upload secrets")
else:
print("successfully uploaded secrets")
def upload_command(args: argparse.Namespace) -> None:
upload_secrets(args.machine)
def register_upload_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"machine",
help="The machine to upload secrets to",
)
parser.set_defaults(func=upload_command)

16
pkgs/clan-cli/clan_cli/webui/app.py
View File

@@ -1,16 +1,30 @@
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.routing import APIRoute
from fastapi.staticfiles import StaticFiles
from .assets import asset_path
from .routers import health, machines, root
from .routers import health, machines, root, vms
origins = [
"http://localhost:3000",
]
def setup_app() -> FastAPI:
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
app.include_router(health.router)
app.include_router(machines.router)
app.include_router(root.router)
app.include_router(vms.router)
app.add_exception_handler(vms.NixBuildException, vms.nix_build_exception_handler)
app.mount("/static", StaticFiles(directory=asset_path()), name="static")

113
pkgs/clan-cli/clan_cli/webui/routers/vms.py Normal file
View File

@@ -0,0 +1,113 @@
import asyncio
import json
import shlex
from typing import Annotated, AsyncIterator
from fastapi import APIRouter, Body, HTTPException, Request, status
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse, StreamingResponse
from ...nix import nix_build, nix_eval
from ..schemas import VmConfig, VmInspectResponse
router = APIRouter()
class NixBuildException(HTTPException):
def __init__(self, msg: str, loc: list = ["body", "flake_attr"]):
detail = [
{
"loc": loc,
"msg": msg,
"type": "value_error",
}
]
super().__init__(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail
)
def nix_build_exception_handler(
request: Request, exc: NixBuildException
) -> JSONResponse:
return JSONResponse(
status_code=exc.status_code,
content=jsonable_encoder(dict(detail=exc.detail)),
)
def nix_inspect_vm(machine: str, flake_url: str) -> list[str]:
return nix_eval(
[
f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.clan.vm.config"
]
)
def nix_build_vm(machine: str, flake_url: str) -> list[str]:
return nix_build(
[
f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.build.vm"
]
)
@router.post("/api/vms/inspect")
async def inspect_vm(
flake_url: Annotated[str, Body()], flake_attr: Annotated[str, Body()]
) -> VmInspectResponse:
cmd = nix_inspect_vm(flake_attr, flake_url=flake_url)
proc = await asyncio.create_subprocess_exec(
cmd[0],
*cmd[1:],
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await proc.communicate()
if proc.returncode != 0:
raise NixBuildException(
f"""
Failed to evaluate vm from '{flake_url}#{flake_attr}'.
command: {shlex.join(cmd)}
exit code: {proc.returncode}
command output:
{stderr.decode("utf-8")}
"""
)
data = json.loads(stdout)
return VmInspectResponse(
config=VmConfig(flake_url=flake_url, flake_attr=flake_attr, **data)
)
async def vm_build(vm: VmConfig) -> AsyncIterator[str]:
cmd = nix_build_vm(vm.flake_attr, flake_url=vm.flake_url)
proc = await asyncio.create_subprocess_exec(
cmd[0],
*cmd[1:],
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
assert proc.stdout is not None and proc.stderr is not None
async for line in proc.stdout:
yield line.decode("utf-8", "ignore")
stderr = ""
async for line in proc.stderr:
stderr += line.decode("utf-8", "ignore")
res = await proc.wait()
if res != 0:
raise NixBuildException(
f"""
Failed to build vm from '{vm.flake_url}#{vm.flake_attr}'.
command: {shlex.join(cmd)}
exit code: {res}
command output:
{stderr}
"""
)
@router.post("/api/vms/create")
async def create_vm(vm: Annotated[VmConfig, Body()]) -> StreamingResponse:
return StreamingResponse(vm_build(vm))

13
pkgs/clan-cli/clan_cli/webui/schemas.py
View File

@@ -32,3 +32,16 @@ class ConfigResponse(BaseModel):
class SchemaResponse(BaseModel):
schema_: dict = Field(alias="schema")
class VmConfig(BaseModel):
flake_url: str
flake_attr: str
cores: int
memory_size: int
graphics: bool
class VmInspectResponse(BaseModel):
config: VmConfig

16
pkgs/clan-cli/clan_cli/webui/server.py
View File

@@ -62,10 +62,18 @@ def start_server(args: argparse.Namespace) -> None:
if ":" in host:
host = f"[{host}]"
headers = [
(
"Access-Control-Allow-Origin",
f"http://{host}:{args.dev_port}",
)
# (
# "Access-Control-Allow-Origin",
# f"http://{host}:{args.dev_port}",
# ),
# (
# "Access-Control-Allow-Methods",
# "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"
# ),
# (
# "Allow",
# "DELETE, GET, HEAD, OPTIONS, PATCH, POST, PUT"
# )
]
else:
open_url = f"http://[{args.host}]:{args.port}"

15
pkgs/clan-cli/default.nix
View File

@@ -22,9 +22,6 @@
, ui-assets
}:
let
# This provides dummy options for testing clan config and prevents it from
# evaluating the flake .#
CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json;
dependencies = [
argcomplete # optional dependency: if not enabled, shell completion will not work
@@ -54,9 +51,9 @@ let
'';
nixpkgs = runCommand "nixpkgs" { nativeBuildInputs = [ pkgs.nix ]; } ''
mkdir $out
mkdir -p $out/unfree
cat > $out/unfree/default.nix <<EOF
import "${pkgs.path}" { config = { allowUnfree = true; overlays = []; }; }
mkdir -p $out/unfree
cat > $out/unfree/default.nix <<EOF
import "${pkgs.path}" { config = { allowUnfree = true; overlays = []; }; }
EOF
cat > $out/flake.nix << EOF
{
@@ -81,8 +78,6 @@ python3.pkgs.buildPythonPackage {
src = source;
format = "pyproject";
inherit CLAN_OPTIONS_FILE;
nativeBuildInputs = [
setuptools
installShellFiles
@@ -93,12 +88,12 @@ python3.pkgs.buildPythonPackage {
{
nativeBuildInputs = [ age zerotierone bubblewrap sops nix openssh rsync stdenv.cc ];
} ''
export CLAN_OPTIONS_FILE="${CLAN_OPTIONS_FILE}"
cp -r ${source} ./src
chmod +w -R ./src
cd ./src
NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1 ${checkPython}/bin/python -m pytest -s ./tests
export NIX_STATE_DIR=$TMPDIR/nix IN_NIX_SANDBOX=1
${checkPython}/bin/python -m pytest -m "not impure" -s ./tests
touch $out
'';
passthru.clan-openapi = runCommand "clan-openapi" { } ''

1
pkgs/clan-cli/pyproject.toml
View File

@@ -18,6 +18,7 @@ clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"]
faulthandler_timeout = 30
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5"
norecursedirs = "tests/helpers"
markers = [ "impure" ]
[tool.mypy]
python_version = "3.10"

6
pkgs/clan-cli/shell.nix
View File

@@ -6,6 +6,7 @@ let
++ clan-cli.devDependencies
++ [
ps.pip
ps.ipdb
]
);
checkScript = writeScriptBin "check" ''
@@ -19,10 +20,9 @@ mkShell {
pythonWithDeps
];
# sets up an editable install and add enty points to $PATH
# This provides dummy options for testing clan config and prevents it from
# evaluating the flake .#
CLAN_OPTIONS_FILE = ./clan_cli/config/jsonschema/options.json;
PYTHONPATH = "${pythonWithDeps}/${pythonWithDeps.sitePackages}";
PYTHONBREAKPOINT = "ipdb.set_trace";
shellHook = ''
tmp_path=$(realpath ./.direnv)

23
pkgs/clan-cli/tests/clan_flake.py
View File

@@ -1,23 +0,0 @@
from pathlib import Path
from typing import Iterator
import pytest
@pytest.fixture
def clan_flake(temporary_dir: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
flake = temporary_dir / "clan-flake"
flake.mkdir()
(flake / ".clan-flake").touch()
(flake / "flake.nix").write_text(
"""
{
description = "A flake for testing clan";
inputs = {};
outputs = { self }: {};
}
"""
)
monkeypatch.chdir(flake)
monkeypatch.setenv("HOME", str(temporary_dir))
yield flake

37
pkgs/clan-cli/tests/conftest.py
View File

@@ -1,51 +1,16 @@
import os
import sys
import tempfile
from pathlib import Path
from typing import Generator
import pytest
from clan_cli.dirs import nixpkgs_source
sys.path.append(os.path.join(os.path.dirname(__file__), "helpers"))
pytest_plugins = [
"api",
"temporary_dir",
"clan_flake",
"root",
"age_keys",
"sshd",
"command",
"ports",
"host_group",
"test_flake",
]
@pytest.fixture(scope="module")
def monkeymodule() -> Generator[pytest.MonkeyPatch, None, None]:
with pytest.MonkeyPatch.context() as mp:
yield mp
@pytest.fixture(scope="module")
def machine_flake(monkeymodule: pytest.MonkeyPatch) -> Generator[Path, None, None]:
template = Path(__file__).parent / "machine_flake"
# copy the template to a new temporary location
with tempfile.TemporaryDirectory() as tmpdir_:
flake = Path(tmpdir_)
for path in template.glob("**/*"):
if path.is_dir():
(flake / path.relative_to(template)).mkdir()
else:
(flake / path.relative_to(template)).write_text(path.read_text())
# in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
# provided by get_clan_flake_toplevel
flake_nix = flake / "flake.nix"
flake_nix.write_text(
flake_nix.read_text().replace("__NIXPKGS__", str(nixpkgs_source()))
)
# check that an empty config is returned if no json file exists
monkeymodule.chdir(flake)
yield flake

11
pkgs/clan-cli/tests/root.py
View File

@@ -4,12 +4,13 @@ import pytest
TEST_ROOT = Path(__file__).parent.resolve()
PROJECT_ROOT = TEST_ROOT.parent
CLAN_CORE = PROJECT_ROOT.parent.parent
@pytest.fixture(scope="session")
def project_root() -> Path:
"""
Root directory of the tests
Root directory the clan-cli
"""
return PROJECT_ROOT
@@ -20,3 +21,11 @@ def test_root() -> Path:
Root directory of the tests
"""
return TEST_ROOT
@pytest.fixture(scope="session")
def clan_core() -> Path:
"""
Directory of the clan-core flake
"""
return CLAN_CORE

14
pkgs/clan-cli/tests/test_admin_cli.py
View File

@@ -1,14 +0,0 @@
from typing import Union
import pytest_subprocess.fake_process
from cli import Cli
from pytest_subprocess import utils
# using fp fixture from pytest-subprocess
def test_create(fp: pytest_subprocess.fake_process.FakeProcess) -> None:
cmd: list[Union[str, utils.Any]] = ["nix", "flake", "init", "-t", fp.any()]
fp.register(cmd)
cli = Cli()
cli.run(["admin", "--folder", "./my-clan", "create"])
assert fp.call_count(cmd) == 1

12
pkgs/clan-cli/tests/test_clan_template.py Normal file
View File

@@ -0,0 +1,12 @@
from pathlib import Path
import pytest
from cli import Cli
@pytest.mark.impure
def test_template(monkeypatch: pytest.MonkeyPatch, temporary_dir: Path) -> None:
monkeypatch.chdir(temporary_dir)
cli = Cli()
cli.run(["create"])
assert (temporary_dir / ".clan-flake").exists()

2
pkgs/clan-cli/tests/test_config.py
View File

@@ -50,7 +50,7 @@ def test_set_some_option(
def test_configure_machine(
machine_flake: Path,
test_flake: Path,
temporary_dir: Path,
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,

53
pkgs/clan-cli/tests/test_flake.py Normal file
View File

@@ -0,0 +1,53 @@
import fileinput
import shutil
import tempfile
from pathlib import Path
from typing import Iterator
import pytest
from root import CLAN_CORE
from clan_cli.dirs import nixpkgs_source
@pytest.fixture(scope="module")
def monkeymodule() -> Iterator[pytest.MonkeyPatch]:
with pytest.MonkeyPatch.context() as mp:
yield mp
def create_flake(
monkeymodule: pytest.MonkeyPatch, name: str, clan_core_flake: Path | None = None
) -> Iterator[Path]:
template = Path(__file__).parent / name
# copy the template to a new temporary location
with tempfile.TemporaryDirectory() as tmpdir_:
home = Path(tmpdir_)
flake = home / name
shutil.copytree(template, flake)
# in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
# provided by get_test_flake_toplevel
flake_nix = flake / "flake.nix"
for line in fileinput.input(flake_nix, inplace=True):
line = line.replace("__NIXPKGS__", str(nixpkgs_source()))
if clan_core_flake:
line = line.replace("__CLAN_CORE__", str(clan_core_flake))
print(line)
# check that an empty config is returned if no json file exists
monkeymodule.chdir(flake)
monkeymodule.setenv("HOME", str(home))
yield flake
@pytest.fixture(scope="module")
def test_flake(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]:
yield from create_flake(monkeymodule, "test_flake")
@pytest.fixture(scope="module")
def test_flake_with_core(monkeymodule: pytest.MonkeyPatch) -> Iterator[Path]:
if not (CLAN_CORE / "flake.nix").exists():
raise Exception(
"clan-core flake not found. This test requires the clan-core flake to be present"
)
yield from create_flake(monkeymodule, "test_flake_with_core", CLAN_CORE)

0
pkgs/clan-cli/tests/machine_flake/.clan-flake → pkgs/clan-cli/tests/test_flake/.clan-flake
View File

10
pkgs/clan-cli/tests/machine_flake/flake.nix → pkgs/clan-cli/tests/test_flake/flake.nix
View File

@@ -1,15 +1,13 @@
{
inputs = {
# this placeholder is replaced by the path to nixpkgs
nixpkgs.url = "__NIXPKGS__";
};
# this placeholder is replaced by the path to nixpkgs
inputs.nixpkgs.url = "__NIXPKGS__";
outputs = inputs: {
nixosConfigurations.machine1 = inputs.nixpkgs.lib.nixosSystem {
modules = [
./nixosModules/machine1.nix
(if builtins.pathExists ./machines/machine1.json
then builtins.fromJSON (builtins.readFile ./machines/machine1.json)
(if builtins.pathExists ./machines/machine1/settings.json
then builtins.fromJSON (builtins.readFile ./machines/machine1/settings.json)
else { })
{
nixpkgs.hostPlatform = "x86_64-linux";

0
pkgs/clan-cli/tests/machine_flake/nixosModules/machine1.nix → pkgs/clan-cli/tests/test_flake/nixosModules/machine1.nix
View File

0
pkgs/clan-cli/tests/test_flake_with_core/.clan-flake Normal file
View File

18
pkgs/clan-cli/tests/test_flake_with_core/flake.nix Normal file
View File

@@ -0,0 +1,18 @@
{
# Use this path to our repo root e.g. for UI test
# inputs.clan-core.url = "../../../../.";
# this placeholder is replaced by the path to nixpkgs
inputs.clan-core.url = "__CLAN_CORE__";
outputs = { self, clan-core }: {
nixosConfigurations = clan-core.lib.buildClan {
directory = self;
machines = {
vm1 = { modulesPath, ... }: {
imports = [ "${toString modulesPath}/virtualisation/qemu-vm.nix" ];
};
};
};
};
}

2
pkgs/clan-cli/tests/test_import_sops_cli.py
View File

@@ -10,7 +10,7 @@ if TYPE_CHECKING:
def test_import_sops(
test_root: Path,
clan_flake: Path,
test_flake: Path,
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,
age_keys: list["KeyPair"],

4
pkgs/clan-cli/tests/test_machines_api.py
View File

@@ -3,7 +3,7 @@ from pathlib import Path
from api import TestClient
def test_machines(api: TestClient, clan_flake: Path) -> None:
def test_machines(api: TestClient, test_flake: Path) -> None:
response = api.get("/api/machines")
assert response.status_code == 200
assert response.json() == {"machines": []}
@@ -21,7 +21,7 @@ def test_machines(api: TestClient, clan_flake: Path) -> None:
assert response.json() == {"machines": [{"name": "test", "status": "unknown"}]}
def test_configure_machine(api: TestClient, machine_flake: Path) -> None:
def test_configure_machine(api: TestClient, test_flake: Path) -> None:
# ensure error 404 if machine does not exist when accessing the config
response = api.get("/api/machines/machine1/config")
assert response.status_code == 404

2
pkgs/clan-cli/tests/test_machines_cli.py
View File

@@ -4,7 +4,7 @@ import pytest
from cli import Cli
def test_machine_subcommands(clan_flake: Path, capsys: pytest.CaptureFixture) -> None:
def test_machine_subcommands(test_flake: Path, capsys: pytest.CaptureFixture) -> None:
cli = Cli()
cli.run(["machines", "create", "machine1"])

4
pkgs/clan-cli/tests/test_machines_config.py
View File

@@ -3,6 +3,6 @@ from pathlib import Path
from clan_cli.config import machine
def test_schema_for_machine(machine_flake: Path) -> None:
schema = machine.schema_for_machine("machine1", machine_flake)
def test_schema_for_machine(test_flake: Path) -> None:
schema = machine.schema_for_machine("machine1", test_flake)
assert "properties" in schema

35
pkgs/clan-cli/tests/test_machines_update_cli.py
View File

@@ -1,35 +0,0 @@
import os
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
import pytest
from host_group import HostGroup
def test_update(
clan_flake: Path, host_group: HostGroup, monkeypatch: pytest.MonkeyPatch
) -> None:
assert len(host_group.hosts) == 1
host = host_group.hosts[0]
with TemporaryDirectory() as tmpdir:
host.meta["flake_uri"] = clan_flake
host.meta["flake_path"] = str(Path(tmpdir) / "rsync-target")
host.ssh_options["SendEnv"] = "REALPATH"
bin = Path(tmpdir).joinpath("bin")
bin.mkdir()
nixos_rebuild = bin.joinpath("nixos-rebuild")
bash = shutil.which("bash")
assert bash is not None
nixos_rebuild.write_text(
f"""#!{bash}
exit 0
"""
)
nixos_rebuild.chmod(0o755)
f"{tmpdir}/bin:{os.environ['PATH']}"
nix_state_dir = Path(tmpdir).joinpath("nix")
nix_state_dir.mkdir()
monkeypatch.setenv("REALPATH", str(nix_state_dir))

20
pkgs/clan-cli/tests/test_secrets_cli.py
View File

@@ -14,12 +14,12 @@ if TYPE_CHECKING:
def _test_identities(
what: str,
clan_flake: Path,
test_flake: Path,
capsys: pytest.CaptureFixture,
age_keys: list["KeyPair"],
) -> None:
cli = Cli()
sops_folder = clan_flake / "sops"
sops_folder = test_flake / "sops"
cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey])
assert (sops_folder / what / "foo" / "key.json").exists()
@@ -60,19 +60,19 @@ def _test_identities(
def test_users(
clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
_test_identities("users", clan_flake, capsys, age_keys)
_test_identities("users", test_flake, capsys, age_keys)
def test_machines(
clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
_test_identities("machines", clan_flake, capsys, age_keys)
_test_identities("machines", test_flake, capsys, age_keys)
def test_groups(
clan_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
cli = Cli()
capsys.readouterr() # empty the buffer
@@ -100,7 +100,7 @@ def test_groups(
cli.run(["secrets", "groups", "remove-user", "group1", "user1"])
cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"])
groups = os.listdir(clan_flake / "sops" / "groups")
groups = os.listdir(test_flake / "sops" / "groups")
assert len(groups) == 0
@@ -114,7 +114,7 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
def test_secrets(
clan_flake: Path,
test_flake: Path,
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,
age_keys: list["KeyPair"],
@@ -125,7 +125,7 @@ def test_secrets(
assert capsys.readouterr().out == ""
monkeypatch.setenv("SOPS_NIX_SECRET", "foo")
monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(clan_flake / ".." / "age.key"))
monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake / ".." / "age.key"))
cli.run(["secrets", "key", "generate"])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "key", "show"])

6
pkgs/clan-cli/tests/test_ssh_cli.py
View File

@@ -30,9 +30,8 @@ def test_ssh_no_pass(
monkeypatch.delenv("IN_NIX_SANDBOX")
cmd: list[Union[str, utils.Any]] = [
"nix",
fp.any(),
"shell",
"--extra-experimental-features",
"nix-command flakes",
fp.any(),
"-c",
"torify",
@@ -61,9 +60,8 @@ def test_ssh_with_pass(
monkeypatch.delenv("IN_NIX_SANDBOX")
cmd: list[Union[str, utils.Any]] = [
"nix",
fp.any(),
"shell",
"--extra-experimental-features",
"nix-command flakes",
fp.any(),
"-c",
"torify",

33
pkgs/clan-cli/tests/test_vms_api.py Normal file
View File

@@ -0,0 +1,33 @@
from pathlib import Path
import pytest
from api import TestClient
@pytest.mark.impure
def test_inspect(api: TestClient, test_flake_with_core: Path) -> None:
response = api.post(
"/api/vms/inspect",
json=dict(flake_url=str(test_flake_with_core), flake_attr="vm1"),
)
assert response.status_code == 200, "Failed to inspect vm"
config = response.json()["config"]
assert config.get("flake_attr") == "vm1"
assert config.get("cores") == 1
assert config.get("memory_size") == 1024
assert config.get("graphics") is True
@pytest.mark.impure
def test_create(api: TestClient, test_flake_with_core: Path) -> None:
response = api.post(
"/api/vms/create",
json=dict(
flake_url=str(test_flake_with_core),
flake_attr="vm1",
cores=1,
memory_size=1024,
graphics=True,
),
)
assert response.status_code == 200, "Failed to inspect vm"

21
pkgs/ui/nix/pdefs.nix
View File

@@ -10986,6 +10986,11 @@
descriptor = "^0.4.1";
pin = "0.4.1";
};
pretty-bytes = {
descriptor = "^6.1.1";
pin = "6.1.1";
runtime = true;
};
react = {
descriptor = "18.2.0";
pin = "18.2.0";
@@ -13086,6 +13091,9 @@
dev = true;
key = "prettier-plugin-tailwindcss/0.4.1";
};
"node_modules/pretty-bytes" = {
key = "pretty-bytes/6.1.1";
};
"node_modules/printable-characters" = {
dev = true;
key = "printable-characters/1.0.42";
@@ -15195,6 +15203,19 @@
version = "0.4.1";
};
};
pretty-bytes = {
"6.1.1" = {
fetchInfo = {
narHash = "sha256-ERXqMD/9tkPebbHVL3n/9EQRz7mFs5VYO6k/wo5JDzQ=";
type = "tarball";
url = "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz";
};
ident = "pretty-bytes";
ltype = "file";
treeInfo = { };
version = "6.1.1";
};
};
printable-characters = {
"1.0.42" = {
fetchInfo = {

2
pkgs/ui/orval.config.ts
View File

@@ -1,5 +1,5 @@
const config = {
petstore: {
clan: {
output: {
mode: "tags-split",
target: "src/api",

12
pkgs/ui/package-lock.json generated
View File

@@ -22,6 +22,7 @@
"hex-rgb": "^5.0.0",
"next": "13.4.12",
"postcss": "8.4.27",
"pretty-bytes": "^6.1.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.45.4",
@@ -6810,6 +6811,17 @@
}
}
},
"node_modules/pretty-bytes": {
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-6.1.1.tgz",
"integrity": "sha512-mQUvGU6aUFQ+rNvTIAcZuWGRT9a6f6Yrg9bHs4ImKF+HZCEK+plBvnAZYSIQztknZF2qnzNtr6F8s0+IuptdlQ==",
"engines": {
"node": "^14.13.1 || >=16.0.0"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/printable-characters": {
"version": "1.0.42",
"resolved": "https://registry.npmjs.org/printable-characters/-/printable-characters-1.0.42.tgz",

1
pkgs/ui/package.json
View File

@@ -26,6 +26,7 @@
"hex-rgb": "^5.0.0",
"next": "13.4.12",
"postcss": "8.4.27",
"pretty-bytes": "^6.1.1",
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.45.4",

184
pkgs/ui/src/app/join/page.tsx Normal file
View File

@@ -0,0 +1,184 @@
"use client";
import React, { useState } from "react";
import { VmConfig } from "@/api/model";
import { useVms } from "@/components/hooks/useVms";
import prettyBytes from "pretty-bytes";
import {
Alert,
AlertTitle,
Button,
Chip,
LinearProgress,
ListSubheader,
Switch,
Typography,
} from "@mui/material";
import { useSearchParams } from "next/navigation";
import { toast } from "react-hot-toast";
import { Error, Numbers } from "@mui/icons-material";
import { createVm, inspectVm } from "@/api/default/default";
interface FlakeBadgeProps {
flakeUrl: string;
flakeAttr: string;
}
const FlakeBadge = (props: FlakeBadgeProps) => (
<Chip
color="secondary"
label={`${props.flakeUrl}#${props.flakeAttr}`}
sx={{ p: 2 }}
/>
);
interface VmPropLabelProps {
children: React.ReactNode;
}
const VmPropLabel = (props: VmPropLabelProps) => (
<div className="col-span-4 flex items-center sm:col-span-1">
{props.children}
</div>
);
interface VmPropContentProps {
children: React.ReactNode;
}
const VmPropContent = (props: VmPropContentProps) => (
<div className="col-span-4 font-bold sm:col-span-3">{props.children}</div>
);
interface VmDetailsProps {
vmConfig: VmConfig;
}
const VmDetails = (props: VmDetailsProps) => {
const { vmConfig } = props;
const { cores, flake_attr, flake_url, graphics, memory_size } = vmConfig;
const [isStarting, setStarting] = useState(false);
const handleStartVm = async () => {
setStarting(true);
const response = await createVm(vmConfig);
setStarting(false);
if (response.statusText === "OK") {
toast.success(("VM created @ " + response?.data) as string);
} else {
toast.error("Could not create VM");
}
};
return (
<div className="grid grid-cols-4 gap-y-10">
<div className="col-span-4">
<ListSubheader>General</ListSubheader>
</div>
<VmPropLabel>Flake</VmPropLabel>
<VmPropContent>
<FlakeBadge flakeAttr={flake_attr} flakeUrl={flake_url} />
</VmPropContent>
<VmPropLabel>Machine</VmPropLabel>
<VmPropContent>{flake_attr}</VmPropContent>
<div className="col-span-4">
<ListSubheader>VM</ListSubheader>
</div>
<VmPropLabel>CPU Cores</VmPropLabel>
<VmPropContent>
<Numbers fontSize="inherit" />
<span className="font-bold text-black">{cores}</span>
</VmPropContent>
<VmPropLabel>Graphics</VmPropLabel>
<VmPropContent>
<Switch checked={graphics} />
</VmPropContent>
<VmPropLabel>Memory Size</VmPropLabel>
<VmPropContent>{prettyBytes(memory_size * 1024 * 1024)}</VmPropContent>
<div className="col-span-4 grid items-center">
{isStarting && <LinearProgress />}
<Button
disabled={isStarting}
variant="contained"
onClick={handleStartVm}
>
Spin up VM
</Button>
</div>
</div>
);
};
interface ErrorLogOptions {
lines: string[];
}
const ErrorLog = (props: ErrorLogOptions) => {
const { lines } = props;
return (
<div className="w-full bg-slate-800 p-4 text-white shadow-inner shadow-black">
<div className="mb-1 text-slate-400">Log</div>
{lines.map((item, idx) => (
<span key={`${idx}`} className="mb-2 block break-words">
{item}
<br />
</span>
))}
</div>
);
};
export default function Page() {
const queryParams = useSearchParams();
const flakeUrl = queryParams.get("flake") || "";
const flakeAttribute = queryParams.get("attr") || "default";
const { config, error, isLoading } = useVms({
url: flakeUrl,
attr: flakeAttribute,
});
const clanName = "Lassul.us";
return (
<div className="grid h-[70vh] w-full place-items-center gap-y-4">
<Typography variant="h4" className="w-full text-center">
Join{" "}
<Typography variant="h4" className="font-bold" component={"span"}>
{clanName}
</Typography>
{"' "}
Clan
</Typography>
{error && (
<Alert severity="error" className="w-full max-w-xl">
<AlertTitle>Error</AlertTitle>
An Error occurred - See details below
</Alert>
)}
<div className="w-full max-w-xl">
{isLoading && (
<div className="w-full">
<Typography variant="subtitle2">Loading Flake</Typography>
<LinearProgress className="mb-2 w-full" />
<div className="grid w-full place-items-center">
<FlakeBadge flakeUrl={flakeUrl} flakeAttr={flakeAttribute} />
</div>
<Typography variant="subtitle1"></Typography>
</div>
)}
{(!flakeUrl || !flakeAttribute) && <div>Invalid URL</div>}
{config && <VmDetails vmConfig={config} />}
{error && (
<ErrorLog
lines={
error?.response?.data?.detail
?.map((err, idx) => err.msg.split("\n"))
?.flat()
.filter(Boolean) || []
}
/>
)}
</div>
</div>
);
}

50
pkgs/ui/src/components/hooks/useVms.tsx Normal file
View File

@@ -0,0 +1,50 @@
import { inspectVm } from "@/api/default/default";
import { HTTPValidationError, VmConfig } from "@/api/model";
import { AxiosError } from "axios";
import { useEffect, useState } from "react";
import { toast } from "react-hot-toast";
interface UseVmsOptions {
url: string;
attr: string;
}
export const useVms = (options: UseVmsOptions) => {
const { url, attr } = options;
const [isLoading, setIsLoading] = useState(true);
const [config, setConfig] = useState<VmConfig>();
const [error, setError] = useState<AxiosError<HTTPValidationError>>();
useEffect(() => {
const getVmInfo = async (url: string, attr: string) => {
if (url === "") {
toast.error("Flake url is missing", { id: "missing.flake.url" });
return undefined;
}
try {
const response = await inspectVm({
flake_attr: attr,
flake_url: url,
});
const {
data: { config },
} = response;
setError(undefined);
return config;
} catch (e) {
const err = e as AxiosError<HTTPValidationError>;
setError(err);
toast.error(err.message);
return undefined;
} finally {
setIsLoading(false);
}
};
getVmInfo(url, attr).then((c) => setConfig(c));
}, [url, attr]);
return {
error,
isLoading,
config,
};
};