Luis/nextjs-python-web-template
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Befor fixing linting problem

Browse Source
This commit is contained in:
Qubasa
2023-10-22 21:03:06 +02:00
parent 545d389df0
commit c7c47b6527
87 changed files with 703 additions and 3929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pkgs/clan-cli/.envrc
View File

@@ -2,6 +2,7 @@
source_up
if type nix_direnv_watch_file &>/dev/null; then
nix_direnv_watch_file flake-module.nix
nix_direnv_watch_file default.nix

8
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -1,11 +1,15 @@
import argparse
import logging
import sys
from types import ModuleType
from typing import Optional
from . import config, flakes, join, machines, secrets, vms, webui
from .custom_logger import register
from .ssh import cli as ssh_cli
log = logging.getLogger(__name__)
argcomplete: Optional[ModuleType] = None
try:
import argcomplete # type: ignore[no-redef]
@@ -52,6 +56,10 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
parser_vms = subparsers.add_parser("vms", help="manage virtual machines")
vms.register_parser(parser_vms)
# if args.debug:
register(logging.DEBUG)
log.debug("Debug log activated")
if argcomplete:
argcomplete.autocomplete(parser)

18
pkgs/clan-cli/clan_cli/config/__init__.py
View File

@@ -11,9 +11,9 @@ from typing import Any, Optional, Tuple, get_origin
from clan_cli.dirs import machine_settings_file, specific_flake_dir
from clan_cli.errors import ClanError
from clan_cli.flakes.types import FlakeName
from clan_cli.git import commit_file
from clan_cli.nix import nix_eval
from clan_cli.types import FlakeName
script_dir = Path(__file__).parent
@@ -161,7 +161,11 @@ def read_machine_option_value(
def get_or_set_option(args: argparse.Namespace) -> None:
if args.value == []:
print(read_machine_option_value(args.machine, args.option, args.show_trace))
print(
read_machine_option_value(
args.flake, args.machine, args.option, args.show_trace
)
)
else:
# load options
if args.options_file is None:
@@ -308,11 +312,6 @@ def register_parser(
# inject callback function to process the input later
parser.set_defaults(func=get_or_set_option)
parser.add_argument(
"flake",
type=str,
help="name of the flake to set machine options for",
)
parser.add_argument(
"--machine",
"-m",
@@ -356,6 +355,11 @@ def register_parser(
nargs="*",
help="option value to set (if omitted, the current value is printed)",
)
parser.add_argument(
"flake",
type=str,
help="name of the flake to set machine options for",
)
def main(argv: Optional[list[str]] = None) -> None:

2
pkgs/clan-cli/clan_cli/config/machine.py
View File

@@ -14,7 +14,7 @@ from clan_cli.dirs import (
from clan_cli.git import commit_file, find_git_repo_root
from clan_cli.nix import nix_eval
from ..flakes.types import FlakeName
from ..types import FlakeName
def config_for_machine(flake_name: FlakeName, machine_name: str) -> dict:

57
pkgs/clan-cli/clan_cli/custom_logger.py
View File

@@ -1,5 +1,7 @@
import inspect
import logging
from typing import Any
from pathlib import Path
from typing import Any, Callable
grey = "\x1b[38;20m"
yellow = "\x1b[33;20m"
@@ -9,11 +11,20 @@ green = "\u001b[32m"
blue = "\u001b[34m"
def get_formatter(color: str) -> logging.Formatter:
reset = "\x1b[0m"
return logging.Formatter(
f"{color}%(levelname)s{reset}:(%(filename)s:%(lineno)d): %(message)s"
)
def get_formatter(color: str) -> Callable[[logging.LogRecord, bool], logging.Formatter]:
def myformatter(
record: logging.LogRecord, with_location: bool
) -> logging.Formatter:
reset = "\x1b[0m"
filepath = Path(record.pathname).resolve()
if not with_location:
return logging.Formatter(f"{color}%(levelname)s{reset}: %(message)s")
return logging.Formatter(
f"{color}%(levelname)s{reset}: %(message)s\n {filepath}:%(lineno)d::%(funcName)s\n"
)
return myformatter
FORMATTER = {
@@ -26,12 +37,34 @@ FORMATTER = {
class CustomFormatter(logging.Formatter):
def format(self, record: Any) -> str:
return FORMATTER[record.levelno].format(record)
def format(self, record: logging.LogRecord) -> str:
return FORMATTER[record.levelno](record, True).format(record)
class ThreadFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
return FORMATTER[record.levelno](record, False).format(record)
def get_caller() -> str:
frame = inspect.currentframe()
if frame is None:
return "unknown"
caller_frame = frame.f_back
if caller_frame is None:
return "unknown"
caller_frame = caller_frame.f_back
if caller_frame is None:
return "unknown"
frame_info = inspect.getframeinfo(caller_frame)
ret = f"{frame_info.filename}:{frame_info.lineno}::{frame_info.function}"
return ret
def register(level: Any) -> None:
ch = logging.StreamHandler()
ch.setLevel(level)
ch.setFormatter(CustomFormatter())
logging.basicConfig(level=level, handlers=[ch])
handler = logging.StreamHandler()
handler.setLevel(level)
handler.setFormatter(CustomFormatter())
logger = logging.getLogger("registerHandler")
logger.addHandler(handler)
# logging.basicConfig(level=level, handlers=[handler])

66
pkgs/clan-cli/clan_cli/debug.py Normal file
View File

@@ -0,0 +1,66 @@
from typing import Dict, Optional, Tuple, Callable, Any, Mapping, List
from pathlib import Path
import ipdb
import os
import stat
import subprocess
from .dirs import find_git_repo_root
import multiprocessing as mp
from .types import FlakeName
import logging
import sys
import shlex
import time
log = logging.getLogger(__name__)
def command_exec(cmd: List[str], work_dir:Path, env: Dict[str, str]) -> None:
subprocess.run(cmd, check=True, env=env, cwd=work_dir.resolve())
def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: Optional[List[str]] = None) -> None:
if env is None:
env = os.environ.copy()
else:
env = env.copy()
# Error checking
if "bash" in env["SHELL"]:
raise Exception("I assumed you use zsh, not bash")
# Cmd appending
args = ["xterm", "-e", "zsh", "-df"]
if cmd is not None:
mycommand = shlex.join(cmd)
write_command(mycommand, work_dir / "cmd.sh")
print(f"Adding to zsh history the command: {mycommand}", file=sys.stderr)
proc = spawn_process(func=command_exec, cmd=args, work_dir=work_dir, env=env)
try:
ipdb.set_trace()
finally:
proc.terminate()
def write_command(command: str, loc:Path) -> None:
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
f.write(command)
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IEXEC)
def spawn_process(func: Callable, **kwargs:Any) -> mp.Process:
mp.set_start_method(method="spawn")
proc = mp.Process(target=func, kwargs=kwargs)
proc.start()
return proc
def dump_env(env: Dict[str, str], loc: Path) -> None:
cenv = env.copy()
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
for k, v in cenv.items():
if v.count('\n') > 0 or v.count("\"") > 0 or v.count("'") > 0:
continue
f.write(f"export {k}='{v}'\n")
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IEXEC)

16
pkgs/clan-cli/clan_cli/dirs.py
View File

@@ -1,10 +1,13 @@
import logging
import os
import sys
from pathlib import Path
from typing import Optional
from .errors import ClanError
from .flakes.types import FlakeName
from .types import FlakeName
log = logging.getLogger(__name__)
def _get_clan_flake_toplevel() -> Path:
@@ -51,28 +54,31 @@ def user_data_dir() -> Path:
def clan_data_dir() -> Path:
path = user_data_dir() / "clan"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def clan_config_dir() -> Path:
path = user_config_dir() / "clan"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def clan_flakes_dir() -> Path:
path = clan_data_dir() / "flake"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def specific_flake_dir(flake_name: FlakeName) -> Path:
flake_dir = clan_flakes_dir() / flake_name
if not flake_dir.exists():
raise ClanError(f"Flake {flake_name} does not exist")
raise ClanError(f"Flake '{flake_name}' does not exist")
return flake_dir

26
pkgs/clan-cli/clan_cli/flakes/create.py
View File

@@ -8,16 +8,20 @@ from pydantic.tools import parse_obj_as
from ..async_cmd import CmdOut, run, runforcli
from ..dirs import clan_flakes_dir
from ..errors import ClanError
from ..nix import nix_command, nix_shell
DEFAULT_URL: AnyUrl = parse_obj_as(
AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan"
AnyUrl,
"git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan", # TODO: Change me back to main branch
)
async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
if not directory.exists():
directory.mkdir()
else:
raise ClanError(f"Flake at '{directory}' already exists")
response = {}
command = nix_command(
[
@@ -27,27 +31,27 @@ async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
url,
]
)
out = await run(command, directory)
out = await run(command, cwd=directory)
response["flake init"] = out
command = nix_shell(["git"], ["git", "init"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git init"] = out
command = nix_shell(["git"], ["git", "add", "."])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git add"] = out
command = nix_shell(["git"], ["git", "config", "user.name", "clan-tool"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git config"] = out
command = nix_shell(["git"], ["git", "config", "user.email", "clan@example.com"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git config"] = out
command = nix_shell(["git"], ["git", "commit", "-a", "-m", "Initial commit"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git commit"] = out
return response
@@ -55,7 +59,7 @@ async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
def create_flake_command(args: argparse.Namespace) -> None:
flake_dir = clan_flakes_dir() / args.name
runforcli(create_flake, flake_dir, DEFAULT_URL)
runforcli(create_flake, flake_dir, args.url)
# takes a (sub)parser and configures it
@@ -65,5 +69,11 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
type=str,
help="name for the flake",
)
parser.add_argument(
"--url",
type=str,
help="url for the flake",
default=DEFAULT_URL,
)
# parser.add_argument("name", type=str, help="name of the flake")
parser.set_defaults(func=create_flake_command)

3
pkgs/clan-cli/clan_cli/flakes/types.py
View File

@@ -1,3 +0,0 @@
from typing import NewType
FlakeName = NewType("FlakeName", str)

4
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -5,14 +5,16 @@ from typing import Dict
from ..async_cmd import CmdOut, run, runforcli
from ..dirs import specific_flake_dir, specific_machine_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..nix import nix_shell
from ..types import FlakeName
log = logging.getLogger(__name__)
async def create_machine(flake_name: FlakeName, machine_name: str) -> Dict[str, CmdOut]:
folder = specific_machine_dir(flake_name, machine_name)
if folder.exists():
raise ClanError(f"Machine '{machine_name}' already exists")
folder.mkdir(parents=True, exist_ok=True)
# create empty settings.json file inside the folder

2
pkgs/clan-cli/clan_cli/machines/facts.py
View File

@@ -1,5 +1,5 @@
from ..dirs import specific_machine_dir
from ..flakes.types import FlakeName
from ..types import FlakeName
def machine_has_fact(flake_name: FlakeName, machine: str, fact: str) -> bool:

2
pkgs/clan-cli/clan_cli/machines/list.py
View File

@@ -3,7 +3,7 @@ import logging
import os
from ..dirs import machines_dir
from ..flakes.types import FlakeName
from ..types import FlakeName
from .types import validate_hostname
log = logging.getLogger(__name__)

2
pkgs/clan-cli/clan_cli/secrets/folders.py
View File

@@ -5,7 +5,7 @@ from typing import Callable
from ..dirs import specific_flake_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
def get_sops_folder(flake_name: FlakeName) -> Path:

45
pkgs/clan-cli/clan_cli/secrets/groups.py
View File

@@ -3,8 +3,8 @@ import os
from pathlib import Path
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..machines.types import machine_name_type, validate_hostname
from ..types import FlakeName
from . import secrets
from .folders import (
sops_groups_folder,
@@ -204,9 +204,17 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
help="the command to run",
required=True,
)
# List groups
list_parser = subparser.add_parser("list", help="list groups")
list_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
list_parser.set_defaults(func=list_command)
# Add user
add_machine_parser = subparser.add_parser(
"add-machine", help="add a machine to group"
)
@@ -214,8 +222,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
add_machine_parser.add_argument(
"machine", help="the name of the machines to add", type=machine_name_type
)
add_machine_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_machine_parser.set_defaults(func=add_machine_command)
# Remove machine
remove_machine_parser = subparser.add_parser(
"remove-machine", help="remove a machine from group"
)
@@ -223,15 +237,27 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_machine_parser.add_argument(
"machine", help="the name of the machines to remove", type=machine_name_type
)
remove_machine_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_machine_parser.set_defaults(func=remove_machine_command)
# Add user
add_user_parser = subparser.add_parser("add-user", help="add a user to group")
add_group_argument(add_user_parser)
add_user_parser.add_argument(
"user", help="the name of the user to add", type=user_name_type
)
add_user_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_user_parser.set_defaults(func=add_user_command)
# Remove user
remove_user_parser = subparser.add_parser(
"remove-user", help="remove a user from group"
)
@@ -239,8 +265,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_user_parser.add_argument(
"user", help="the name of the user to remove", type=user_name_type
)
remove_user_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_user_parser.set_defaults(func=remove_user_command)
# Add secret
add_secret_parser = subparser.add_parser(
"add-secret", help="allow a user to access a secret"
)
@@ -250,8 +282,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
# Remove secret
remove_secret_parser = subparser.add_parser(
"remove-secret", help="remove a group's access to a secret"
)
@@ -261,4 +299,9 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

38
pkgs/clan-cli/clan_cli/secrets/machines.py
View File

@@ -1,7 +1,7 @@
import argparse
from ..flakes.types import FlakeName
from ..machines.types import machine_name_type, validate_hostname
from ..types import FlakeName
from . import secrets
from .folders import list_objects, remove_object, sops_machines_folder
from .sops import read_key, write_key
@@ -96,11 +96,6 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
action="store_true",
default=False,
)
add_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
@@ -109,6 +104,11 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
help="public key or private key of the user",
type=public_or_private_age_key_type,
)
add_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_parser.set_defaults(func=add_command)
# Parser
@@ -125,46 +125,46 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
# Parser
remove_parser = subparser.add_parser("remove", help="remove a machine")
remove_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
remove_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
remove_parser.set_defaults(func=remove_command)
# Parser
add_secret_parser = subparser.add_parser(
"add-secret", help="allow a machine to access a secret"
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
# Parser
remove_secret_parser = subparser.add_parser(
"remove-secret", help="remove a group's access to a secret"
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.add_argument(
"machine", help="the name of the group", type=machine_name_type
)
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

29
pkgs/clan-cli/clan_cli/secrets/secrets.py
View File

@@ -8,7 +8,7 @@ from typing import IO
from .. import tty
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
from .folders import (
list_objects,
sops_groups_folder,
@@ -253,24 +253,24 @@ def rename_command(args: argparse.Namespace) -> None:
def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
parser_list = subparser.add_parser("list", help="list secrets")
parser_list.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_list.set_defaults(func=list_command)
parser_get = subparser.add_parser("get", help="get a secret")
add_secret_argument(parser_get)
parser_get.set_defaults(func=get_command)
parser_get.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_get.set_defaults(func=get_command)
parser_set = subparser.add_parser("set", help="set a secret")
add_secret_argument(parser_set)
parser_set.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_set.add_argument(
"--group",
type=str,
@@ -299,13 +299,28 @@ def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
default=False,
help="edit the secret with $EDITOR instead of pasting it",
)
parser_set.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_set.set_defaults(func=set_command)
parser_rename = subparser.add_parser("rename", help="rename a secret")
add_secret_argument(parser_rename)
parser_rename.add_argument("new_name", type=str, help="the new name of the secret")
parser_rename.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_rename.set_defaults(func=rename_command)
parser_remove = subparser.add_parser("remove", help="remove a secret")
add_secret_argument(parser_remove)
parser_remove.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_remove.set_defaults(func=remove_command)

2
pkgs/clan-cli/clan_cli/secrets/sops.py
View File

@@ -9,8 +9,8 @@ from typing import IO, Iterator
from ..dirs import user_config_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..nix import nix_shell
from ..types import FlakeName
from .folders import sops_machines_folder, sops_users_folder

6
pkgs/clan-cli/clan_cli/secrets/sops_generate.py
View File

@@ -6,17 +6,19 @@ import sys
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
import logging
from clan_cli.nix import nix_shell
from ..dirs import specific_flake_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
from .folders import sops_secrets_folder
from .machines import add_machine, has_machine
from .secrets import decrypt_secret, encrypt_secret, has_secret
from .sops import generate_private_key
log = logging.getLogger(__name__)
def generate_host_key(flake_name: FlakeName, machine_name: str) -> None:
if has_machine(flake_name, machine_name):
@@ -95,6 +97,7 @@ def generate_secrets_from_nix(
) -> None:
generate_host_key(flake_name, machine_name)
errors = {}
log.debug("Generating secrets for machine %s and flake %s", machine_name, flake_name)
with TemporaryDirectory() as d:
# if any of the secrets are missing, we regenerate all connected facts/secrets
for secret_group, secret_options in secret_submodules.items():
@@ -116,6 +119,7 @@ def upload_age_key_from_nix(
flake_name: FlakeName,
machine_name: str,
) -> None:
log.debug("Uploading secrets for machine %s and flake %s", machine_name, flake_name)
secret_name = f"{machine_name}-age.key"
if not has_secret(
flake_name, secret_name

12
pkgs/clan-cli/clan_cli/secrets/users.py
View File

@@ -1,6 +1,6 @@
import argparse
from ..flakes.types import FlakeName
from ..types import FlakeName
from . import secrets
from .folders import list_objects, remove_object, sops_users_folder
from .sops import read_key, write_key
@@ -131,6 +131,11 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
remove_secret_parser = subparser.add_parser(
@@ -142,4 +147,9 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

22
pkgs/clan-cli/clan_cli/task_manager.py
View File

@@ -12,6 +12,7 @@ from pathlib import Path
from typing import Any, Iterator, Optional, Type, TypeVar
from uuid import UUID, uuid4
from .custom_logger import ThreadFormatter, get_caller
from .errors import ClanError
@@ -38,7 +39,8 @@ class Command:
cwd: Optional[Path] = None,
) -> None:
self.running = True
self.log.debug(f"Running command: {shlex.join(cmd)}")
self.log.debug(f"Command: {shlex.join(cmd)}")
self.log.debug(f"Caller: {get_caller()}")
cwd_res = None
if cwd is not None:
@@ -68,10 +70,10 @@ class Command:
try:
for line in fd:
if fd == self.p.stderr:
print(f"[{cmd[0]}] stderr: {line}")
self.log.debug(f"[{cmd[0]}] stderr: {line}")
self.stderr.append(line)
else:
print(f"[{cmd[0]}] stdout: {line}")
self.log.debug(f"[{cmd[0]}] stdout: {line}")
self.stdout.append(line)
self._output.put(line)
except BlockingIOError:
@@ -80,8 +82,6 @@ class Command:
if self.p.returncode != 0:
raise ClanError(f"Failed to run command: {shlex.join(cmd)}")
self.log.debug("Successfully ran command")
class TaskStatus(str, Enum):
NOTSTARTED = "NOTSTARTED"
@@ -94,7 +94,13 @@ class BaseTask:
def __init__(self, uuid: UUID, num_cmds: int) -> None:
# constructor
self.uuid: UUID = uuid
self.log = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(ThreadFormatter())
logger = logging.getLogger(__name__)
logger.addHandler(handler)
self.log = logger
self.log = logger
self.procs: list[Command] = []
self.status = TaskStatus.NOTSTARTED
self.logs_lock = threading.Lock()
@@ -108,6 +114,10 @@ class BaseTask:
self.status = TaskStatus.RUNNING
try:
self.run()
# TODO: We need to check, if too many commands have been initialized,
# but not run. This would deadlock the log_lines() function.
# Idea: Run next(cmds) and check if it raises StopIteration if not,
# we have too many commands
except Exception as e:
# FIXME: fix exception handling here
traceback.print_exception(*sys.exc_info())

23
pkgs/clan-cli/clan_cli/types.py Normal file
View File

@@ -0,0 +1,23 @@
import logging
from pathlib import Path
from typing import NewType
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path

192
pkgs/clan-cli/clan_cli/vms/create.py
View File

@@ -4,20 +4,22 @@ import json
import os
import shlex
import sys
import tempfile
from pathlib import Path
from typing import Iterator
from typing import Iterator, Dict
from uuid import UUID
from ..dirs import specific_flake_dir
from ..nix import nix_build, nix_config, nix_shell
from ..dirs import clan_flakes_dir, specific_flake_dir
from ..nix import nix_build, nix_config, nix_eval, nix_shell
from ..task_manager import BaseTask, Command, create_task
from ..types import validate_path
from .inspect import VmConfig, inspect_vm
from ..errors import ClanError
from ..debug import repro_env_break
class BuildVmTask(BaseTask):
def __init__(self, uuid: UUID, vm: VmConfig) -> None:
super().__init__(uuid, num_cmds=6)
super().__init__(uuid, num_cmds=7)
self.vm = vm
def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
@@ -34,11 +36,18 @@ class BuildVmTask(BaseTask):
]
)
)
vm_json = "".join(cmd.stdout)
vm_json = "".join(cmd.stdout).strip()
self.log.debug(f"VM JSON path: {vm_json}")
with open(vm_json.strip()) as f:
with open(vm_json) as f:
return json.load(f)
def get_clan_name(self, cmds: Iterator[Command]) -> str:
clan_dir = self.vm.flake_url
cmd = next(cmds)
cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]))
clan_name = cmd.stdout[0].strip().strip('"')
return clan_name
def run(self) -> None:
cmds = self.commands()
@@ -47,99 +56,106 @@ class BuildVmTask(BaseTask):
# TODO: We should get this from the vm argument
vm_config = self.get_vm_create_info(cmds)
clan_name = self.get_clan_name(cmds)
with tempfile.TemporaryDirectory() as tmpdir_:
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir()
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir()
disk_img = f"{tmpdir_}/disk.img"
self.log.debug(f"Building VM for clan name: {clan_name}")
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
flake_dir = clan_flakes_dir() / clan_name
validate_path(clan_flakes_dir(), flake_dir)
flake_dir.mkdir(exist_ok=True)
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["generateSecrets"]],
env=env,
)
else:
cmd.run(["echo", "won't generate secrets for non local clan"])
xchg_dir = flake_dir / "xchg"
xchg_dir.mkdir()
secrets_dir = flake_dir / "secrets"
secrets_dir.mkdir()
disk_img = f"{flake_dir}/disk.img"
cmd = next(cmds)
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
cmd = next(cmds)
repro_env_break(work_dir=flake_dir, env=env, cmd=[vm_config["generateSecrets"], clan_name])
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["uploadSecrets"]],
[vm_config["generateSecrets"], clan_name],
env=env,
)
else:
self.log.warning("won't generate secrets for non local clan")
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
cmd = next(cmds)
cmd.run(
[vm_config["uploadSecrets"]],
env=env,
)
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
)
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
def create_vm(vm: VmConfig) -> BuildVmTask:

16
pkgs/clan-cli/clan_cli/webui/api_inputs.py
View File

@@ -6,25 +6,11 @@ from pydantic import AnyUrl, BaseModel, validator
from ..dirs import clan_data_dir, clan_flakes_dir
from ..flakes.create import DEFAULT_URL
from ..types import validate_path
log = logging.getLogger(__name__)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path
class ClanDataPath(BaseModel):
dest: Path

2
pkgs/clan-cli/clan_cli/webui/routers/machines.py
View File

@@ -9,9 +9,9 @@ from ...config.machine import (
schema_for_machine,
set_config_for_machine,
)
from ...flakes.types import FlakeName
from ...machines.create import create_machine as _create_machine
from ...machines.list import list_machines as _list_machines
from ...types import FlakeName
from ..api_outputs import (
ConfigResponse,
Machine,

10
pkgs/clan-cli/default.nix
View File

@@ -8,9 +8,11 @@
, openssh
, pytest
, pytest-cov
, pytest-xdist
, pytest-subprocess
, pytest-parallel
, pytest-timeout
, remote-pdb
, ipdb
, python3
, runCommand
, setuptools
@@ -45,8 +47,10 @@ let
pytest
pytest-cov
pytest-subprocess
pytest-parallel
pytest-xdist
pytest-timeout
remote-pdb
ipdb
openssh
git
gnupg
@@ -80,9 +84,7 @@ let
source = runCommand "clan-cli-source" { } ''
cp -r ${./.} $out
chmod -R +w $out
rm $out/clan_cli/config/jsonschema
ln -s ${nixpkgs'} $out/clan_cli/nixpkgs
cp -r ${../../lib/jsonschema} $out/clan_cli/config/jsonschema
ln -s ${ui-assets} $out/clan_cli/webui/assets
'';
nixpkgs' = runCommand "nixpkgs" { nativeBuildInputs = [ nix ]; } ''

7
pkgs/clan-cli/pyproject.toml
View File

@@ -14,9 +14,14 @@ exclude = ["clan_cli.nixpkgs*"]
[tool.setuptools.package-data]
clan_cli = [ "config/jsonschema/*", "webui/assets/**/*"]
[tool.pytest.ini_options]
testpaths = "tests"
faulthandler_timeout = 60
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --workers auto --durations 5"
log_level = "DEBUG"
log_format = "%(levelname)s: %(message)s"
addopts = "--cov . --cov-report term --cov-report html:.reports/html --no-cov-on-fail --durations 5 --color=yes --maxfail=1 --new-first -nauto" # Add --pdb for debugging
norecursedirs = "tests/helpers"
markers = [ "impure" ]

57
pkgs/clan-cli/shell.nix
View File

@@ -22,38 +22,41 @@ mkShell {
];
shellHook = ''
tmp_path=$(realpath ./.direnv)
tmp_path=$(realpath ./.direnv)
repo_root=$(realpath .)
mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}"
repo_root=$(realpath .)
mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}"
# Install the package in editable mode
# This allows executing `clan` from within the dev-shell using the current
# version of the code and its dependencies.
${pythonWithDeps.interpreter} -m pip install \
--quiet \
--disable-pip-version-check \
--no-index \
--no-build-isolation \
--prefix "$tmp_path/python" \
--editable $repo_root
# Install the package in editable mode
# This allows executing `clan` from within the dev-shell using the current
# version of the code and its dependencies.
${pythonWithDeps.interpreter} -m pip install \
--quiet \
--disable-pip-version-check \
--no-index \
--no-build-isolation \
--prefix "$tmp_path/python" \
--editable $repo_root
rm -f clan_cli/nixpkgs clan_cli/webui/assets
ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs
ln -sf ${ui-assets} clan_cli/webui/assets
rm -f clan_cli/nixpkgs clan_cli/webui/assets
ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs
ln -sf ${ui-assets} clan_cli/webui/assets
export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH"
export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:"
export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH"
export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:"
export PYTHONBREAKPOINT=ipdb.set_trace
export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
mkdir -p \
$tmp_path/share/fish/vendor_completions.d \
$tmp_path/share/bash-completion/completions \
$tmp_path/share/zsh/site-functions
register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish
register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan
export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
mkdir -p \
$tmp_path/share/fish/vendor_completions.d \
$tmp_path/share/bash-completion/completions \
$tmp_path/share/zsh/site-functions
register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish
register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan
./bin/clan machines create example
./bin/clan flakes create example_clan
./bin/clan machines create example_machine example_clan
'';
}

99
pkgs/clan-cli/tests/fixtures_flakes.py
View File

@@ -1,4 +1,5 @@
import fileinput
import logging
import shutil
import tempfile
from pathlib import Path
@@ -8,7 +9,9 @@ import pytest
from root import CLAN_CORE
from clan_cli.dirs import nixpkgs_source
from clan_cli.flakes.types import FlakeName
from clan_cli.types import FlakeName
log = logging.getLogger(__name__)
# substitutes string sin a file.
@@ -28,73 +31,85 @@ def substitute(
print(line, end="")
class TestFlake(NamedTuple):
class FlakeForTest(NamedTuple):
name: FlakeName
path: Path
def create_flake(
monkeypatch: pytest.MonkeyPatch,
temporary_dir: Path,
flake_name: FlakeName,
clan_core_flake: Path | None = None,
machines: list[str] = [],
remote: bool = False,
) -> Iterator[TestFlake]:
) -> Iterator[FlakeForTest]:
"""
Creates a flake with the given name and machines.
The machine names map to the machines in ./test_machines
"""
template = Path(__file__).parent / flake_name
# copy the template to a new temporary location
with tempfile.TemporaryDirectory() as tmpdir_:
home = Path(tmpdir_)
flake = home / flake_name
shutil.copytree(template, flake)
# lookup the requested machines in ./test_machines and include them
if machines:
(flake / "machines").mkdir(parents=True, exist_ok=True)
for machine_name in machines:
machine_path = Path(__file__).parent / "machines" / machine_name
shutil.copytree(machine_path, flake / "machines" / machine_name)
substitute(flake / "machines" / machine_name / "default.nix", flake)
# in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
# provided by get_test_flake_toplevel
flake_nix = flake / "flake.nix"
# this is where we would install the sops key to, when updating
substitute(flake_nix, clan_core_flake, flake)
if remote:
with tempfile.TemporaryDirectory() as workdir:
monkeypatch.chdir(workdir)
monkeypatch.setenv("HOME", str(home))
yield TestFlake(flake_name, flake)
else:
monkeypatch.chdir(flake)
home = Path(temporary_dir)
flake = home / ".local/state/clan/flake" / flake_name
shutil.copytree(template, flake)
# lookup the requested machines in ./test_machines and include them
if machines:
(flake / "machines").mkdir(parents=True, exist_ok=True)
for machine_name in machines:
machine_path = Path(__file__).parent / "machines" / machine_name
shutil.copytree(machine_path, flake / "machines" / machine_name)
substitute(flake / "machines" / machine_name / "default.nix", flake)
# in the flake.nix file replace the string __CLAN_URL__ with the the clan flake
# provided by get_test_flake_toplevel
flake_nix = flake / "flake.nix"
# this is where we would install the sops key to, when updating
substitute(flake_nix, clan_core_flake, flake)
if remote:
with tempfile.TemporaryDirectory() as workdir:
monkeypatch.chdir(workdir)
monkeypatch.setenv("HOME", str(home))
yield TestFlake(flake_name, flake)
yield FlakeForTest(flake_name, flake)
else:
monkeypatch.chdir(flake)
monkeypatch.setenv("HOME", str(home))
yield FlakeForTest(flake_name, flake)
@pytest.fixture
def test_flake(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestFlake]:
yield from create_flake(monkeypatch, FlakeName("test_flake"))
def test_flake(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
) -> Iterator[FlakeForTest]:
yield from create_flake(monkeypatch, temporary_home, FlakeName("test_flake"))
@pytest.fixture
def test_flake_with_core(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestFlake]:
if not (CLAN_CORE / "flake.nix").exists():
raise Exception(
"clan-core flake not found. This test requires the clan-core flake to be present"
)
yield from create_flake(monkeypatch, FlakeName("test_flake_with_core"), CLAN_CORE)
@pytest.fixture
def test_flake_with_core_and_pass(
monkeypatch: pytest.MonkeyPatch,
) -> Iterator[TestFlake]:
def test_flake_with_core(
monkeypatch: pytest.MonkeyPatch, temporary_dir: Path
) -> Iterator[FlakeForTest]:
if not (CLAN_CORE / "flake.nix").exists():
raise Exception(
"clan-core flake not found. This test requires the clan-core flake to be present"
)
yield from create_flake(
monkeypatch, FlakeName("test_flake_with_core_and_pass"), CLAN_CORE
monkeypatch, temporary_dir, FlakeName("test_flake_with_core"), CLAN_CORE
)
@pytest.fixture
def test_flake_with_core_and_pass(
monkeypatch: pytest.MonkeyPatch,
temporary_dir: Path,
) -> Iterator[FlakeForTest]:
if not (CLAN_CORE / "flake.nix").exists():
raise Exception(
"clan-core flake not found. This test requires the clan-core flake to be present"
)
yield from create_flake(
monkeypatch,
temporary_dir,
FlakeName("test_flake_with_core_and_pass"),
CLAN_CORE,
)

8
pkgs/clan-cli/tests/helpers/cli.py
View File

@@ -1,6 +1,11 @@
import argparse
import logging
import shlex
from clan_cli import create_parser
from clan_cli.custom_logger import get_caller
log = logging.getLogger(__name__)
class Cli:
@@ -8,6 +13,9 @@ class Cli:
self.parser = create_parser(prog="clan")
def run(self, args: list[str]) -> argparse.Namespace:
cmd = shlex.join(["clan"] + args)
log.debug(f"$ {cmd}")
log.debug(f"Caller {get_caller()}")
parsed = self.parser.parse_args(args)
if hasattr(parsed, "func"):
parsed.func(parsed)

17
pkgs/clan-cli/tests/temporary_dir.py
View File

@@ -1,3 +1,4 @@
import logging
import os
import tempfile
from pathlib import Path
@@ -5,14 +6,20 @@ from typing import Iterator
import pytest
log = logging.getLogger(__name__)
@pytest.fixture
def temporary_dir() -> Iterator[Path]:
if os.getenv("TEST_KEEP_TEMPORARY_DIR"):
temp_dir = tempfile.mkdtemp(prefix="pytest-")
path = Path(temp_dir)
def temporary_home(monkeypatch: pytest.MonkeyPatch) -> Iterator[Path]:
env_dir = os.getenv("TEST_TEMPORARY_DIR")
if env_dir is not None:
path = Path(env_dir).resolve()
log.debug("Temp HOME directory: %s", str(path))
monkeypatch.setenv("HOME", str(path))
yield path
print("=========> Keeping temporary directory: ", path)
else:
log.debug("TEST_TEMPORARY_DIR not set, using TemporaryDirectory")
with tempfile.TemporaryDirectory(prefix="pytest-") as dirpath:
monkeypatch.setenv("HOME", str(dirpath))
log.debug("Temp HOME directory: %s", str(dirpath))
yield Path(dirpath)

13
pkgs/clan-cli/tests/test_config.py
View File

@@ -9,6 +9,7 @@ from cli import Cli
from clan_cli import config
from clan_cli.config import parsing
from clan_cli.errors import ClanError
from fixtures_flakes import FlakeForTest
example_options = f"{Path(config.__file__).parent}/jsonschema/options.json"
@@ -29,7 +30,7 @@ example_options = f"{Path(config.__file__).parent}/jsonschema/options.json"
def test_set_some_option(
args: list[str],
expected: dict[str, Any],
test_flake: Path,
test_flake: FlakeForTest,
) -> None:
# create temporary file for out_file
with tempfile.NamedTemporaryFile() as out_file:
@@ -46,24 +47,24 @@ def test_set_some_option(
out_file.name,
]
+ args
+ [test_flake.name]
)
json_out = json.loads(open(out_file.name).read())
assert json_out == expected
def test_configure_machine(
test_flake: Path,
temporary_dir: Path,
test_flake: FlakeForTest,
temporary_home: Path,
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("HOME", str(temporary_dir))
cli = Cli()
cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true"])
cli.run(["config", "-m", "machine1", "clan.jitsi.enable", "true", test_flake.name])
# clear the output buffer
capsys.readouterr()
# read a option value
cli.run(["config", "-m", "machine1", "clan.jitsi.enable"])
cli.run(["config", "-m", "machine1", "clan.jitsi.enable", test_flake.name])
# read the output
assert capsys.readouterr().out == "true\n"

45
pkgs/clan-cli/tests/test_create_flake.py
View File

@@ -6,6 +6,9 @@ import pytest
from api import TestClient
from cli import Cli
from clan_cli.dirs import clan_flakes_dir
from clan_cli.flakes.create import DEFAULT_URL
@pytest.fixture
def cli() -> Cli:
@@ -14,15 +17,16 @@ def cli() -> Cli:
@pytest.mark.impure
def test_create_flake_api(
monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_dir: Path
monkeypatch: pytest.MonkeyPatch, api: TestClient, temporary_home: Path
) -> None:
flake_dir = temporary_dir / "flake_dir"
flake_dir_str = str(flake_dir.resolve())
monkeypatch.chdir(clan_flakes_dir())
flake_name = "flake_dir"
flake_dir = clan_flakes_dir() / flake_name
response = api.post(
"/api/flake/create",
json=dict(
dest=flake_dir_str,
url="git+https://git.clan.lol/clan/clan-core#new-clan",
dest=str(flake_dir),
url=str(DEFAULT_URL),
),
)
@@ -34,19 +38,21 @@ def test_create_flake_api(
@pytest.mark.impure
def test_create_flake(
monkeypatch: pytest.MonkeyPatch,
temporary_dir: Path,
capsys: pytest.CaptureFixture,
temporary_home: Path,
cli: Cli,
) -> None:
monkeypatch.chdir(temporary_dir)
flake_dir = temporary_dir / "flake_dir"
flake_dir_str = str(flake_dir.resolve())
cli.run(["flake", "create", flake_dir_str])
monkeypatch.chdir(clan_flakes_dir())
flake_name = "flake_dir"
flake_dir = clan_flakes_dir() / flake_name
cli.run(["flakes", "create", flake_name])
assert (flake_dir / ".clan-flake").exists()
monkeypatch.chdir(flake_dir)
cli.run(["machines", "create", "machine1"])
cli.run(["machines", "create", "machine1", flake_name])
capsys.readouterr() # flush cache
cli.run(["machines", "list"])
cli.run(["machines", "list", flake_name])
assert "machine1" in capsys.readouterr().out
flake_show = subprocess.run(
["nix", "flake", "show", "--json"],
@@ -61,6 +67,17 @@ def test_create_flake(
pytest.fail("nixosConfigurations.machine1 not found in flake outputs")
# configure machine1
capsys.readouterr()
cli.run(["config", "--machine", "machine1", "services.openssh.enable"])
cli.run(
["config", "--machine", "machine1", "services.openssh.enable", "", flake_name]
)
capsys.readouterr()
cli.run(["config", "--machine", "machine1", "services.openssh.enable", "true"])
cli.run(
[
"config",
"--machine",
"machine1",
"services.openssh.enable",
"true",
flake_name,
]
)

1
pkgs/clan-cli/tests/test_flake_with_core/flake.nix
View File

@@ -9,6 +9,7 @@
let
clan = clan-core.lib.buildClan {
directory = self;
clanName = "test_with_core_clan";
machines = {
vm1 = { lib, ... }: {
clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";

1
pkgs/clan-cli/tests/test_flake_with_core_and_pass/flake.nix
View File

@@ -9,6 +9,7 @@
let
clan = clan-core.lib.buildClan {
directory = self;
clanName = "test_with_core_and_pass_clan";
machines = {
vm1 = { lib, ... }: {
clan.networking.deploymentAddress = "__CLAN_DEPLOYMENT_ADDRESS__";

1
pkgs/clan-cli/tests/test_flake_with_core_dynamic_machines/flake.nix
View File

@@ -9,6 +9,7 @@
let
clan = clan-core.lib.buildClan {
directory = self;
clanName = "core_dynamic_machine_clan";
machines =
let
machineModules = builtins.readDir (self + "/machines");

4
pkgs/clan-cli/tests/test_machines_config.py
View File

@@ -1,8 +1,8 @@
from fixtures_flakes import TestFlake
from fixtures_flakes import FlakeForTest
from clan_cli.config import machine
def test_schema_for_machine(test_flake: TestFlake) -> None:
def test_schema_for_machine(test_flake: FlakeForTest) -> None:
schema = machine.schema_for_machine(test_flake.name, "machine1")
assert "properties" in schema

140
pkgs/clan-cli/tests/test_secrets_cli.py
View File

@@ -1,30 +1,33 @@
import logging
import os
from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Iterator
import pytest
from cli import Cli
from fixtures_flakes import FlakeForTest
from clan_cli.errors import ClanError
if TYPE_CHECKING:
from age_keys import KeyPair
log = logging.getLogger(__name__)
def _test_identities(
what: str,
test_flake: Path,
test_flake: FlakeForTest,
capsys: pytest.CaptureFixture,
age_keys: list["KeyPair"],
) -> None:
cli = Cli()
sops_folder = test_flake / "sops"
sops_folder = test_flake.path / "sops"
cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey])
cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey, test_flake.name])
assert (sops_folder / what / "foo" / "key.json").exists()
with pytest.raises(ClanError):
cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey])
cli.run(["secrets", what, "add", "foo", age_keys[0].pubkey, test_flake.name])
cli.run(
[
@@ -34,73 +37,80 @@ def _test_identities(
"-f",
"foo",
age_keys[0].privkey,
test_flake.name,
]
)
capsys.readouterr() # empty the buffer
cli.run(["secrets", what, "get", "foo"])
cli.run(["secrets", what, "get", "foo", test_flake.name])
out = capsys.readouterr() # empty the buffer
assert age_keys[0].pubkey in out.out
capsys.readouterr() # empty the buffer
cli.run(["secrets", what, "list"])
cli.run(["secrets", what, "list", test_flake.name])
out = capsys.readouterr() # empty the buffer
assert "foo" in out.out
cli.run(["secrets", what, "remove", "foo"])
cli.run(["secrets", what, "remove", "foo", test_flake.name])
assert not (sops_folder / what / "foo" / "key.json").exists()
with pytest.raises(ClanError): # already removed
cli.run(["secrets", what, "remove", "foo"])
cli.run(["secrets", what, "remove", "foo", test_flake.name])
capsys.readouterr()
cli.run(["secrets", what, "list"])
cli.run(["secrets", what, "list", test_flake.name])
out = capsys.readouterr()
assert "foo" not in out.out
def test_users(
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
_test_identities("users", test_flake, capsys, age_keys)
def test_machines(
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
_test_identities("machines", test_flake, capsys, age_keys)
def test_groups(
test_flake: Path, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
test_flake: FlakeForTest, capsys: pytest.CaptureFixture, age_keys: list["KeyPair"]
) -> None:
cli = Cli()
capsys.readouterr() # empty the buffer
cli.run(["secrets", "groups", "list"])
cli.run(["secrets", "groups", "list", test_flake.name])
assert capsys.readouterr().out == ""
with pytest.raises(ClanError): # machine does not exist yet
cli.run(["secrets", "groups", "add-machine", "group1", "machine1"])
cli.run(
["secrets", "groups", "add-machine", "group1", "machine1", test_flake.name]
)
with pytest.raises(ClanError): # user does not exist yet
cli.run(["secrets", "groups", "add-user", "groupb1", "user1"])
cli.run(["secrets", "machines", "add", "machine1", age_keys[0].pubkey])
cli.run(["secrets", "groups", "add-machine", "group1", "machine1"])
cli.run(["secrets", "groups", "add-user", "groupb1", "user1", test_flake.name])
cli.run(
["secrets", "machines", "add", "machine1", age_keys[0].pubkey, test_flake.name]
)
cli.run(["secrets", "groups", "add-machine", "group1", "machine1", test_flake.name])
# Should this fail?
cli.run(["secrets", "groups", "add-machine", "group1", "machine1"])
cli.run(["secrets", "groups", "add-machine", "group1", "machine1", test_flake.name])
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
cli.run(["secrets", "groups", "add-user", "group1", "user1"])
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey, test_flake.name])
cli.run(["secrets", "groups", "add-user", "group1", "user1", test_flake.name])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "groups", "list"])
cli.run(["secrets", "groups", "list", test_flake.name])
out = capsys.readouterr().out
assert "user1" in out
assert "machine1" in out
cli.run(["secrets", "groups", "remove-user", "group1", "user1"])
cli.run(["secrets", "groups", "remove-machine", "group1", "machine1"])
groups = os.listdir(test_flake / "sops" / "groups")
cli.run(["secrets", "groups", "remove-user", "group1", "user1", test_flake.name])
cli.run(
["secrets", "groups", "remove-machine", "group1", "machine1", test_flake.name]
)
groups = os.listdir(test_flake.path / "sops" / "groups")
assert len(groups) == 0
@@ -117,104 +127,114 @@ def use_key(key: str, monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
def test_secrets(
test_flake: Path,
test_flake: FlakeForTest,
capsys: pytest.CaptureFixture,
monkeypatch: pytest.MonkeyPatch,
age_keys: list["KeyPair"],
) -> None:
cli = Cli()
capsys.readouterr() # empty the buffer
cli.run(["secrets", "list"])
cli.run(["secrets", "list", test_flake.name])
assert capsys.readouterr().out == ""
monkeypatch.setenv("SOPS_NIX_SECRET", "foo")
monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake / ".." / "age.key"))
monkeypatch.setenv("SOPS_AGE_KEY_FILE", str(test_flake.path / ".." / "age.key"))
cli.run(["secrets", "key", "generate"])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "key", "show"])
key = capsys.readouterr().out
assert key.startswith("age1")
cli.run(["secrets", "users", "add", "testuser", key])
cli.run(["secrets", "users", "add", "testuser", key, test_flake.name])
with pytest.raises(ClanError): # does not exist yet
cli.run(["secrets", "get", "nonexisting"])
cli.run(["secrets", "set", "initialkey"])
cli.run(["secrets", "get", "nonexisting", test_flake.name])
cli.run(["secrets", "set", "initialkey", test_flake.name])
capsys.readouterr()
cli.run(["secrets", "get", "initialkey"])
cli.run(["secrets", "get", "initialkey", test_flake.name])
assert capsys.readouterr().out == "foo"
capsys.readouterr()
cli.run(["secrets", "users", "list"])
cli.run(["secrets", "users", "list", test_flake.name])
users = capsys.readouterr().out.rstrip().split("\n")
assert len(users) == 1, f"users: {users}"
owner = users[0]
monkeypatch.setenv("EDITOR", "cat")
cli.run(["secrets", "set", "--edit", "initialkey"])
cli.run(["secrets", "set", "--edit", "initialkey", test_flake.name])
monkeypatch.delenv("EDITOR")
cli.run(["secrets", "rename", "initialkey", "key"])
cli.run(["secrets", "rename", "initialkey", "key", test_flake.name])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "list"])
cli.run(["secrets", "list", test_flake.name])
assert capsys.readouterr().out == "key\n"
cli.run(["secrets", "machines", "add", "machine1", age_keys[0].pubkey])
cli.run(["secrets", "machines", "add-secret", "machine1", "key"])
cli.run(
["secrets", "machines", "add", "machine1", age_keys[0].pubkey, test_flake.name]
)
cli.run(["secrets", "machines", "add-secret", "machine1", "key", test_flake.name])
capsys.readouterr()
cli.run(["secrets", "machines", "list"])
cli.run(["secrets", "machines", "list", test_flake.name])
assert capsys.readouterr().out == "machine1\n"
with use_key(age_keys[0].privkey, monkeypatch):
capsys.readouterr()
cli.run(["secrets", "get", "key"])
cli.run(["secrets", "get", "key", test_flake.name])
assert capsys.readouterr().out == "foo"
cli.run(["secrets", "machines", "remove-secret", "machine1", "key"])
cli.run(
["secrets", "machines", "remove-secret", "machine1", "key", test_flake.name]
)
cli.run(["secrets", "users", "add", "user1", age_keys[1].pubkey])
cli.run(["secrets", "users", "add-secret", "user1", "key"])
cli.run(["secrets", "users", "add", "user1", age_keys[1].pubkey, test_flake.name])
cli.run(["secrets", "users", "add-secret", "user1", "key", test_flake.name])
capsys.readouterr()
with use_key(age_keys[1].privkey, monkeypatch):
cli.run(["secrets", "get", "key"])
cli.run(["secrets", "get", "key", test_flake.name])
assert capsys.readouterr().out == "foo"
cli.run(["secrets", "users", "remove-secret", "user1", "key"])
cli.run(["secrets", "users", "remove-secret", "user1", "key", test_flake.name])
with pytest.raises(ClanError): # does not exist yet
cli.run(["secrets", "groups", "add-secret", "admin-group", "key"])
cli.run(["secrets", "groups", "add-user", "admin-group", "user1"])
cli.run(["secrets", "groups", "add-user", "admin-group", owner])
cli.run(["secrets", "groups", "add-secret", "admin-group", "key"])
cli.run(
["secrets", "groups", "add-secret", "admin-group", "key", test_flake.name]
)
cli.run(["secrets", "groups", "add-user", "admin-group", "user1", test_flake.name])
cli.run(["secrets", "groups", "add-user", "admin-group", owner, test_flake.name])
cli.run(["secrets", "groups", "add-secret", "admin-group", "key", test_flake.name])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "set", "--group", "admin-group", "key2"])
cli.run(["secrets", "set", "--group", "admin-group", "key2", test_flake.name])
with use_key(age_keys[1].privkey, monkeypatch):
capsys.readouterr()
cli.run(["secrets", "get", "key"])
cli.run(["secrets", "get", "key", test_flake.name])
assert capsys.readouterr().out == "foo"
# extend group will update secrets
cli.run(["secrets", "users", "add", "user2", age_keys[2].pubkey])
cli.run(["secrets", "groups", "add-user", "admin-group", "user2"])
cli.run(["secrets", "users", "add", "user2", age_keys[2].pubkey, test_flake.name])
cli.run(["secrets", "groups", "add-user", "admin-group", "user2", test_flake.name])
with use_key(age_keys[2].privkey, monkeypatch): # user2
capsys.readouterr()
cli.run(["secrets", "get", "key"])
cli.run(["secrets", "get", "key", test_flake.name])
assert capsys.readouterr().out == "foo"
cli.run(["secrets", "groups", "remove-user", "admin-group", "user2"])
cli.run(
["secrets", "groups", "remove-user", "admin-group", "user2", test_flake.name]
)
with pytest.raises(ClanError), use_key(age_keys[2].privkey, monkeypatch):
# user2 is not in the group anymore
capsys.readouterr()
cli.run(["secrets", "get", "key"])
cli.run(["secrets", "get", "key", test_flake.name])
print(capsys.readouterr().out)
cli.run(["secrets", "groups", "remove-secret", "admin-group", "key"])
cli.run(
["secrets", "groups", "remove-secret", "admin-group", "key", test_flake.name]
)
cli.run(["secrets", "remove", "key"])
cli.run(["secrets", "remove", "key2"])
cli.run(["secrets", "remove", "key", test_flake.name])
cli.run(["secrets", "remove", "key2", test_flake.name])
capsys.readouterr() # empty the buffer
cli.run(["secrets", "list"])
cli.run(["secrets", "list", test_flake.name])
assert capsys.readouterr().out == ""

4
pkgs/clan-cli/tests/test_secrets_generate.py
View File

@@ -2,7 +2,7 @@ from typing import TYPE_CHECKING
import pytest
from cli import Cli
from fixtures_flakes import TestFlake
from fixtures_flakes import FlakeForTest
from clan_cli.machines.facts import machine_get_fact
from clan_cli.secrets.folders import sops_secrets_folder
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
@pytest.mark.impure
def test_generate_secret(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core: TestFlake,
test_flake_with_core: FlakeForTest,
age_keys: list["KeyPair"],
) -> None:
monkeypatch.chdir(test_flake_with_core.path)

4
pkgs/clan-cli/tests/test_secrets_password_store.py
View File

@@ -3,7 +3,7 @@ from pathlib import Path
import pytest
from cli import Cli
from fixtures_flakes import TestFlake
from fixtures_flakes import FlakeForTest
from clan_cli.machines.facts import machine_get_fact
from clan_cli.nix import nix_shell
@@ -13,7 +13,7 @@ from clan_cli.ssh import HostGroup
@pytest.mark.impure
def test_upload_secret(
monkeypatch: pytest.MonkeyPatch,
test_flake_with_core_and_pass: TestFlake,
test_flake_with_core_and_pass: FlakeForTest,
temporary_dir: Path,
host_group: HostGroup,
) -> None:

25
pkgs/clan-cli/tests/test_vms_api_create.py
View File

@@ -5,20 +5,23 @@ from typing import TYPE_CHECKING, Iterator
import pytest
from api import TestClient
from cli import Cli
from fixtures_flakes import TestFlake, create_flake
from fixtures_flakes import FlakeForTest, create_flake
from httpx import SyncByteStream
from root import CLAN_CORE
from clan_cli.flakes.types import FlakeName
from clan_cli.types import FlakeName
if TYPE_CHECKING:
from age_keys import KeyPair
@pytest.fixture
def flake_with_vm_with_secrets(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestFlake]:
def flake_with_vm_with_secrets(
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
) -> Iterator[FlakeForTest]:
yield from create_flake(
monkeypatch,
temporary_home,
FlakeName("test_flake_with_core_dynamic_machines"),
CLAN_CORE,
machines=["vm_with_secrets"],
@@ -27,10 +30,11 @@ def flake_with_vm_with_secrets(monkeypatch: pytest.MonkeyPatch) -> Iterator[Test
@pytest.fixture
def remote_flake_with_vm_without_secrets(
monkeypatch: pytest.MonkeyPatch,
) -> Iterator[TestFlake]:
monkeypatch: pytest.MonkeyPatch, temporary_home: Path
) -> Iterator[FlakeForTest]:
yield from create_flake(
monkeypatch,
temporary_home,
FlakeName("test_flake_with_core_dynamic_machines"),
CLAN_CORE,
machines=["vm_without_secrets"],
@@ -41,11 +45,12 @@ def remote_flake_with_vm_without_secrets(
@pytest.fixture
def create_user_with_age_key(
monkeypatch: pytest.MonkeyPatch,
test_flake: FlakeForTest,
age_keys: list["KeyPair"],
) -> None:
monkeypatch.setenv("SOPS_AGE_KEY", age_keys[0].privkey)
cli = Cli()
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey])
cli.run(["secrets", "users", "add", "user1", age_keys[0].pubkey, test_flake.name])
def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
@@ -91,10 +96,10 @@ def generic_create_vm_test(api: TestClient, flake: Path, vm: str) -> None:
def test_create_local(
api: TestClient,
monkeypatch: pytest.MonkeyPatch,
flake_with_vm_with_secrets: Path,
flake_with_vm_with_secrets: FlakeForTest,
create_user_with_age_key: None,
) -> None:
generic_create_vm_test(api, flake_with_vm_with_secrets, "vm_with_secrets")
generic_create_vm_test(api, flake_with_vm_with_secrets.path, "vm_with_secrets")
@pytest.mark.skipif(not os.path.exists("/dev/kvm"), reason="Requires KVM")
@@ -102,8 +107,8 @@ def test_create_local(
def test_create_remote(
api: TestClient,
monkeypatch: pytest.MonkeyPatch,
remote_flake_with_vm_without_secrets: Path,
remote_flake_with_vm_without_secrets: FlakeForTest,
) -> None:
generic_create_vm_test(
api, remote_flake_with_vm_without_secrets, "vm_without_secrets"
api, remote_flake_with_vm_without_secrets.path, "vm_without_secrets"
)