Luis/nextjs-python-web-template
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Befor fixing linting problem

Browse Source
This commit is contained in:
Qubasa
2023-10-22 21:03:06 +02:00
parent 545d389df0
commit c7c47b6527
87 changed files with 703 additions and 3929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
pkgs/clan-cli/clan_cli/__init__.py
View File

@@ -1,11 +1,15 @@
import argparse
import logging
import sys
from types import ModuleType
from typing import Optional
from . import config, flakes, join, machines, secrets, vms, webui
from .custom_logger import register
from .ssh import cli as ssh_cli
log = logging.getLogger(__name__)
argcomplete: Optional[ModuleType] = None
try:
import argcomplete # type: ignore[no-redef]
@@ -52,6 +56,10 @@ def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
parser_vms = subparsers.add_parser("vms", help="manage virtual machines")
vms.register_parser(parser_vms)
# if args.debug:
register(logging.DEBUG)
log.debug("Debug log activated")
if argcomplete:
argcomplete.autocomplete(parser)

18
pkgs/clan-cli/clan_cli/config/__init__.py
View File

@@ -11,9 +11,9 @@ from typing import Any, Optional, Tuple, get_origin
from clan_cli.dirs import machine_settings_file, specific_flake_dir
from clan_cli.errors import ClanError
from clan_cli.flakes.types import FlakeName
from clan_cli.git import commit_file
from clan_cli.nix import nix_eval
from clan_cli.types import FlakeName
script_dir = Path(__file__).parent
@@ -161,7 +161,11 @@ def read_machine_option_value(
def get_or_set_option(args: argparse.Namespace) -> None:
if args.value == []:
print(read_machine_option_value(args.machine, args.option, args.show_trace))
print(
read_machine_option_value(
args.flake, args.machine, args.option, args.show_trace
)
)
else:
# load options
if args.options_file is None:
@@ -308,11 +312,6 @@ def register_parser(
# inject callback function to process the input later
parser.set_defaults(func=get_or_set_option)
parser.add_argument(
"flake",
type=str,
help="name of the flake to set machine options for",
)
parser.add_argument(
"--machine",
"-m",
@@ -356,6 +355,11 @@ def register_parser(
nargs="*",
help="option value to set (if omitted, the current value is printed)",
)
parser.add_argument(
"flake",
type=str,
help="name of the flake to set machine options for",
)
def main(argv: Optional[list[str]] = None) -> None:

2
pkgs/clan-cli/clan_cli/config/machine.py
View File

@@ -14,7 +14,7 @@ from clan_cli.dirs import (
from clan_cli.git import commit_file, find_git_repo_root
from clan_cli.nix import nix_eval
from ..flakes.types import FlakeName
from ..types import FlakeName
def config_for_machine(flake_name: FlakeName, machine_name: str) -> dict:

57
pkgs/clan-cli/clan_cli/custom_logger.py
View File

@@ -1,5 +1,7 @@
import inspect
import logging
from typing import Any
from pathlib import Path
from typing import Any, Callable
grey = "\x1b[38;20m"
yellow = "\x1b[33;20m"
@@ -9,11 +11,20 @@ green = "\u001b[32m"
blue = "\u001b[34m"
def get_formatter(color: str) -> logging.Formatter:
reset = "\x1b[0m"
return logging.Formatter(
f"{color}%(levelname)s{reset}:(%(filename)s:%(lineno)d): %(message)s"
)
def get_formatter(color: str) -> Callable[[logging.LogRecord, bool], logging.Formatter]:
def myformatter(
record: logging.LogRecord, with_location: bool
) -> logging.Formatter:
reset = "\x1b[0m"
filepath = Path(record.pathname).resolve()
if not with_location:
return logging.Formatter(f"{color}%(levelname)s{reset}: %(message)s")
return logging.Formatter(
f"{color}%(levelname)s{reset}: %(message)s\n {filepath}:%(lineno)d::%(funcName)s\n"
)
return myformatter
FORMATTER = {
@@ -26,12 +37,34 @@ FORMATTER = {
class CustomFormatter(logging.Formatter):
def format(self, record: Any) -> str:
return FORMATTER[record.levelno].format(record)
def format(self, record: logging.LogRecord) -> str:
return FORMATTER[record.levelno](record, True).format(record)
class ThreadFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
return FORMATTER[record.levelno](record, False).format(record)
def get_caller() -> str:
frame = inspect.currentframe()
if frame is None:
return "unknown"
caller_frame = frame.f_back
if caller_frame is None:
return "unknown"
caller_frame = caller_frame.f_back
if caller_frame is None:
return "unknown"
frame_info = inspect.getframeinfo(caller_frame)
ret = f"{frame_info.filename}:{frame_info.lineno}::{frame_info.function}"
return ret
def register(level: Any) -> None:
ch = logging.StreamHandler()
ch.setLevel(level)
ch.setFormatter(CustomFormatter())
logging.basicConfig(level=level, handlers=[ch])
handler = logging.StreamHandler()
handler.setLevel(level)
handler.setFormatter(CustomFormatter())
logger = logging.getLogger("registerHandler")
logger.addHandler(handler)
# logging.basicConfig(level=level, handlers=[handler])

66
pkgs/clan-cli/clan_cli/debug.py Normal file
View File

@@ -0,0 +1,66 @@
from typing import Dict, Optional, Tuple, Callable, Any, Mapping, List
from pathlib import Path
import ipdb
import os
import stat
import subprocess
from .dirs import find_git_repo_root
import multiprocessing as mp
from .types import FlakeName
import logging
import sys
import shlex
import time
log = logging.getLogger(__name__)
def command_exec(cmd: List[str], work_dir:Path, env: Dict[str, str]) -> None:
subprocess.run(cmd, check=True, env=env, cwd=work_dir.resolve())
def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: Optional[List[str]] = None) -> None:
if env is None:
env = os.environ.copy()
else:
env = env.copy()
# Error checking
if "bash" in env["SHELL"]:
raise Exception("I assumed you use zsh, not bash")
# Cmd appending
args = ["xterm", "-e", "zsh", "-df"]
if cmd is not None:
mycommand = shlex.join(cmd)
write_command(mycommand, work_dir / "cmd.sh")
print(f"Adding to zsh history the command: {mycommand}", file=sys.stderr)
proc = spawn_process(func=command_exec, cmd=args, work_dir=work_dir, env=env)
try:
ipdb.set_trace()
finally:
proc.terminate()
def write_command(command: str, loc:Path) -> None:
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
f.write(command)
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IEXEC)
def spawn_process(func: Callable, **kwargs:Any) -> mp.Process:
mp.set_start_method(method="spawn")
proc = mp.Process(target=func, kwargs=kwargs)
proc.start()
return proc
def dump_env(env: Dict[str, str], loc: Path) -> None:
cenv = env.copy()
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
for k, v in cenv.items():
if v.count('\n') > 0 or v.count("\"") > 0 or v.count("'") > 0:
continue
f.write(f"export {k}='{v}'\n")
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IEXEC)

16
pkgs/clan-cli/clan_cli/dirs.py
View File

@@ -1,10 +1,13 @@
import logging
import os
import sys
from pathlib import Path
from typing import Optional
from .errors import ClanError
from .flakes.types import FlakeName
from .types import FlakeName
log = logging.getLogger(__name__)
def _get_clan_flake_toplevel() -> Path:
@@ -51,28 +54,31 @@ def user_data_dir() -> Path:
def clan_data_dir() -> Path:
path = user_data_dir() / "clan"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def clan_config_dir() -> Path:
path = user_config_dir() / "clan"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def clan_flakes_dir() -> Path:
path = clan_data_dir() / "flake"
if not path.exists():
path.mkdir()
log.debug(f"Creating path with parents {path}")
path.mkdir(parents=True)
return path.resolve()
def specific_flake_dir(flake_name: FlakeName) -> Path:
flake_dir = clan_flakes_dir() / flake_name
if not flake_dir.exists():
raise ClanError(f"Flake {flake_name} does not exist")
raise ClanError(f"Flake '{flake_name}' does not exist")
return flake_dir

26
pkgs/clan-cli/clan_cli/flakes/create.py
View File

@@ -8,16 +8,20 @@ from pydantic.tools import parse_obj_as
from ..async_cmd import CmdOut, run, runforcli
from ..dirs import clan_flakes_dir
from ..errors import ClanError
from ..nix import nix_command, nix_shell
DEFAULT_URL: AnyUrl = parse_obj_as(
AnyUrl, "git+https://git.clan.lol/clan/clan-core#new-clan"
AnyUrl,
"git+https://git.clan.lol/clan/clan-core?ref=Qubasa-main#new-clan", # TODO: Change me back to main branch
)
async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
if not directory.exists():
directory.mkdir()
else:
raise ClanError(f"Flake at '{directory}' already exists")
response = {}
command = nix_command(
[
@@ -27,27 +31,27 @@ async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
url,
]
)
out = await run(command, directory)
out = await run(command, cwd=directory)
response["flake init"] = out
command = nix_shell(["git"], ["git", "init"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git init"] = out
command = nix_shell(["git"], ["git", "add", "."])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git add"] = out
command = nix_shell(["git"], ["git", "config", "user.name", "clan-tool"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git config"] = out
command = nix_shell(["git"], ["git", "config", "user.email", "clan@example.com"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git config"] = out
command = nix_shell(["git"], ["git", "commit", "-a", "-m", "Initial commit"])
out = await run(command, directory)
out = await run(command, cwd=directory)
response["git commit"] = out
return response
@@ -55,7 +59,7 @@ async def create_flake(directory: Path, url: AnyUrl) -> Dict[str, CmdOut]:
def create_flake_command(args: argparse.Namespace) -> None:
flake_dir = clan_flakes_dir() / args.name
runforcli(create_flake, flake_dir, DEFAULT_URL)
runforcli(create_flake, flake_dir, args.url)
# takes a (sub)parser and configures it
@@ -65,5 +69,11 @@ def register_create_parser(parser: argparse.ArgumentParser) -> None:
type=str,
help="name for the flake",
)
parser.add_argument(
"--url",
type=str,
help="url for the flake",
default=DEFAULT_URL,
)
# parser.add_argument("name", type=str, help="name of the flake")
parser.set_defaults(func=create_flake_command)

3
pkgs/clan-cli/clan_cli/flakes/types.py
View File

@@ -1,3 +0,0 @@
from typing import NewType
FlakeName = NewType("FlakeName", str)

4
pkgs/clan-cli/clan_cli/machines/create.py
View File

@@ -5,14 +5,16 @@ from typing import Dict
from ..async_cmd import CmdOut, run, runforcli
from ..dirs import specific_flake_dir, specific_machine_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..nix import nix_shell
from ..types import FlakeName
log = logging.getLogger(__name__)
async def create_machine(flake_name: FlakeName, machine_name: str) -> Dict[str, CmdOut]:
folder = specific_machine_dir(flake_name, machine_name)
if folder.exists():
raise ClanError(f"Machine '{machine_name}' already exists")
folder.mkdir(parents=True, exist_ok=True)
# create empty settings.json file inside the folder

2
pkgs/clan-cli/clan_cli/machines/facts.py
View File

@@ -1,5 +1,5 @@
from ..dirs import specific_machine_dir
from ..flakes.types import FlakeName
from ..types import FlakeName
def machine_has_fact(flake_name: FlakeName, machine: str, fact: str) -> bool:

2
pkgs/clan-cli/clan_cli/machines/list.py
View File

@@ -3,7 +3,7 @@ import logging
import os
from ..dirs import machines_dir
from ..flakes.types import FlakeName
from ..types import FlakeName
from .types import validate_hostname
log = logging.getLogger(__name__)

2
pkgs/clan-cli/clan_cli/secrets/folders.py
View File

@@ -5,7 +5,7 @@ from typing import Callable
from ..dirs import specific_flake_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
def get_sops_folder(flake_name: FlakeName) -> Path:

45
pkgs/clan-cli/clan_cli/secrets/groups.py
View File

@@ -3,8 +3,8 @@ import os
from pathlib import Path
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..machines.types import machine_name_type, validate_hostname
from ..types import FlakeName
from . import secrets
from .folders import (
sops_groups_folder,
@@ -204,9 +204,17 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
help="the command to run",
required=True,
)
# List groups
list_parser = subparser.add_parser("list", help="list groups")
list_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
list_parser.set_defaults(func=list_command)
# Add user
add_machine_parser = subparser.add_parser(
"add-machine", help="add a machine to group"
)
@@ -214,8 +222,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
add_machine_parser.add_argument(
"machine", help="the name of the machines to add", type=machine_name_type
)
add_machine_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_machine_parser.set_defaults(func=add_machine_command)
# Remove machine
remove_machine_parser = subparser.add_parser(
"remove-machine", help="remove a machine from group"
)
@@ -223,15 +237,27 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_machine_parser.add_argument(
"machine", help="the name of the machines to remove", type=machine_name_type
)
remove_machine_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_machine_parser.set_defaults(func=remove_machine_command)
# Add user
add_user_parser = subparser.add_parser("add-user", help="add a user to group")
add_group_argument(add_user_parser)
add_user_parser.add_argument(
"user", help="the name of the user to add", type=user_name_type
)
add_user_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_user_parser.set_defaults(func=add_user_command)
# Remove user
remove_user_parser = subparser.add_parser(
"remove-user", help="remove a user from group"
)
@@ -239,8 +265,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_user_parser.add_argument(
"user", help="the name of the user to remove", type=user_name_type
)
remove_user_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_user_parser.set_defaults(func=remove_user_command)
# Add secret
add_secret_parser = subparser.add_parser(
"add-secret", help="allow a user to access a secret"
)
@@ -250,8 +282,14 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
# Remove secret
remove_secret_parser = subparser.add_parser(
"remove-secret", help="remove a group's access to a secret"
)
@@ -261,4 +299,9 @@ def register_groups_parser(parser: argparse.ArgumentParser) -> None:
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

38
pkgs/clan-cli/clan_cli/secrets/machines.py
View File

@@ -1,7 +1,7 @@
import argparse
from ..flakes.types import FlakeName
from ..machines.types import machine_name_type, validate_hostname
from ..types import FlakeName
from . import secrets
from .folders import list_objects, remove_object, sops_machines_folder
from .sops import read_key, write_key
@@ -96,11 +96,6 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
action="store_true",
default=False,
)
add_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
@@ -109,6 +104,11 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
help="public key or private key of the user",
type=public_or_private_age_key_type,
)
add_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_parser.set_defaults(func=add_command)
# Parser
@@ -125,46 +125,46 @@ def register_machines_parser(parser: argparse.ArgumentParser) -> None:
# Parser
remove_parser = subparser.add_parser("remove", help="remove a machine")
remove_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
remove_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
remove_parser.set_defaults(func=remove_command)
# Parser
add_secret_parser = subparser.add_parser(
"add-secret", help="allow a machine to access a secret"
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.add_argument(
"machine", help="the name of the machine", type=machine_name_type
)
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
# Parser
remove_secret_parser = subparser.add_parser(
"remove-secret", help="remove a group's access to a secret"
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.add_argument(
"machine", help="the name of the group", type=machine_name_type
)
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

29
pkgs/clan-cli/clan_cli/secrets/secrets.py
View File

@@ -8,7 +8,7 @@ from typing import IO
from .. import tty
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
from .folders import (
list_objects,
sops_groups_folder,
@@ -253,24 +253,24 @@ def rename_command(args: argparse.Namespace) -> None:
def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
parser_list = subparser.add_parser("list", help="list secrets")
parser_list.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_list.set_defaults(func=list_command)
parser_get = subparser.add_parser("get", help="get a secret")
add_secret_argument(parser_get)
parser_get.set_defaults(func=get_command)
parser_get.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_get.set_defaults(func=get_command)
parser_set = subparser.add_parser("set", help="set a secret")
add_secret_argument(parser_set)
parser_set.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_set.add_argument(
"--group",
type=str,
@@ -299,13 +299,28 @@ def register_secrets_parser(subparser: argparse._SubParsersAction) -> None:
default=False,
help="edit the secret with $EDITOR instead of pasting it",
)
parser_set.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_set.set_defaults(func=set_command)
parser_rename = subparser.add_parser("rename", help="rename a secret")
add_secret_argument(parser_rename)
parser_rename.add_argument("new_name", type=str, help="the new name of the secret")
parser_rename.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_rename.set_defaults(func=rename_command)
parser_remove = subparser.add_parser("remove", help="remove a secret")
add_secret_argument(parser_remove)
parser_remove.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
parser_remove.set_defaults(func=remove_command)

2
pkgs/clan-cli/clan_cli/secrets/sops.py
View File

@@ -9,8 +9,8 @@ from typing import IO, Iterator
from ..dirs import user_config_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..nix import nix_shell
from ..types import FlakeName
from .folders import sops_machines_folder, sops_users_folder

6
pkgs/clan-cli/clan_cli/secrets/sops_generate.py
View File

@@ -6,17 +6,19 @@ import sys
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Any
import logging
from clan_cli.nix import nix_shell
from ..dirs import specific_flake_dir
from ..errors import ClanError
from ..flakes.types import FlakeName
from ..types import FlakeName
from .folders import sops_secrets_folder
from .machines import add_machine, has_machine
from .secrets import decrypt_secret, encrypt_secret, has_secret
from .sops import generate_private_key
log = logging.getLogger(__name__)
def generate_host_key(flake_name: FlakeName, machine_name: str) -> None:
if has_machine(flake_name, machine_name):
@@ -95,6 +97,7 @@ def generate_secrets_from_nix(
) -> None:
generate_host_key(flake_name, machine_name)
errors = {}
log.debug("Generating secrets for machine %s and flake %s", machine_name, flake_name)
with TemporaryDirectory() as d:
# if any of the secrets are missing, we regenerate all connected facts/secrets
for secret_group, secret_options in secret_submodules.items():
@@ -116,6 +119,7 @@ def upload_age_key_from_nix(
flake_name: FlakeName,
machine_name: str,
) -> None:
log.debug("Uploading secrets for machine %s and flake %s", machine_name, flake_name)
secret_name = f"{machine_name}-age.key"
if not has_secret(
flake_name, secret_name

12
pkgs/clan-cli/clan_cli/secrets/users.py
View File

@@ -1,6 +1,6 @@
import argparse
from ..flakes.types import FlakeName
from ..types import FlakeName
from . import secrets
from .folders import list_objects, remove_object, sops_users_folder
from .sops import read_key, write_key
@@ -131,6 +131,11 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
add_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
add_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
add_secret_parser.set_defaults(func=add_secret_command)
remove_secret_parser = subparser.add_parser(
@@ -142,4 +147,9 @@ def register_users_parser(parser: argparse.ArgumentParser) -> None:
remove_secret_parser.add_argument(
"secret", help="the name of the secret", type=secret_name_type
)
remove_secret_parser.add_argument(
"flake",
type=str,
help="name of the flake to create machine for",
)
remove_secret_parser.set_defaults(func=remove_secret_command)

22
pkgs/clan-cli/clan_cli/task_manager.py
View File

@@ -12,6 +12,7 @@ from pathlib import Path
from typing import Any, Iterator, Optional, Type, TypeVar
from uuid import UUID, uuid4
from .custom_logger import ThreadFormatter, get_caller
from .errors import ClanError
@@ -38,7 +39,8 @@ class Command:
cwd: Optional[Path] = None,
) -> None:
self.running = True
self.log.debug(f"Running command: {shlex.join(cmd)}")
self.log.debug(f"Command: {shlex.join(cmd)}")
self.log.debug(f"Caller: {get_caller()}")
cwd_res = None
if cwd is not None:
@@ -68,10 +70,10 @@ class Command:
try:
for line in fd:
if fd == self.p.stderr:
print(f"[{cmd[0]}] stderr: {line}")
self.log.debug(f"[{cmd[0]}] stderr: {line}")
self.stderr.append(line)
else:
print(f"[{cmd[0]}] stdout: {line}")
self.log.debug(f"[{cmd[0]}] stdout: {line}")
self.stdout.append(line)
self._output.put(line)
except BlockingIOError:
@@ -80,8 +82,6 @@ class Command:
if self.p.returncode != 0:
raise ClanError(f"Failed to run command: {shlex.join(cmd)}")
self.log.debug("Successfully ran command")
class TaskStatus(str, Enum):
NOTSTARTED = "NOTSTARTED"
@@ -94,7 +94,13 @@ class BaseTask:
def __init__(self, uuid: UUID, num_cmds: int) -> None:
# constructor
self.uuid: UUID = uuid
self.log = logging.getLogger(__name__)
handler = logging.StreamHandler()
handler.setLevel(logging.DEBUG)
handler.setFormatter(ThreadFormatter())
logger = logging.getLogger(__name__)
logger.addHandler(handler)
self.log = logger
self.log = logger
self.procs: list[Command] = []
self.status = TaskStatus.NOTSTARTED
self.logs_lock = threading.Lock()
@@ -108,6 +114,10 @@ class BaseTask:
self.status = TaskStatus.RUNNING
try:
self.run()
# TODO: We need to check, if too many commands have been initialized,
# but not run. This would deadlock the log_lines() function.
# Idea: Run next(cmds) and check if it raises StopIteration if not,
# we have too many commands
except Exception as e:
# FIXME: fix exception handling here
traceback.print_exception(*sys.exc_info())

23
pkgs/clan-cli/clan_cli/types.py Normal file
View File

@@ -0,0 +1,23 @@
import logging
from pathlib import Path
from typing import NewType
log = logging.getLogger(__name__)
FlakeName = NewType("FlakeName", str)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path

192
pkgs/clan-cli/clan_cli/vms/create.py
View File

@@ -4,20 +4,22 @@ import json
import os
import shlex
import sys
import tempfile
from pathlib import Path
from typing import Iterator
from typing import Iterator, Dict
from uuid import UUID
from ..dirs import specific_flake_dir
from ..nix import nix_build, nix_config, nix_shell
from ..dirs import clan_flakes_dir, specific_flake_dir
from ..nix import nix_build, nix_config, nix_eval, nix_shell
from ..task_manager import BaseTask, Command, create_task
from ..types import validate_path
from .inspect import VmConfig, inspect_vm
from ..errors import ClanError
from ..debug import repro_env_break
class BuildVmTask(BaseTask):
def __init__(self, uuid: UUID, vm: VmConfig) -> None:
super().__init__(uuid, num_cmds=6)
super().__init__(uuid, num_cmds=7)
self.vm = vm
def get_vm_create_info(self, cmds: Iterator[Command]) -> dict:
@@ -34,11 +36,18 @@ class BuildVmTask(BaseTask):
]
)
)
vm_json = "".join(cmd.stdout)
vm_json = "".join(cmd.stdout).strip()
self.log.debug(f"VM JSON path: {vm_json}")
with open(vm_json.strip()) as f:
with open(vm_json) as f:
return json.load(f)
def get_clan_name(self, cmds: Iterator[Command]) -> str:
clan_dir = self.vm.flake_url
cmd = next(cmds)
cmd.run(nix_eval([f"{clan_dir}#clanInternals.clanName"]))
clan_name = cmd.stdout[0].strip().strip('"')
return clan_name
def run(self) -> None:
cmds = self.commands()
@@ -47,99 +56,106 @@ class BuildVmTask(BaseTask):
# TODO: We should get this from the vm argument
vm_config = self.get_vm_create_info(cmds)
clan_name = self.get_clan_name(cmds)
with tempfile.TemporaryDirectory() as tmpdir_:
tmpdir = Path(tmpdir_)
xchg_dir = tmpdir / "xchg"
xchg_dir.mkdir()
secrets_dir = tmpdir / "secrets"
secrets_dir.mkdir()
disk_img = f"{tmpdir_}/disk.img"
self.log.debug(f"Building VM for clan name: {clan_name}")
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
flake_dir = clan_flakes_dir() / clan_name
validate_path(clan_flakes_dir(), flake_dir)
flake_dir.mkdir(exist_ok=True)
cmd = next(cmds)
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["generateSecrets"]],
env=env,
)
else:
cmd.run(["echo", "won't generate secrets for non local clan"])
xchg_dir = flake_dir / "xchg"
xchg_dir.mkdir()
secrets_dir = flake_dir / "secrets"
secrets_dir.mkdir()
disk_img = f"{flake_dir}/disk.img"
cmd = next(cmds)
env = os.environ.copy()
env["CLAN_DIR"] = str(self.vm.flake_url)
env["PYTHONPATH"] = str(
":".join(sys.path)
) # TODO do this in the clanCore module
env["SECRETS_DIR"] = str(secrets_dir)
cmd = next(cmds)
repro_env_break(work_dir=flake_dir, env=env, cmd=[vm_config["generateSecrets"], clan_name])
if Path(self.vm.flake_url).is_dir():
cmd.run(
[vm_config["uploadSecrets"]],
[vm_config["generateSecrets"], clan_name],
env=env,
)
else:
self.log.warning("won't generate secrets for non local clan")
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
cmd = next(cmds)
cmd.run(
[vm_config["uploadSecrets"]],
env=env,
)
cmd = next(cmds)
cmd.run(
nix_shell(
["qemu"],
[
"qemu-img",
"create",
"-f",
"raw",
disk_img,
"1024M",
],
)
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
cmd = next(cmds)
cmd.run(
nix_shell(
["e2fsprogs"],
[
"mkfs.ext4",
"-L",
"nixos",
disk_img,
],
)
)
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
cmd = next(cmds)
cmdline = [
(Path(vm_config["toplevel"]) / "kernel-params").read_text(),
f'init={vm_config["toplevel"]}/init',
f'regInfo={vm_config["regInfo"]}/registration',
"console=ttyS0,115200n8",
"console=tty0",
]
qemu_command = [
# fmt: off
"qemu-kvm",
"-name", machine,
"-m", f'{vm_config["memorySize"]}M',
"-smp", str(vm_config["cores"]),
"-device", "virtio-rng-pci",
"-net", "nic,netdev=user.0,model=virtio", "-netdev", "user,id=user.0",
"-virtfs", "local,path=/nix/store,security_model=none,mount_tag=nix-store",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=shared",
"-virtfs", f"local,path={xchg_dir},security_model=none,mount_tag=xchg",
"-virtfs", f"local,path={secrets_dir},security_model=none,mount_tag=secrets",
"-drive", f'cache=writeback,file={disk_img},format=raw,id=drive1,if=none,index=1,werror=report',
"-device", "virtio-blk-pci,bootindex=1,drive=drive1,serial=root",
"-device", "virtio-keyboard",
"-usb",
"-device", "usb-tablet,bus=usb-bus.0",
"-kernel", f'{vm_config["toplevel"]}/kernel',
"-initrd", vm_config["initrd"],
"-append", " ".join(cmdline),
# fmt: on
]
if not self.vm.graphics:
qemu_command.append("-nographic")
print("$ " + shlex.join(qemu_command))
cmd.run(nix_shell(["qemu"], qemu_command))
def create_vm(vm: VmConfig) -> BuildVmTask:

16
pkgs/clan-cli/clan_cli/webui/api_inputs.py
View File

@@ -6,25 +6,11 @@ from pydantic import AnyUrl, BaseModel, validator
from ..dirs import clan_data_dir, clan_flakes_dir
from ..flakes.create import DEFAULT_URL
from ..types import validate_path
log = logging.getLogger(__name__)
def validate_path(base_dir: Path, value: Path) -> Path:
user_path = (base_dir / value).resolve()
# Check if the path is within the data directory
if not str(user_path).startswith(str(base_dir)):
if not str(user_path).startswith("/tmp/pytest"):
raise ValueError(
f"Destination out of bounds. Expected {user_path} to start with {base_dir}"
)
else:
log.warning(
f"Detected pytest tmpdir. Skipping path validation for {user_path}"
)
return user_path
class ClanDataPath(BaseModel):
dest: Path

2
pkgs/clan-cli/clan_cli/webui/routers/machines.py
View File

@@ -9,9 +9,9 @@ from ...config.machine import (
schema_for_machine,
set_config_for_machine,
)
from ...flakes.types import FlakeName
from ...machines.create import create_machine as _create_machine
from ...machines.list import list_machines as _list_machines
from ...types import FlakeName
from ..api_outputs import (
ConfigResponse,
Machine,