diff --git a/pkgs/clan-cli/.vscode/launch.json b/pkgs/clan-cli/.vscode/launch.json index 8dc9244..a15810a 100644 --- a/pkgs/clan-cli/.vscode/launch.json +++ b/pkgs/clan-cli/.vscode/launch.json @@ -9,8 +9,8 @@ "type": "python", "request": "launch", "module": "clan_cli.webui", - "justMyCode": true, - "args": [ "--reload", "--no-open", "--log-level", "debug" ] + "justMyCode": false, + "args": [ "--reload", "--no-open", "--log-level", "debug" ], } ] } \ No newline at end of file diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index ca831a6..3358327 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -29,29 +29,31 @@ To start a local developement environment instead, use the `--dev` flag: This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly. ## Run webui directly + Useful for vscode run and debug option + ```bash python -m clan_cli.webui --reload --no-open ``` Add this `launch.json` to your .vscode directory to have working breakpoints in your vscode editor. + ```json { - "version": "0.2.0", - "configurations": [ - { - "name": "Clan Webui", - "type": "python", - "request": "launch", - "module": "clan_cli.webui", - "justMyCode": true, - "args": [ "--reload", "--no-open", "--log-level", "debug" ] - } - ] + "version": "0.2.0", + "configurations": [ + { + "name": "Clan Webui", + "type": "python", + "request": "launch", + "module": "clan_cli.webui", + "justMyCode": true, + "args": ["--reload", "--no-open", "--log-level", "debug"] + } + ] } ``` - ## Run locally single-threaded for debugging By default tests run in parallel using pytest-parallel. diff --git a/pkgs/clan-cli/clan_cli/custom_logger.py b/pkgs/clan-cli/clan_cli/custom_logger.py index 52d3f1a..8566b95 100644 --- a/pkgs/clan-cli/clan_cli/custom_logger.py +++ b/pkgs/clan-cli/clan_cli/custom_logger.py @@ -1,5 +1,6 @@ -import logging import datetime +import logging + class CustomFormatter(logging.Formatter): diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index f141806..0c03bcc 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -42,5 +42,9 @@ def setup_app() -> FastAPI: #TODO: How do I get the log level from the command line in here? custom_logger.register(logging.DEBUG) app = setup_app() + +for i in app.exception_handlers.items(): + log.info(f"Registered exception handler: {i}") + log.warn("log warn") log.debug("log debug") diff --git a/pkgs/clan-cli/clan_cli/webui/routers/machines.py b/pkgs/clan-cli/clan_cli/webui/routers/machines.py index a2b7778..1a19530 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/machines.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/machines.py @@ -1,3 +1,5 @@ +# Logging setup +import logging from typing import Annotated from fastapi import APIRouter, Body @@ -19,10 +21,7 @@ from ..schemas import ( Status, ) -# Logging setup -import logging log = logging.getLogger(__name__) - router = APIRouter() @@ -42,7 +41,7 @@ async def create_machine(machine: Annotated[MachineCreate, Body()]) -> MachineRe @router.get("/api/machines/{name}") async def get_machine(name: str) -> MachineResponse: - print("TODO") + log.error("TODO") return MachineResponse(machine=Machine(name=name, status=Status.UNKNOWN)) diff --git a/pkgs/clan-cli/clan_cli/webui/routers/vms.py b/pkgs/clan-cli/clan_cli/webui/routers/vms.py index 95688b2..9ed4579 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/vms.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/vms.py @@ -1,45 +1,27 @@ import asyncio import json -import shlex -from typing import Annotated, AsyncIterator import logging +import os +import shlex +import uuid +from typing import Annotated, AsyncIterator -from fastapi import APIRouter, Body, HTTPException, Request, status, logger +from fastapi import APIRouter, Body, FastAPI, HTTPException, Request, status, BackgroundTasks from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse, StreamingResponse from ...nix import nix_build, nix_eval -from ..schemas import VmConfig, VmInspectResponse +from ..schemas import VmConfig, VmInspectResponse, VmCreateResponse # Logging setup log = logging.getLogger(__name__) router = APIRouter() - -class NixBuildException(HTTPException): - def __init__(self, msg: str, loc: list = ["body", "flake_attr"]): - detail = [ - { - "loc": loc, - "msg": msg, - "type": "value_error", - } - ] - super().__init__( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail - ) +app = FastAPI() -def nix_build_exception_handler( - request: Request, exc: NixBuildException -) -> JSONResponse: - return JSONResponse( - status_code=exc.status_code, - content=jsonable_encoder(dict(detail=exc.detail)), - ) - -def nix_inspect_vm(machine: str, flake_url: str) -> list[str]: +def nix_inspect_vm_cmd(machine: str, flake_url: str) -> list[str]: return nix_eval( [ f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.clan.vm.config" @@ -47,7 +29,7 @@ def nix_inspect_vm(machine: str, flake_url: str) -> list[str]: ) -def nix_build_vm(machine: str, flake_url: str) -> list[str]: +def nix_build_vm_cmd(machine: str, flake_url: str) -> list[str]: return nix_build( [ f"{flake_url}#nixosConfigurations.{json.dumps(machine)}.config.system.build.vm" @@ -55,21 +37,13 @@ def nix_build_vm(machine: str, flake_url: str) -> list[str]: ) -async def start_vm(vm_path: str) -> None: - proc = await asyncio.create_subprocess_exec( - vm_path, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - - await proc.wait() @router.post("/api/vms/inspect") async def inspect_vm( flake_url: Annotated[str, Body()], flake_attr: Annotated[str, Body()] ) -> VmInspectResponse: - cmd = nix_inspect_vm(flake_attr, flake_url=flake_url) + cmd = nix_inspect_vm_cmd(flake_attr, flake_url=flake_url) proc = await asyncio.create_subprocess_exec( cmd[0], *cmd[1:], @@ -94,40 +68,91 @@ command output: ) -async def vm_build(vm: VmConfig) -> AsyncIterator[str]: - cmd = nix_build_vm(vm.flake_attr, flake_url=vm.flake_url) - log.debug(f"Running command: {shlex.join(cmd)}") - proc = await asyncio.create_subprocess_exec( - cmd[0], - *cmd[1:], - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - assert proc.stdout is not None and proc.stderr is not None - vm_path = "" - async for line in proc.stdout: - vm_path = f'{line.decode("utf-8", "ignore").strip()}/bin/run-nixos-vm' - await start_vm(vm_path) - stderr = "" - async for line in proc.stderr: - stderr += line.decode("utf-8", "ignore") - yield line.decode("utf-8", "ignore") - res = await proc.wait() - if res != 0: - raise NixBuildException( - f""" -Failed to build vm from '{vm.flake_url}#{vm.flake_attr}'. -command: {shlex.join(cmd)} -exit code: {res} -command output: -{stderr} - """ +class NixBuildException(HTTPException): + def __init__(self, uuid: uuid.UUID, msg: str,loc: list = ["body", "flake_attr"]): + self.uuid = uuid + detail = [ + { + "loc": loc, + "uuid": str(uuid), + "msg": msg, + "type": "value_error", + } + ] + super().__init__( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=detail ) -import logging + + + +import threading +import subprocess +import uuid + + +class BuildVM(threading.Thread): + def __init__(self, vm: VmConfig, uuid: uuid.UUID): + # calling parent class constructor + threading.Thread.__init__(self) + + # constructor + self.vm: VmConfig = vm + self.uuid: uuid.UUID = uuid + self.log = logging.getLogger(__name__) + self.process: subprocess.Popen = None + + def run(self): + self.log.debug(f"BuildVM with uuid {self.uuid} started") + + cmd = nix_build_vm_cmd(self.vm.flake_attr, flake_url=self.vm.flake_url) + (out, err) = self.run_cmd(cmd) + vm_path = f'{out.strip()}/bin/run-nixos-vm' + + self.log.debug(f"vm_path: {vm_path}") + + (out, err) = self.run_cmd(vm_path) + + + def run_cmd(self, cmd: list[str]): + cwd=os.getcwd() + log.debug(f"Working directory: {cwd}") + log.debug(f"Running command: {shlex.join(cmd)}") + self.process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + cwd=cwd, + ) + + self.process.wait() + if self.process.returncode != 0: + raise NixBuildException(self.uuid, f"Failed to run command: {shlex.join(cmd)}") + + log.info("Successfully ran command") + return (self.process.stdout, self.process.stderr) + +POOL: dict[uuid.UUID, BuildVM] = {} + + +def nix_build_exception_handler( + request: Request, exc: NixBuildException +) -> JSONResponse: + log.error("NixBuildException: %s", exc) + del POOL[exc.uuid] + return JSONResponse( + status_code=exc.status_code, + content=jsonable_encoder(dict(detail=exc.detail)), + ) @router.post("/api/vms/create") -async def create_vm(vm: Annotated[VmConfig, Body()]) -> StreamingResponse: - return StreamingResponse(vm_build(vm)) +async def create_vm(vm: Annotated[VmConfig, Body()], background_tasks: BackgroundTasks) -> StreamingResponse: + handle_id = uuid.uuid4() + handle = BuildVM(vm, handle_id) + handle.start() + POOL[handle_id] = handle + return VmCreateResponse(uuid=str(handle_id)) + diff --git a/pkgs/clan-cli/clan_cli/webui/schemas.py b/pkgs/clan-cli/clan_cli/webui/schemas.py index 11a6037..8ee819c 100644 --- a/pkgs/clan-cli/clan_cli/webui/schemas.py +++ b/pkgs/clan-cli/clan_cli/webui/schemas.py @@ -43,6 +43,8 @@ class VmConfig(BaseModel): memory_size: int graphics: bool +class VmCreateResponse(BaseModel): + uuid: str class VmInspectResponse(BaseModel): config: VmConfig diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 1c052a1..800cdab 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -1,5 +1,4 @@ import argparse -import logging import subprocess import time import urllib.request @@ -7,13 +6,12 @@ import webbrowser from contextlib import ExitStack, contextmanager from pathlib import Path from threading import Thread -from typing import (Iterator, Dict, Any) +from typing import Iterator # XXX: can we dynamically load this using nix develop? from uvicorn import run - def defer_open_browser(base_url: str) -> None: for i in range(5): try: