Merge pull request 'Fully working ui and cli' (#1) from Luis-main into main
Some checks failed
checks-impure / test (push) Successful in 24s
checks / test (push) Successful in 1m24s
assets1 / test (push) Failing after 51s

Reviewed-on: Luis/consulting-website#1
This commit was merged in pull request #1.
This commit is contained in:
2023-10-23 22:37:35 +02:00
33 changed files with 74 additions and 435 deletions

View File

@@ -2,28 +2,16 @@
imports = [
./impure/flake-module.nix
];
perSystem = { pkgs, lib, self', ... }: {
perSystem = { lib, self', ... }: {
checks =
let
nixosTestArgs = {
# reference to nixpkgs for the current system
inherit pkgs;
# this gives us a reference to our flake but also all flake inputs
inherit self;
};
nixosTests = lib.optionalAttrs (pkgs.stdenv.isLinux) {
# import our test
secrets = import ./secrets nixosTestArgs;
};
schemaTests = pkgs.callPackages ./schemas.nix {
inherit self;
};
flakeOutputs = lib.mapAttrs' (name: config: lib.nameValuePair "nixos-${name}" config.config.system.build.toplevel) self.nixosConfigurations
// lib.mapAttrs' (n: lib.nameValuePair "package-${n}") self'.packages
// lib.mapAttrs' (n: lib.nameValuePair "devShell-${n}") self'.devShells
// lib.mapAttrs' (name: config: lib.nameValuePair "home-manager-${name}" config.activation-script) (self'.legacyPackages.homeConfigurations or { });
in
nixosTests // schemaTests // flakeOutputs;
flakeOutputs;
};
}

View File

@@ -1,54 +0,0 @@
{ self, lib, inputs, ... }:
let
inherit (builtins)
mapAttrs
toJSON
toFile
;
inherit (lib)
mapAttrs'
;
clanLib = self.lib;
clanModules = self.clanModules;
in
{
perSystem = { pkgs, ... }:
let
baseModule = {
imports =
(import (inputs.nixpkgs + "/nixos/modules/module-list.nix"))
++ [{
nixpkgs.hostPlatform = pkgs.system;
}];
};
optionsFromModule = module:
let
evaled = lib.evalModules {
modules = [ module baseModule ];
};
in
evaled.options.clan.networking;
clanModuleSchemas =
mapAttrs
(_: module: clanLib.jsonschema.parseOptions (optionsFromModule module))
clanModules;
mkTest = name: schema: pkgs.runCommand "schema-${name}" { } ''
${pkgs.check-jsonschema}/bin/check-jsonschema \
--check-metaschema ${toFile "schema-${name}" (toJSON schema)}
touch $out
'';
in
{
checks = mapAttrs'
(name: schema: {
name = "schema-${name}";
value = mkTest name schema;
})
clanModuleSchemas;
};
}

View File

@@ -1,34 +0,0 @@
{ self, runCommand, check-jsonschema, pkgs, lib, ... }:
let
clanModules.clanCore = self.nixosModules.clanCore;
baseModule = {
imports =
(import (pkgs.path + "/nixos/modules/module-list.nix"))
++ [{
nixpkgs.hostPlatform = "x86_64-linux";
}];
};
optionsFromModule = module:
let
evaled = lib.evalModules {
modules = [ module baseModule ];
};
in
evaled.options.clan;
clanModuleSchemas = lib.mapAttrs (_: module: self.lib.jsonschema.parseOptions (optionsFromModule module)) clanModules;
mkTest = name: schema: runCommand "schema-${name}" { } ''
${check-jsonschema}/bin/check-jsonschema \
--check-metaschema ${builtins.toFile "schema-${name}" (builtins.toJSON schema)}
touch $out
'';
in
lib.mapAttrs'
(name: schema: {
name = "schema-${name}";
value = mkTest name schema;
})
clanModuleSchemas

View File

@@ -1,6 +0,0 @@
#!/usr/bin/env bash
set -eux -o pipefail
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
export SOPS_AGE_KEY_FILE="${SCRIPT_DIR}/key.age"
nix run .# -- secrets "$@"

View File

@@ -1,21 +0,0 @@
(import ../lib/test-base.nix) {
name = "secrets";
nodes.machine = { self, config, ... }: {
imports = [
(self.nixosModules.clanCore)
];
environment.etc."secret".source = config.sops.secrets.secret.path;
environment.etc."group-secret".source = config.sops.secrets.group-secret.path;
sops.age.keyFile = ./key.age;
clanCore.clanDir = "${./.}";
clanCore.machineName = "machine";
networking.hostName = "machine";
};
testScript = ''
machine.succeed("cat /etc/secret >&2")
machine.succeed("cat /etc/group-secret >&2")
'';
}

View File

@@ -1 +0,0 @@
AGE-SECRET-KEY-1UCXEUJH6JXF8LFKWFHDM4N9AQE2CCGQZGXLUNV4TKR5KY0KC8FDQ2TY4NX

View File

@@ -1 +0,0 @@
../../../machines/machine

View File

@@ -1,4 +0,0 @@
{
"publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
"type": "age"
}

View File

@@ -1 +0,0 @@
../../../groups/group

View File

@@ -1,20 +0,0 @@
{
"data": "ENC[AES256_GCM,data:FgF3,iv:QBbnqZ6405qmwGKhbolPr9iobngXt8rtfUwCBOnmwRA=,tag:7gqI1zLVnTkZ0xrNn/LEkA==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": [
{
"recipient": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArMHcxKzhUZzNHQmQrb28x\nRC9UMlZMeDN3S1l1eHdUWmV4VUVReHhhQ0RnCjAyUXVlY1FmclVmL2lEdFZuTmll\nVENpa3AwbjlDck5zdGdHUTRnNEdEOUkKLS0tIER3ZlNMSVFnRElkRDcxajZnVmFl\nZThyYzcvYUUvaWJYUmlwQ3dsSDdjSjgK+tj34yBzrsIjm6V+T9wTgz5FdNGOR7I/\nVB4fh8meW0vi/PCK/rajC8NbqmK8qq/lwsF/JwfZKDSdG0FOJUB1AA==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2023-09-03T12:44:56Z",
"mac": "ENC[AES256_GCM,data:d5a0WfE5ZRLKF1NZkBfOl+cVI8ZZHd2rC+qX/giALjyrzk09rLxBeY4lO827GFfMmVy/oC7ceH9pjv2O7ibUiQtcbGIQVBg/WP+dVn8fRMWtF0jpv9BhYTutkVk3kiddqPGhp3mpwvls2ot5jtCRczTPk3JSxN3B1JSJCmj9GfQ=,iv:YmlkTYFNUaFRWozO8+OpEVKaSQmh+N9zpatwUNMPNyw=,tag:mEGQ4tdo82qlhKWalQuufg==,type:str]",
"pgp": null,
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}

View File

@@ -1 +0,0 @@
../../../machines/machine

View File

@@ -1,20 +0,0 @@
{
"data": "ENC[AES256_GCM,data:bhxF,iv:iNs+IfSU/7EwssZ0GVTF2raxJkVlddfQEPGIBeUYAy8=,tag:JMOKTMW3/ic3UTj9eT9YFQ==,type:str]",
"sops": {
"kms": null,
"gcp_kms": null,
"azure_kv": null,
"hc_vault": null,
"age": [
{
"recipient": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBxS0g4TEt4S09LQnFKdCtk\nZTlUQWhNUHZmcmZqdGtuZkhhTkMzZDVaWWdNCi9vNnZQeklNaFBBU2x0ditlUDR0\nNGJlRmFFb09WSUFGdEh5TGViTWtacFEKLS0tIE1OMWdQMHhGeFBwSlVEamtHUkcy\ndzI1VHRkZ1o4SStpekVNZmpQSnRkeUkKYmPS9sR6U0NHxd55DjRk29LNFINysOl6\nEM2MTrntLxOHFWZ1QgNx34l4rYIIXx97ONvR0SRpxN0ECL9VonQeZg==\n-----END AGE ENCRYPTED FILE-----\n"
}
],
"lastmodified": "2023-08-23T09:11:08Z",
"mac": "ENC[AES256_GCM,data:8z819mP4FJXE/ExWM1+/dhaXIXzCglhBuZwE6ikl/jNLUAnv3jYL9c9vPrPFl2by3wXSNzqB4AOiTKDQoxDx2SBQKxeWaUnOajD6hbzskoLqCCBfVx7qOHrk/BULcBvMSxBca4RnzXXoMFTwKs2A1fXqAPvSQd1X4gX6Xm9VXWM=,iv:3YxZX+gaEcRKDN0Kuf9y1oWL+sT/J5B/5CtCf4iur9Y=,tag:0dwyjpvjCqbm9vIrz6WSWQ==,type:str]",
"pgp": null,
"unencrypted_suffix": "_unencrypted",
"version": "3.7.3"
}
}

View File

@@ -1 +0,0 @@
../../../users/admin

View File

@@ -1,4 +0,0 @@
{
"publickey": "age15x8u838dwqflr3t6csf4tlghxm4tx77y379ncqxav7y2n8qp7yzqgrwt00",
"type": "age"
}

View File

@@ -1,3 +1,4 @@
#!/usr/bin/env bash
# Because we depend on nixpkgs sources, uploading to builders takes a long time
source_up

View File

@@ -1,23 +1,27 @@
from typing import Dict, Optional, Tuple, Callable, Any, Mapping, List
from pathlib import Path
import ipdb
import logging
import multiprocessing as mp
import os
import shlex
import stat
import subprocess
from .dirs import find_git_repo_root
import multiprocessing as mp
from .types import FlakeName
import logging
import sys
import shlex
import time
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
import ipdb
log = logging.getLogger(__name__)
def command_exec(cmd: List[str], work_dir:Path, env: Dict[str, str]) -> None:
def command_exec(cmd: List[str], work_dir: Path, env: Dict[str, str]) -> None:
subprocess.run(cmd, check=True, env=env, cwd=work_dir.resolve())
def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: Optional[List[str]] = None) -> None:
def repro_env_break(
work_dir: Path,
env: Optional[Dict[str, str]] = None,
cmd: Optional[List[str]] = None,
) -> None:
if env is None:
env = os.environ.copy()
else:
@@ -40,14 +44,16 @@ def repro_env_break(work_dir: Path, env: Optional[Dict[str, str]] = None, cmd: O
finally:
proc.terminate()
def write_command(command: str, loc:Path) -> None:
def write_command(command: str, loc: Path) -> None:
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
f.write(command)
st = os.stat(loc)
os.chmod(loc, st.st_mode | stat.S_IEXEC)
def spawn_process(func: Callable, **kwargs:Any) -> mp.Process:
def spawn_process(func: Callable, **kwargs: Any) -> mp.Process:
mp.set_start_method(method="spawn")
proc = mp.Process(target=func, kwargs=kwargs)
proc.start()
@@ -59,7 +65,7 @@ def dump_env(env: Dict[str, str], loc: Path) -> None:
with open(loc, "w") as f:
f.write("#!/usr/bin/env bash\n")
for k, v in cenv.items():
if v.count('\n') > 0 or v.count("\"") > 0 or v.count("'") > 0:
if v.count("\n") > 0 or v.count('"') > 0 or v.count("'") > 0:
continue
f.write(f"export {k}='{v}'\n")
st = os.stat(loc)

View File

@@ -1,88 +0,0 @@
import shlex
import subprocess
from pathlib import Path
from typing import Optional
from clan_cli.dirs import find_git_repo_root
from clan_cli.errors import ClanError
from clan_cli.nix import nix_shell
# generic vcs agnostic commit function
def commit_file(
file_path: Path,
repo_dir: Optional[Path] = None,
commit_message: Optional[str] = None,
) -> None:
if repo_dir is None:
repo_dir = find_git_repo_root()
if repo_dir is None:
return
# check that the file is in the git repository and exists
if not Path(file_path).resolve().is_relative_to(repo_dir.resolve()):
raise ClanError(f"File {file_path} is not in the git repository {repo_dir}")
if not file_path.exists():
raise ClanError(f"File {file_path} does not exist")
# generate commit message if not provided
if commit_message is None:
# ensure that mentioned file path is relative to repo
commit_message = f"Add {file_path.relative_to(repo_dir)}"
# check if the repo is a git repo and commit
if (repo_dir / ".git").exists():
_commit_file_to_git(repo_dir, file_path, commit_message)
else:
return
def _commit_file_to_git(repo_dir: Path, file_path: Path, commit_message: str) -> None:
"""Commit a file to a git repository.
:param repo_dir: The path to the git repository.
:param file_path: The path to the file to commit.
:param commit_message: The commit message.
:raises ClanError: If the file is not in the git repository.
"""
cmd = nix_shell(
["git"],
["git", "-C", str(repo_dir), "add", str(file_path)],
)
# add the file to the git index
try:
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as e:
raise ClanError(
f"Failed to add {file_path} to git repository {repo_dir}:\n{shlex.join(cmd)}\n exited with {e.returncode}"
) from e
# check if there is a diff
cmd = nix_shell(
["git"],
["git", "-C", str(repo_dir), "diff", "--cached", "--exit-code"],
)
result = subprocess.run(cmd, cwd=repo_dir)
# if there is no diff, return
if result.returncode == 0:
return
# commit only that file
cmd = nix_shell(
["git"],
[
"git",
"-C",
str(repo_dir),
"commit",
"-m",
commit_message,
str(file_path.relative_to(repo_dir)),
],
)
try:
subprocess.run(
cmd,
check=True,
)
except subprocess.CalledProcessError as e:
raise ClanError(
f"Failed to commit {file_path} to git repository {repo_dir}:\n{shlex.join(cmd)}\n exited with {e.returncode}"
) from e

View File

@@ -1,25 +0,0 @@
import sys
from typing import IO, Any, Callable
def is_interactive() -> bool:
"""Returns true if the current process is interactive"""
return sys.stdin.isatty() and sys.stdout.isatty()
def color_text(code: int, file: IO[Any] = sys.stdout) -> Callable[[str], None]:
"""
Print with color if stderr is a tty
"""
def wrapper(text: str) -> None:
if file.isatty():
print(f"\x1b[{code}m{text}\x1b[0m", file=file)
else:
print(text, file=file)
return wrapper
warn = color_text(91, file=sys.stderr)
info = color_text(92, file=sys.stderr)

View File

@@ -3,11 +3,13 @@ from pathlib import Path
from typing import Any
from pydantic import AnyUrl, BaseModel, validator
from pydantic.tools import parse_obj_as
from ..dirs import clan_data_dir, clan_flakes_dir
from ..flakes.create import DEFAULT_URL
from ..types import validate_path
DEFAULT_URL = parse_obj_as(AnyUrl, "http://localhost:8000")
log = logging.getLogger(__name__)

View File

@@ -1,11 +1,6 @@
from enum import Enum
from typing import Dict, List
from pydantic import BaseModel, Field
from ..async_cmd import CmdOut
from ..task_manager import TaskStatus
from ..vms.inspect import VmConfig
from pydantic import BaseModel
class Status(Enum):
@@ -17,54 +12,3 @@ class Status(Enum):
class Machine(BaseModel):
name: str
status: Status
class MachineCreate(BaseModel):
name: str
class MachinesResponse(BaseModel):
machines: list[Machine]
class MachineResponse(BaseModel):
machine: Machine
class ConfigResponse(BaseModel):
config: dict
class SchemaResponse(BaseModel):
schema_: dict = Field(alias="schema")
class VmStatusResponse(BaseModel):
error: str | None
status: TaskStatus
class VmCreateResponse(BaseModel):
uuid: str
class FlakeAttrResponse(BaseModel):
flake_attrs: list[str]
class VmInspectResponse(BaseModel):
config: VmConfig
class FlakeAction(BaseModel):
id: str
uri: str
class FlakeCreateResponse(BaseModel):
cmd_out: Dict[str, CmdOut]
class FlakeResponse(BaseModel):
content: str
actions: List[FlakeAction]

View File

@@ -29,7 +29,6 @@ def setup_app() -> FastAPI:
app.include_router(health.router)
# Needs to be last in register. Because of wildcard route
app.include_router(root.router)
app.add_exception_handler(ClanError, clan_error_handler)

View File

@@ -13,6 +13,7 @@ from typing import Iterator
import uvicorn
from pydantic import AnyUrl, IPvAnyAddress
from pydantic.tools import parse_obj_as
from clan_cli.errors import ClanError
log = logging.getLogger(__name__)
@@ -25,9 +26,7 @@ def open_browser(base_url: AnyUrl, sub_url: str) -> None:
break
except OSError:
time.sleep(i)
url = parse_obj_as(
AnyUrl, f"{base_url}/{sub_url.removeprefix('/')}"
)
url = parse_obj_as(AnyUrl, f"{base_url}/{sub_url.removeprefix('/')}")
_open_browser(url)

View File

@@ -37,6 +37,10 @@ exclude = "clan_cli.nixpkgs"
module = "argcomplete.*"
ignore_missing_imports = true
[[tool.mypy.overrides]]
module = "ipdb.*"
ignore_missing_imports = true
[[tool.mypy.overrides]]
module = "jsonschema.*"
ignore_missing_imports = true
@@ -52,7 +56,7 @@ ignore_missing_imports = true
[tool.ruff]
line-length = 88
select = [ "E", "F", "I", "U", "N"]
select = [ "E", "F", "I", "N"]
ignore = [ "E501" ]
[tool.black]

View File

@@ -22,37 +22,37 @@ mkShell {
];
shellHook = ''
tmp_path=$(realpath ./.direnv)
tmp_path=$(realpath ./.direnv)
repo_root=$(realpath .)
mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}"
repo_root=$(realpath .)
mkdir -p "$tmp_path/python/${pythonWithDeps.sitePackages}"
# Install the package in editable mode
# This allows executing `clan` from within the dev-shell using the current
# version of the code and its dependencies.
${pythonWithDeps.interpreter} -m pip install \
--quiet \
--disable-pip-version-check \
--no-index \
--no-build-isolation \
--prefix "$tmp_path/python" \
--editable $repo_root
# Install the package in editable mode
# This allows executing `clan` from within the dev-shell using the current
# version of the code and its dependencies.
${pythonWithDeps.interpreter} -m pip install \
--quiet \
--disable-pip-version-check \
--no-index \
--no-build-isolation \
--prefix "$tmp_path/python" \
--editable $repo_root
rm -f clan_cli/nixpkgs clan_cli/webui/assets
ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs
ln -sf ${ui-assets} clan_cli/webui/assets
rm -f clan_cli/nixpkgs clan_cli/webui/assets
ln -sf ${clan-cli.nixpkgs} clan_cli/nixpkgs
ln -sf ${ui-assets} clan_cli/webui/assets
export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH"
export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:"
export PYTHONBREAKPOINT=ipdb.set_trace
export PATH="$tmp_path/python/bin:${checkScript}/bin:$PATH"
export PYTHONPATH="$repo_root:$tmp_path/python/${pythonWithDeps.sitePackages}:"
export PYTHONBREAKPOINT=ipdb.set_trace
export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
mkdir -p \
$tmp_path/share/fish/vendor_completions.d \
$tmp_path/share/bash-completion/completions \
$tmp_path/share/zsh/site-functions
register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish
register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan
export XDG_DATA_DIRS="$tmp_path/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}"
export fish_complete_path="$tmp_path/share/fish/vendor_completions.d''${fish_complete_path:+:$fish_complete_path}"
mkdir -p \
$tmp_path/share/fish/vendor_completions.d \
$tmp_path/share/bash-completion/completions \
$tmp_path/share/zsh/site-functions
register-python-argcomplete --shell fish clan > $tmp_path/share/fish/vendor_completions.d/clan.fish
register-python-argcomplete --shell bash clan > $tmp_path/share/bash-completion/completions/clan
'';
}

View File

@@ -9,6 +9,11 @@ import pytest
from ports import PortFunction
@pytest.mark.impure
def test_nothing_much() -> None:
pass
@pytest.mark.timeout(10)
def test_start_server(unused_tcp_port: PortFunction, temporary_home: Path) -> None:
port = unused_tcp_port()

View File

@@ -10,6 +10,8 @@
packages = {
tea-create-pr = pkgs.callPackage ./tea-create-pr { };
#theme = pkgs.callPackage ./theme { inherit (self.inputs) floco; clanPkgs = self'.packages; };
merge-after-ci = pkgs.callPackage ./merge-after-ci {
inherit (config.packages) tea-create-pr;
};

View File

@@ -21,6 +21,6 @@ writeShellApplication {
remoteName="''${1:-origin}"
targetBranch="''${2:-main}"
shift && shift
tea-create-pr "$remoteName" "$targetBranch" --assignees clan-bot "$@"
tea-create-pr "$remoteName" "$targetBranch" --assignees consulting-bot "$@"
'';
}

View File

@@ -6,7 +6,7 @@ targetBranch="${2:-main}"
shift && shift
TMPDIR="$(mktemp -d)"
currentBranch="$(git rev-parse --abbrev-ref HEAD)"
user="$(tea login list -o simple | cut -d" " -f4)"
user="$(git config --get user.name)"
tempRemoteBranch="$user-$currentBranch"
nix fmt -- --fail-on-change

View File

@@ -8,7 +8,7 @@
{
packages = {
ui = base.pkg.global;
theme = base.pkg.theme;
ui-assets = pkgs.callPackage ./nix/ui-assets.nix { };
# EXAMPLE: GITEA_TOKEN=$(rbw get -f GITEA_TOKEN git.clan.lol) nix run .#update-ui-assets
update-ui-assets = pkgs.callPackage ./nix/update-ui-assets.nix { };

View File

@@ -18438,4 +18438,4 @@
};
};
};
}
}

View File

@@ -6,7 +6,7 @@ import {
CssBaseline,
IconButton,
ThemeProvider,
useMediaQuery
useMediaQuery,
} from "@mui/material";
import { StyledEngineProvider } from "@mui/material/styles";
import axios from "axios";

View File

@@ -1,30 +0,0 @@
"use client";
import { useGetVmLogs } from "@/api/default/default";
import { Log } from "./log";
import { LoadingOverlay } from "./loadingOverlay";
interface VmBuildLogsProps {
vmUuid: string;
}
export const VmBuildLogs = (props: VmBuildLogsProps) => {
const { vmUuid } = props;
const { data: logs, isLoading } = useGetVmLogs(vmUuid as string, {
swr: {
enabled: vmUuid !== null,
},
axios: {
responseType: "stream",
},
});
return (
<div className="w-full">
{isLoading && <LoadingOverlay title="Initializing" subtitle="" />}
<Log
lines={(logs?.data as string)?.split("\n") || ["..."]}
title="Building..."
/>
</div>
);
};