From 79dbefcfe60ef772e18ec47a898d0bf99cb5b535 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sun, 21 Jan 2024 19:30:48 +0100 Subject: [PATCH] Added docker file. Improved Documentation --- README.md | 146 +++--------------- pkgs/clan-cli/README.md | 128 +++++++++++---- pkgs/clan-cli/clan_cli/config.py | 2 +- pkgs/clan-cli/flake-module.nix | 13 ++ pkgs/ui/README.md | 32 ++-- pkgs/ui/src/app/home/page.tsx | 5 +- .../src/components/sequence_diagram/index.tsx | 8 +- 7 files changed, 164 insertions(+), 170 deletions(-) diff --git a/README.md b/README.md index d506cd8..2389b63 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Let's get your development environment up and running: 1. **Install Nix Package Manager**: - - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command: + - You can install the Nix package manager by running this command: ```bash curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install ``` @@ -34,7 +34,7 @@ sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' 2. **Install direnv**: - - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command: + - Install the direnv package by running the following command: ```bash curl -sfL https://direnv.net/install.sh | bash ``` @@ -75,11 +75,20 @@ sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' ```bash clan webui --reload --no-open --log-level debug --populate --emulate ``` - - The server will automatically restart if any Python files change. + - The server will automatically restart if any Python files change. Emulated services however will not. - The `--populate` flag will automatically populate the database with dummy data + - To look into the endpoints open up a swagger instance by visiting: http://localhost:2979/docs - The `--emulate` flag will automatically run servers the database with dummy data for the fronted to communicate with (ap, dlg, c1 and c2) + - To look into the emulated endpoints go to http://localhost:2979/emulate -8. **Build the Frontend**: +8. **Detailed Backend Documentation** + + - For detailed backend documentation go to [pkgs/clan-cli/README.md](pkgs/clan-cli/README.md) + - We explain: + - How to build and run a docker image + - Internal workings of the App + +9. **Build the Frontend**: - In a different shell, navigate to the `pkgs/ui` directory and execute: ```bash @@ -87,31 +96,20 @@ sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' ``` - Wait for the frontend to build. -9. **Start the Frontend**: - - To start the frontend, execute: - ```bash - npm run dev - ``` - - Access the website by going to [http://localhost:3000](http://localhost:3000). +10. **Start the Frontend**: + +- To start the frontend, execute: + ```bash + npm run dev + ``` +- Access the website by going to [http://localhost:3000](http://localhost:3000). + +11. **Detailed Frontend Documentation** + +- For detailed frontend documentation go to [pkgs/ui/README.md](pkgs/ui/README.md) # Setting Up Your Git Workflow -Let's set up your Git workflow to collaborate effectively: - -1. **Register Your Gitea Account Locally**: - - - Execute the following command to add your Gitea account locally: - ```bash - tea login add - ``` - - Go to https://gitea.gchq.icu/user/settings/applications and create token with all privileges - - Fill out the prompt as follows: - - URL of Gitea instance: `https://gitea.gchq.icu` - - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171` - - Do you have an access token? Yes - - Token: \***\*\*\*\*** - - Set Optional settings: No - 2. **Git Workflow**: 1. Add your changes to Git using `git add `. @@ -124,99 +122,3 @@ Let's set up your Git workflow to collaborate effectively: 5. Use `git status` to check for merge conflicts. 6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts). 7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts. - -3. **Create a Pull Request**: - - - To automatically open a pull request that gets merged if all tests pass, execute: - ```bash - merge-after-ci - ``` - - If it fails and says something along the lines off `[ERROR] fail-on-change` then the formatter complained. - Execute `nix fmt` from the project root by hand and then make a new git commit. Afterwards redo step 3 and it should work. - -4. **Review Your Pull Request**: - - - Visit https://gitea.gchq.icu and go to the project page. Check under "Pull Requests" for any issues with your pull request. - -5. **Push Your Changes**: - - If there are issues, fix them and redo step 2. Afterward, execute: - ```bash - git push origin HEAD:YourUsername-main - ``` - - This will directly push to your open pull request. - -# Debugging - -When working on the backend of your project, debugging is an essential part of the development process. Here are some methods for debugging and testing the backend of your application: - -## Test Backend Locally in Devshell with Breakpoints - -To test the backend locally in a development environment and set breakpoints for debugging, follow these steps: - -1. Run the following command to execute your tests and allow for debugging with breakpoints: - ```bash - pytest -n0 -s --maxfail=1 - ``` - You can place `breakpoint()` in your Python code where you want to trigger a breakpoint for debugging. - -## Test Backend Locally in a Nix Sandbox - -To run your backend tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not: - -### Running Tests Marked as Impure - -If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command: - -```bash -nix run .#impure-checks -``` - -This command will run the impure test functions. - -### Running Pure Tests - -For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command: - -```bash -nix build .#checks.x86_64-linux.clan-pytest --rebuild -``` - -This command will run all pure test functions. - -### Inspecting the Nix Sandbox - -If you need to inspect the Nix sandbox while running tests, follow these steps: - -1. Insert an endless sleep into your test code where you want to pause the execution. For example: - - ```python - import time - time.sleep(3600) # Sleep for one hour - ``` - -2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example: - - ```bash - psgrep -a -x your_python_process_name - cntr attach - ``` - -These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application. - -# Using this Template - -To make the most of this template: - -1. Set up a new Gitea account named `ui-asset-bot`. Generate an access token with all access permissions and set it under `settings/actions/secrets` as a secret called `BOT_ACCESS_TOKEN`. - - - Also, edit the file `.gitea/workflows/ui_assets.yaml` and change the `BOT_EMAIL` variable to match the email you set for that account. Gitea matches commits to accounts by their email address, so this step is essential. - -2. Create a second Gitea account named `merge-bot`. Edit the file `pkgs/merge-after-ci/default.nix` if the name should be different. Under "Branches," set the main branch to be protected and add `merge-bot` to the whitelisted users for pushing. Set the unprotected file pattern to `**/ui-assets.nix`. - - - Enable the status check for "build / test (pull_request)." - -3. Add both `merge-bot` and `ui-asset-bot` as collaborators. - - Set the option to "Delete pull request branch after merge by default." - - Also, set the default merge style to "Rebase then create merge commit." - -With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!. diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index a85fdd0..3a11936 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -2,33 +2,112 @@ The clan-cli contains the command line interface as well as the graphical webui through the `clan webui` command. -## Hacking on the cli +Start the web ui with -We recommend setting up [direnv](https://direnv.net/) to load the developement with nix. -If you do not have it set up you can also use `nix develop` directly like this: - -``` -use flake .#clan-cli --builders '' +```bash +clan webui --reload --no-open --log-level debug --populate --emulate ``` -After you can use the local bin wrapper to test things in the cli: +- The `--populate` flag will automatically populate the database with dummy data + - To look into the endpoints open up a swagger instance by visiting: http://localhost:2979/docs +- The `--emulate` flag will automatically run servers the database with dummy data for the fronted to communicate with (ap, dlg, c1 and c2) + - To look into the emulated endpoints go to http://localhost:2979/emulate -``` -./bin/clan +# Building a Docker Image + +To build a docker image of the frontend and backend be inside the `pkgs/clan-cli` folder and execute: + +```bash +nix build .#clan-docker ``` -## Hacking on the webui +This will create a symlink directory called `result` to a tar.gz docker file. Import it by executing: -By default the webui is build from a tarball available https://git.clan.lol/clan/-/packages/generic/ui/. -To start a local developement environment instead, use the `--dev` flag: - -``` -./bin/clan webui --dev +```bash +docker load < result ``` -This will spawn two webserver, a python one to for the api and a nodejs one that rebuilds the ui on the fly. +And then run the docker file by executing: -## Run webui directly +```bash +docker run -p 127.0.0.1:2979:2979 clan-docker:latest +``` + +# Auto Generating a Python Client + +For the tests we automatically generate a python client for the API endpoints. To do this execute while inside the `pkgs/clan-cli` folder: + +```bash +./bin/gen-python-client +``` + +This will replace the folder +`tests/openapi_client`. + +# API Documentation + +Api documentation can be found in the folder `pkgs/clan-cli/tests/openapi_client/docs/` +For Entity object go to [tests/openapi_client/docs/EntityCreate.md](tests/openapi_client/docs/EntityCreate.md) + +# Debugging + +When working on the backend of your project, debugging is an essential part of the development process. Here are some methods for debugging and testing the backend of your application: + +## Test Backend Locally in Devshell with Breakpoints + +To test the backend locally in a development environment and set breakpoints for debugging, follow these steps: + +1. Run the following command to execute your tests and allow for debugging with breakpoints: + ```bash + rm -f sql_app.db && pytest -s + ``` + You can place `breakpoint()` in your Python code where you want to trigger a breakpoint for debugging. + +## Test Backend Locally in a Nix Sandbox + +To run your backend tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not: + +### Running Tests Marked as Impure + +If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command: + +```bash +nix run .#impure-checks +``` + +This command will run the impure test functions. + +### Running Pure Tests + +For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command: + +```bash +nix build .#checks.x86_64-linux.clan-pytest --rebuild +``` + +This command will run all pure test functions. + +### Inspecting the Nix Sandbox + +If you need to inspect the Nix sandbox while running tests, follow these steps: + +1. Insert an endless sleep into your test code where you want to pause the execution. For example: + + ```python + import time + time.sleep(3600) # Sleep for one hour + ``` + +2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example: + + ```bash + psgrep -a -x your_python_process_name + cntr attach + ``` + +These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application. + +## Run Web UI in VSCode Useful for vscode run and debug option @@ -53,18 +132,3 @@ Add this `launch.json` to your .vscode directory to have working breakpoints in ] } ``` - -## Run locally single-threaded for debugging - -By default tests run in parallel using pytest-xdist. -pytest-xdist however breaks `breakpoint()`. To disable it, use this: - -```console -pytest -n0 -s -``` - -You can also run a single test like this: - -```console -pytest -n0 -s tests/test_secrets_cli.py::test_users -``` diff --git a/pkgs/clan-cli/clan_cli/config.py b/pkgs/clan-cli/clan_cli/config.py index 0f9de51..24e5fcd 100644 --- a/pkgs/clan-cli/clan_cli/config.py +++ b/pkgs/clan-cli/clan_cli/config.py @@ -5,7 +5,7 @@ cors_url = [ "http://0.0.0.0", "http://[::]", ] -cors_ports = [2979, 3000] +cors_ports = ["*", 3000, 2979] # host for the server, frontend, backend and emulators host = "127.0.0.1" diff --git a/pkgs/clan-cli/flake-module.nix b/pkgs/clan-cli/flake-module.nix index e854320..0040db3 100644 --- a/pkgs/clan-cli/flake-module.nix +++ b/pkgs/clan-cli/flake-module.nix @@ -12,6 +12,19 @@ }; inherit (self'.packages.clan-cli) clan-openapi; default = self'.packages.clan-cli; + + + clan-docker = pkgs.dockerTools.buildImage { + name = "clan-docker"; + tag = "latest"; + created = "now"; + config = { + Cmd = [ "${self'.packages.clan-cli}/bin/clan" "webui" "--no-open" "--host" "0.0.0.0" ]; + ExposedPorts = { + "2979/tcp" = { }; + }; + }; + }; }; checks = self'.packages.clan-cli.tests; diff --git a/pkgs/ui/README.md b/pkgs/ui/README.md index ed8aeb6..a1ed4f2 100644 --- a/pkgs/ui/README.md +++ b/pkgs/ui/README.md @@ -1,5 +1,25 @@ # cLan - awesome UI +The files in `src/api` are autogenerated from the openapi.json. The openapi.json comes from the backend, from fastapi which autogenerates the openapi.json file from the python code. + +We then use orval to generate typescript files to `src/api` + +## Build Development Web UI + +To build the dev web ui with hotreloading + +```bash +npm run dev +``` + +## Build release Web UI + +To build the release version execute + +```bash +npm run build +``` + ## Updating dependencies After changing dependencies with @@ -16,14 +36,6 @@ To sort classnames manually: `cd /clan-core/pkgs/ui/` -## Upload ui to gitea +## Troubleshooting -Create a gitea token here: https://git.clan.lol/user/settings/applications - -Than run this command: - -``` -GITEA_TOKEN= nix run .#update-ui-assets -``` - -. +Sometimes new endpoints don't appear in `src/api`. Delete the `ui/openapi.json` file and execute `direnv reload` diff --git a/pkgs/ui/src/app/home/page.tsx b/pkgs/ui/src/app/home/page.tsx index a458011..121ed7e 100644 --- a/pkgs/ui/src/app/home/page.tsx +++ b/pkgs/ui/src/app/home/page.tsx @@ -7,6 +7,7 @@ import { HomeTableConfig } from "@/config/home"; import dynamic from "next/dynamic"; import { useEffect } from "react"; import { mutate } from "swr"; +import ErrorBoundary from "@/components/error_boundary"; const NoSSRSequenceDiagram = dynamic( () => import("../../components/sequence_diagram"), @@ -56,7 +57,9 @@ export default function Home() {

Sequence Diagram

- + + +
); diff --git a/pkgs/ui/src/components/sequence_diagram/index.tsx b/pkgs/ui/src/components/sequence_diagram/index.tsx index 43d29f3..b921e66 100644 --- a/pkgs/ui/src/components/sequence_diagram/index.tsx +++ b/pkgs/ui/src/components/sequence_diagram/index.tsx @@ -14,20 +14,20 @@ import { NoDataOverlay } from "../noDataOverlay"; import { useGetAllEventmessages } from "@/api/eventmessages/eventmessages"; import { mutate } from "swr"; import { LoadingOverlay } from "../join/loadingOverlay"; -import { generateMermaidString } from "./helpers"; +//import { generateMermaidString } from "./helpers"; const SequenceDiagram = () => { const { - data: eventMessagesData, + // data: eventMessagesData, isLoading: loadingEventMessages, swrKey: eventMessagesKeyFunc, } = useGetAllEventmessages(); const mermaidRef: any = useRef(null); const [scale, setScale] = useState(1); - const hasData = eventMessagesData?.data && eventMessagesData?.data.length > 0; + const hasData = false; // TODO: Readd this, right now it's always false - const mermaidString = generateMermaidString(eventMessagesData?.data); + const mermaidString = ""; //generateMermaidString(eventMessagesData?.data); useEffect(() => { if (!loadingEventMessages && hasData)