From 672c8364bcef877bed5aeb698f99f763a68090b3 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Wed, 24 Jan 2024 23:33:02 +0100 Subject: [PATCH 1/4] Fixed upload script --- pkgs/clan-cli/README.md | 135 ++++++++++++++++++++---------- pkgs/clan-cli/build_docker.sh | 34 +------- pkgs/clan-cli/upload_ui_assets.sh | 48 +++++++++++ pkgs/ui/nix/ui-assets.nix | 4 +- 4 files changed, 143 insertions(+), 78 deletions(-) create mode 100755 pkgs/clan-cli/upload_ui_assets.sh diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index 53ae7dd..32f9a42 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -13,6 +13,20 @@ clan webui --reload --no-open --log-level debug --populate --emulate - The `--emulate` flag will automatically run servers the database with dummy data for the fronted to communicate with (ap, dlg, c1 and c2) - To look into the emulated endpoints go to http://localhost:2979/emulate +# Using the Uploaded Docker Image + +Pull the image + +```bash +docker pull git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest +``` + +Run the image + +```bash +docker run -p 127.0.0.1:2979:2979 git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest +``` + # API Documentation Api documentation can be found in the folder `pkgs/clan-cli/tests/openapi_client/docs/` @@ -24,12 +38,24 @@ For Entity object go to - [tests/openapi_client/docs/ResolutionApi.md](tests/openapi_client/docs/ResolutionApi.md) - [tests/openapi_client/docs/RepositoriesApi.md](tests/openapi_client/docs/RepositoriesApi.md) -# Building a Docker Image if the Backend Changed -To build a new docker image when the backend code changed be inside the `pkgs/clan-cli` folder and execute: +# Building a Docker Image if the Frontend Changed + +To build a new docker image when the frontend code and/or backend code changed you first need +to get the `GITLAB_TOKEN` go to [repo access tokens](https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens) and generate one. + +- Make sure the Gitlab token has access to package registry. + +Then execute ```bash -nix build .#clan-docker +export GITLAB_TOKEN="" +``` + +Afterwards you can execute: + +```bash +./build_docker.sh ``` This will create a symlink directory called `result` to a tar.gz docker file. Import it by executing: @@ -44,28 +70,61 @@ And then run the docker file by executing: docker run -p 127.0.0.1:2979:2979 clan-docker:latest ``` -- To change parameters in the generated docker image edit the file : - [flake-module.nix at line 22](flake-module.nix) -- Documentation on `dockerTools.buildImage` you can find here: https://nix.dev/tutorials/nixos/building-and-running-docker-images.html +# Uploading a Docker Image -## Building a Docker Image if the Frontend Changed - -To build a new docker image when the frontend code changed you first need -to get the `GITLAB_TOKEN` go to [repo access tokens](https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens) and generate one. Then execute +You can use the script: ```bash -export GITLAB_TOKEN="" -``` - -Afterwards you can execute: - -```bash -./build_docker.sh +./push_docker.sh ``` ### The Script Explained -If changes to the UI have been made, and you want them to propagate to the docker container edit the file: [../ui/nix/ui-assets.nix](../ui/nix/ui-assets.nix). +Login to the tu docker image server + +```bash +docker login git.tu-berlin.de:5000 +``` + +Tag the imported image + +```bash +docker image tag clan-docker:latest git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest +``` + +Push the image to the git registry + +```bash +docker image push git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest +``` + + +# Upload UI assets as a package + +To upload the release build UI assets to gitlab as a package +first get the `GITLAB_TOKEN`. Go to [repo access tokens](https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens) and generate one. + +- Make sure the Gitlab token has access to package registry. + +To upload the UI assets as a package then execute: + +```bash +./upload_ui_assets.sh +``` + +Please commit the changes to ui-assets.nix and push them to the repository. +If you want clan webui to use the new ui assets. +```bash +$ git commit -m "Update ui-assets.nix" "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix" +$ git push +``` + + +If you execute `clan webui` the page you will see is a precompiled release version of the UI. This above script will update said precompiled release version. The `./build_docker.sh` script execute this to make sure that the included UI in the docker is up to date. + +### The Script Explained + +If changes to the UI have been made, and you want them to propagate to the docker container and the `clan webui` command edit the file: [../ui/nix/ui-assets.nix](../ui/nix/ui-assets.nix). This is where a release version of the frontend is downloaded and integrated into the cli and the docker build. To do this first execute ```bash @@ -75,7 +134,7 @@ nix build .#ui --out-link ui-release Make a tarball out of it called `ui-assets.tar.gz` ```bash -tar -czvf ui-assets.tar.gz ui-release/lib/node_modules/clan-ui/out/ +tar --transform 's,^\.,assets,' -czvf "ui-assets.tar.gz" -C ui-release/result/lib/node_modules/*/out . ``` Upload ui-assets.tar.gz to gitlab. @@ -112,45 +171,29 @@ And now build the docker image: nix build .#clan-docker ``` -# Uploading a Docker Image +# Building a Docker Image if only the Backend Changed -You can use the script: +To build a new docker image only when the backend code changed execute: ```bash -./push_docker.sh +nix build .#clan-docker ``` - -Login to the tu docker image server +This is much faster then the `./build_docker.sh` script as it needs not to build the frontend and again. +This will create a symlink directory called `result` to a tar.gz docker file. Import it by executing: ```bash -docker login git.tu-berlin.de:5000 +docker load < result ``` -Tag the imported image +And then run the docker file by executing: ```bash -docker image tag clan-docker:latest git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest +docker run -p 127.0.0.1:2979:2979 clan-docker:latest ``` -Push the image to the git registry - -```bash -docker image push git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest -``` - -# Using the Uploaded Docker Image - -Pull the image - -```bash -docker pull git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest -``` - -Run the image - -```bash -docker run -p 127.0.0.1:2979:2979 git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest -``` +- To change parameters in the generated docker image edit the file : + [flake-module.nix at line 22](flake-module.nix) +- Documentation on `dockerTools.buildImage` you can find here: https://nix.dev/tutorials/nixos/building-and-running-docker-images.html # Auto Generating a Python Client diff --git a/pkgs/clan-cli/build_docker.sh b/pkgs/clan-cli/build_docker.sh index f6efe0e..b279f51 100755 --- a/pkgs/clan-cli/build_docker.sh +++ b/pkgs/clan-cli/build_docker.sh @@ -1,40 +1,14 @@ #!/usr/bin/env bash -# shellcheck shell=bash set -euo pipefail -# GITLAB_TOKEN -if [[ -z "${GITLAB_TOKEN:-}" ]]; then -cat < "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix" < "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix" < Date: Wed, 24 Jan 2024 23:33:28 +0100 Subject: [PATCH 2/4] nix fmt --- pkgs/clan-cli/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkgs/clan-cli/README.md b/pkgs/clan-cli/README.md index 32f9a42..5a845ac 100644 --- a/pkgs/clan-cli/README.md +++ b/pkgs/clan-cli/README.md @@ -38,7 +38,6 @@ For Entity object go to - [tests/openapi_client/docs/ResolutionApi.md](tests/openapi_client/docs/ResolutionApi.md) - [tests/openapi_client/docs/RepositoriesApi.md](tests/openapi_client/docs/RepositoriesApi.md) - # Building a Docker Image if the Frontend Changed To build a new docker image when the frontend code and/or backend code changed you first need @@ -98,7 +97,6 @@ Push the image to the git registry docker image push git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest ``` - # Upload UI assets as a package To upload the release build UI assets to gitlab as a package @@ -114,12 +112,12 @@ To upload the UI assets as a package then execute: Please commit the changes to ui-assets.nix and push them to the repository. If you want clan webui to use the new ui assets. + ```bash $ git commit -m "Update ui-assets.nix" "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix" $ git push ``` - If you execute `clan webui` the page you will see is a precompiled release version of the UI. This above script will update said precompiled release version. The `./build_docker.sh` script execute this to make sure that the included UI in the docker is up to date. ### The Script Explained @@ -178,6 +176,7 @@ To build a new docker image only when the backend code changed execute: ```bash nix build .#clan-docker ``` + This is much faster then the `./build_docker.sh` script as it needs not to build the frontend and again. This will create a symlink directory called `result` to a tar.gz docker file. Import it by executing: From b978aabdd633b898201cf6bc7a895faf6e28581d Mon Sep 17 00:00:00 2001 From: ui-asset-bot Date: Wed, 24 Jan 2024 22:42:32 +0000 Subject: [PATCH 3/4] update ui-assets.nix --- pkgs/ui/nix/ui-assets.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/ui/nix/ui-assets.nix b/pkgs/ui/nix/ui-assets.nix index 915da6f..b641d9b 100644 --- a/pkgs/ui/nix/ui-assets.nix +++ b/pkgs/ui/nix/ui-assets.nix @@ -1,5 +1,5 @@ { fetchzip }: fetchzip { - url = "https://git.tu-berlin.de/api/v4/projects/internet-of-services-lab%2Fservice-aware-network-front-end/packages/generic/ui-assets/1.0.0/ui-assets.tar.gz"; - sha256 = "1npc1f4pa2d0q16bdygbclnx0c2lsi4j1w8r708xc35s1lvn1a4y"; + url = "https://gitea.gchq.icu/api/packages/IoSL/generic/IoSL-service-aware-frontend/1qs2jv3jvphjn15klq8pykald7db19jaswdql386yhfx6z3bi8j9/assets.tar.gz"; + sha256 = "1qs2jv3jvphjn15klq8pykald7db19jaswdql386yhfx6z3bi8j9"; } From da99e71b541efffa706f4eb8d5b700baacabd5d0 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Wed, 24 Jan 2024 23:44:12 +0100 Subject: [PATCH 4/4] fix src_name and des_name being NULL --- pkgs/clan-cli/clan_cli/webui/routers/endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/webui/routers/endpoints.py b/pkgs/clan-cli/clan_cli/webui/routers/endpoints.py index edb4bbd..bbf36b7 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/endpoints.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/endpoints.py @@ -386,9 +386,9 @@ def get_all_eventmessages( # Get the name of the src and des entity from the database src_name = sql_crud.get_entity_by_did(db, msg.src_did) - src_name = src_name if src_name is None else src_name.name + src_name = msg.src_did if src_name is None else src_name.name des_name = sql_crud.get_entity_by_did(db, msg.des_did) - des_name = des_name if des_name is None else des_name.name + des_name = msg.des_did if des_name is None else des_name.name result = cresult[cresult_idx]