Merge pull request 'Fixed wrong ordering of eventmessages' (#66) from Qubasa-main into main

Reviewed-on: #66

.gitignore

@@ -23,3 +23,5 @@ htmlcov
 # georgs
 pkgs/.vs/
 pkgs/clan-cli/.hypothesis/
+ui-assets.tar.gz
+ui-release

pkgs/clan-cli/README.md

@@ -24,9 +24,9 @@ For Entity object go to
 - [tests/openapi_client/docs/ResolutionApi.md](tests/openapi_client/docs/ResolutionApi.md)
 - [tests/openapi_client/docs/RepositoriesApi.md](tests/openapi_client/docs/RepositoriesApi.md)
 
-# Building a Docker Image
+# Building a Docker Image if the Backend Changed
 
-To build a docker image of the frontend and backend be inside the `pkgs/clan-cli` folder and execute:
+To build a new docker image when the backend code changed be inside the `pkgs/clan-cli` folder and execute:
 
 ```bash
 nix build .#clan-docker
@@ -48,7 +48,22 @@ docker run -p 127.0.0.1:2979:2979 clan-docker:latest
   [flake-module.nix at line 22](flake-module.nix)
 - Documentation on `dockerTools.buildImage` you can find here: https://nix.dev/tutorials/nixos/building-and-running-docker-images.html
 
-## Docker build with UI changes
+## Building a Docker Image if the Frontend Changed
+
+To build a new docker image when the frontend code changed you first need
+to get the `GITLAB_TOKEN` go to [repo access tokens](https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens) and generate one. Then execute
+
+```bash
+export GITLAB_TOKEN="<your-access-token>"
+```
+
+Afterwards you can execute:
+
+```bash
+./build_docker.sh
+```
+
+### The Script Explained
 
 If changes to the UI have been made, and you want them to propagate to the docker container edit the file: [../ui/nix/ui-assets.nix](../ui/nix/ui-assets.nix).
 This is where a release version of the frontend is downloaded and integrated into the cli and the docker build. To do this first execute
@@ -63,11 +78,7 @@ Make a tarball out of it called `ui-assets.tar.gz`
 tar -czvf ui-assets.tar.gz ui-release/lib/node_modules/clan-ui/out/
 ```
 
-Upload ui-assets.tar.gz to gitlab. To get the `GITLAB_TOKEN` go to [repo access tokens](https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens) and generate one. Then execute
+Upload ui-assets.tar.gz to gitlab.
-
-```bash
-export GITLAB_TOKEN="<your-access-token>"
-```
 
 ```bash
 curl --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
@@ -103,6 +114,12 @@ nix build .#clan-docker
 
 # Uploading a Docker Image
 
+You can use the script:
+
+```bash
+./push_docker.sh
+```
+
 Login to the tu docker image server
 
 ```bash
@@ -121,6 +138,8 @@ Push the image to the git registry
 docker image push git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest
 ```
 
+# Using the Uploaded Docker Image
+
 Pull the image
 
 ```bash

pkgs/clan-cli/build_docker.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# shellcheck shell=bash
+set -euo pipefail
+
+# GITLAB_TOKEN
+if [[ -z "${GITLAB_TOKEN:-}" ]]; then
+cat <<EOF
+GITLAB_TOKEN environment var is not set. Please generate a new token under
+https://git.tu-berlin.de/internet-of-services-lab/service-aware-network-front-end/-/settings/access_tokens
+EOF
+  exit 1
+fi
+
+# Create a new ui build
+nix build .#ui --out-link ui-release
+tar -czvf ui-assets.tar.gz ui-release/lib/node_modules/clan-ui/out/
+
+# upload ui assets to gitlab
+gitlab_base="https://git.tu-berlin.de/api/v4/projects/internet-of-services-lab%2Fservice-aware-network-front-end"
+curl --header "PRIVATE-TOKEN: $GITLAB_TOKEN" \
+     --upload-file ./ui-assets.tar.gz \
+     "$gitlab_base/packages/generic/ui-assets/1.0.0/ui-assets.tar.gz"
+
+
+# write url and hash to ui-assets.nix
+url="$gitlab_base/packages/generic/ui-assets/1.0.0/ui-assets.tar.gz"
+PROJECT_DIR=$(git rev-parse --show-toplevel)
+cat > "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix" <<EOF
+{ fetchzip }:
+fetchzip {
+  url = "$url";
+  sha256 = "$(nix-prefetch-url --unpack $url)";
+}
+EOF
+
+cat <<EOF
+Please commit the changes to ui-assets.nix and push them to the repository.
+If you want clan webui to use the new ui assets.
+$ git commit -m "Update ui-assets.nix" "$PROJECT_DIR/pkgs/ui/nix/ui-assets.nix"
+$ git push
+EOF

pkgs/clan-cli/clan_cli/webui/routers/endpoints.py

@@ -370,12 +370,16 @@ def get_all_eventmessages(
 ) -> PlainTextResponse:
     # SQL sorts eventmessages by timestamp, so we don't need to sort them here
     eventmessages = sql_crud.get_eventmessages(db, skip=skip, limit=limit)
-    result: dict[int, dict[int, List[Eventmessage]]] = OrderedDict()
+    cresult: List[OrderedDict[int, OrderedDict[int, List[Eventmessage]]]] = []
 
-    for msg in eventmessages:
+    cresult_idx = 0
+    cresult.append(OrderedDict())
+    for idx, msg in enumerate(eventmessages):
         # Use the group_type_to_label from config.py to get the group name and msg_type name
         group = group_type_to_label.get(msg.group, None)
-        group_name = group.get("name", None) if group is not None else str(msg.group)
+        group_name = (
+            str(group.get("name", None)) if group is not None else str(msg.group)
+        )
         msg_type_name = (
             group.get(msg.msg_type, None) if group is not None else str(msg.msg_type)
         )
@@ -386,14 +390,26 @@ def get_all_eventmessages(
         des_name = sql_crud.get_entity_by_did(db, msg.des_did)
         des_name = des_name if des_name is None else des_name.name
 
-        # Initialize the result array and dictionary
+        result = cresult[cresult_idx]
-        if result.get(group_name) is None:
+
-            result[group_name] = OrderedDict()
+        if result.get("group_name") is None:
-        if result[group_name].get(msg.group_id) is None:
+            # Initialize the result array and dictionary
-            result[group_name][msg.group_id] = []
+            result["group_name"] = group_name
+        elif result["group_name"] != group_name:
+            # If the group name changed, create a new result array and dictionary
+            cresult_idx += 1
+            cresult.append(OrderedDict())
+            result = cresult[cresult_idx]
+            result["group_name"] = group_name
+
+        if result.get("groups") is None:
+            result["groups"] = OrderedDict()
+
+        if result["groups"].get(msg.group_id) is None:
+            result["groups"][msg.group_id] = []
 
         # Append the eventmessage to the result array
-        result_arr = result[group_name][msg.group_id]
+        result_arr = result["groups"][msg.group_id]
         result_arr.append(
             Eventmessage(
                 id=msg.id,
@@ -411,7 +427,7 @@ def get_all_eventmessages(
             ).dict()
         )
 
-    return PlainTextResponse(content=json.dumps(result, indent=4), status_code=200)
+    return PlainTextResponse(content=json.dumps(cresult, indent=4), status_code=200)
 
 
 ##############################

pkgs/clan-cli/clan_cli/webui/sql_models.py

@@ -101,7 +101,7 @@ class Eventmessage(Base):
     ## Queryable body ##
     # Primary Key
     id = Column(Integer, primary_key=True, autoincrement=True)
-    timestamp = Column(Integer, unique=True, index=True)
+    timestamp = Column(Integer, index=True)
     group = Column(Integer, index=True)
     group_id = Column(Integer, index=True)
     msg_type = Column(Integer, index=True)  # message type for the label

pkgs/clan-cli/push_docker.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# shellcheck shell=bash
+set -euo pipefail
+
+docker login git.tu-berlin.de:5000
+docker load < result
+docker image tag clan-docker:latest git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest
+docker image push git.tu-berlin.de:5000/internet-of-services-lab/service-aware-network-front-end:latest

pkgs/clan-cli/tests/test_db_api.py

@@ -122,49 +122,50 @@ def test_create_services(api_client: ApiClient) -> None:
 random.seed(77)
 
 
-def create_eventmessages(num: int = 2) -> list[EventmessageCreate]:
+def create_eventmessages(num: int = 4) -> list[EventmessageCreate]:
     res = []
     starttime = int(time.time())
-    for i in range(num):
+    for idx in range(num):
-        group_id = i % 5 + random.getrandbits(6)
+        i2 = idx + 1
+        group_id = i2 % 5 + random.getrandbits(6) + 1
         em_req_send = EventmessageCreate(
-            timestamp=starttime + i * 10,
+            timestamp=starttime + i2 * 10,
-            group=i % 5,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=1,
-            src_did=f"did:sov:test:12{i}",
+            src_did=f"did:sov:test:12{i2}",
-            des_did=f"did:sov:test:12{i+1}",
+            des_did=f"did:sov:test:12{i2+1}",
             msg={},
         )
         res.append(em_req_send)
         em_req_rec = EventmessageCreate(
-            timestamp=starttime + (i * 10) + 2,
+            timestamp=starttime + (i2 * 10) + 2,
-            group=i % 5,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=2,
-            src_did=f"did:sov:test:12{i}",
+            src_did=f"did:sov:test:12{i2}",
-            des_did=f"did:sov:test:12{i+1}",
+            des_did=f"did:sov:test:12{i2+1}",
             msg={},
         )
         res.append(em_req_rec)
-        group_id = i % 5 + random.getrandbits(6)
+        group_id = i2 % 5 + random.getrandbits(6)
         em_res_send = EventmessageCreate(
-            timestamp=starttime + i * 10 + 4,
+            timestamp=starttime + i2 * 10 + 4,
-            group=i % 5,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=3,
-            src_did=f"did:sov:test:12{i+1}",
+            src_did=f"did:sov:test:12{i2+1}",
-            des_did=f"did:sov:test:12{i}",
+            des_did=f"did:sov:test:12{i2}",
             msg={},
         )
         res.append(em_res_send)
         em_res_rec = EventmessageCreate(
-            timestamp=starttime + (i * 10) + 8,
+            timestamp=starttime + (i2 * 10) + 8,
-            group=i % 5,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=4,
-            src_did=f"did:sov:test:12{i+1}",
+            src_did=f"did:sov:test:12{i2+1}",
-            des_did=f"did:sov:test:12{i}",
+            des_did=f"did:sov:test:12{i2}",
             msg={},
         )
         res.append(em_res_rec)