Fixed incorrect grouping, in eventmessages

pkgs/clan-cli/clan_cli/webui/routers/endpoints.py

@@ -1,11 +1,13 @@
+import json
 import logging
 import time
 import typing
+from collections import OrderedDict
 from typing import Any, List, Optional
 
 import httpx
 from fastapi import APIRouter, BackgroundTasks, Depends, Query
-from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse
 from sqlalchemy.orm import Session
 
 from clan_cli.config import ap_url, c1_url, c2_url, dlg_url, group_type_to_label
@@ -360,19 +362,24 @@ def create_eventmessage(
 @typing.no_type_check
 @router.get(
     "/api/v1/event_messages",
-    response_class=JSONResponse,
+    response_class=PlainTextResponse,
     tags=[Tags.eventmessages],
 )
 def get_all_eventmessages(
     skip: int = 0, limit: int = 100, db: Session = Depends(sql_db.get_db)
-) -> JSONResponse:
+) -> PlainTextResponse:
+    # SQL sorts eventmessages by timestamp, so we don't need to sort them here
     eventmessages = sql_crud.get_eventmessages(db, skip=skip, limit=limit)
-    result: dict[int, dict[int, List[Eventmessage]]] = {}
+    cresult: List[OrderedDict[int, OrderedDict[int, List[Eventmessage]]]] = []
 
-    for msg in eventmessages:
+    cresult_idx = 0
+    cresult.append(OrderedDict())
+    for idx, msg in enumerate(eventmessages):
         # Use the group_type_to_label from config.py to get the group name and msg_type name
         group = group_type_to_label.get(msg.group, None)
-        group_name = group.get("name", None) if group is not None else str(msg.group)
+        group_name = (
+            str(group.get("name", None)) if group is not None else str(msg.group)
+        )
         msg_type_name = (
             group.get(msg.msg_type, None) if group is not None else str(msg.msg_type)
         )
@@ -383,14 +390,26 @@ def get_all_eventmessages(
         des_name = sql_crud.get_entity_by_did(db, msg.des_did)
         des_name = des_name if des_name is None else des_name.name
 
-        # Initialize the result array and dictionary
-        if result.get(group_name) is None:
-            result[group_name] = {}
-        if result[group_name].get(msg.group_id) is None:
-            result[group_name][msg.group_id] = []
+        result = cresult[cresult_idx]
+
+        if result.get("group_name") is None:
+            # Initialize the result array and dictionary
+            result["group_name"] = group_name
+        elif result["group_name"] != group_name:
+            # If the group name changed, create a new result array and dictionary
+            cresult_idx += 1
+            cresult.append(OrderedDict())
+            result = cresult[cresult_idx]
+            result["group_name"] = group_name
+
+        if result.get("groups") is None:
+            result["groups"] = OrderedDict()
+
+        if result["groups"].get(msg.group_id) is None:
+            result["groups"][msg.group_id] = []
 
         # Append the eventmessage to the result array
-        result_arr = result[group_name][msg.group_id]
+        result_arr = result["groups"][msg.group_id]
         result_arr.append(
             Eventmessage(
                 id=msg.id,
@@ -408,9 +427,7 @@ def get_all_eventmessages(
             ).dict()
         )
 
-        # sort by timestamp
-        result_arr.sort(key=lambda x: x["timestamp"])
-    return JSONResponse(content=result, status_code=200)
+    return PlainTextResponse(content=json.dumps(cresult, indent=4), status_code=200)
 
 
 ##############################

pkgs/clan-cli/clan_cli/webui/sql_crud.py

@@ -1,7 +1,7 @@
 # Imports
 from typing import List, Optional
 
-from sqlalchemy import func
+from sqlalchemy import asc, func
 from sqlalchemy.orm import Session
 from sqlalchemy.sql.expression import true
 
@@ -319,4 +319,11 @@ def create_eventmessage(
 def get_eventmessages(
     db: Session, skip: int = 0, limit: int = 100
 ) -> List[sql_models.Eventmessage]:
-    return db.query(sql_models.Eventmessage).offset(skip).limit(limit).all()
+    # Use order_by and desc to sort by timestamp
+    return (
+        db.query(sql_models.Eventmessage)
+        .order_by(asc(sql_models.Eventmessage.timestamp))
+        .offset(skip)
+        .limit(limit)
+        .all()
+    )

pkgs/clan-cli/clan_cli/webui/sql_models.py

@@ -101,7 +101,7 @@ class Eventmessage(Base):
     ## Queryable body ##
     # Primary Key
     id = Column(Integer, primary_key=True, autoincrement=True)
-    timestamp = Column(Integer, unique=True, index=True)
+    timestamp = Column(Integer, index=True)
     group = Column(Integer, index=True)
     group_id = Column(Integer, index=True)
     msg_type = Column(Integer, index=True)  # message type for the label

pkgs/clan-cli/tests/test_db_api.py

@@ -122,49 +122,50 @@ def test_create_services(api_client: ApiClient) -> None:
 random.seed(77)
 
 
-def create_eventmessages(num: int = 2) -> list[EventmessageCreate]:
+def create_eventmessages(num: int = 4) -> list[EventmessageCreate]:
     res = []
     starttime = int(time.time())
-    for i in range(num):
-        group_id = i % 5 + random.getrandbits(6)
+    for idx in range(num):
+        i2 = idx + 1
+        group_id = i2 % 5 + random.getrandbits(6) + 1
         em_req_send = EventmessageCreate(
-            timestamp=starttime + i * 10,
-            group=i % 5,
+            timestamp=starttime + i2 * 10,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=1,
-            src_did=f"did:sov:test:12{i}",
-            des_did=f"did:sov:test:12{i+1}",
+            src_did=f"did:sov:test:12{i2}",
+            des_did=f"did:sov:test:12{i2+1}",
             msg={},
         )
         res.append(em_req_send)
         em_req_rec = EventmessageCreate(
-            timestamp=starttime + (i * 10) + 2,
-            group=i % 5,
+            timestamp=starttime + (i2 * 10) + 2,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=2,
-            src_did=f"did:sov:test:12{i}",
-            des_did=f"did:sov:test:12{i+1}",
+            src_did=f"did:sov:test:12{i2}",
+            des_did=f"did:sov:test:12{i2+1}",
             msg={},
         )
         res.append(em_req_rec)
-        group_id = i % 5 + random.getrandbits(6)
+        group_id = i2 % 5 + random.getrandbits(6)
         em_res_send = EventmessageCreate(
-            timestamp=starttime + i * 10 + 4,
-            group=i % 5,
+            timestamp=starttime + i2 * 10 + 4,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=3,
-            src_did=f"did:sov:test:12{i+1}",
-            des_did=f"did:sov:test:12{i}",
+            src_did=f"did:sov:test:12{i2+1}",
+            des_did=f"did:sov:test:12{i2}",
             msg={},
         )
         res.append(em_res_send)
         em_res_rec = EventmessageCreate(
-            timestamp=starttime + (i * 10) + 8,
-            group=i % 5,
+            timestamp=starttime + (i2 * 10) + 8,
+            group=i2 % 5,
             group_id=group_id,
             msg_type=4,
-            src_did=f"did:sov:test:12{i+1}",
-            des_did=f"did:sov:test:12{i}",
+            src_did=f"did:sov:test:12{i2+1}",
+            des_did=f"did:sov:test:12{i2}",
             msg={},
         )
         res.append(em_res_rec)