From 7d4d6f59c290bf67a34c5a2be54ec9d8749dc2ed Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sun, 3 Dec 2023 14:22:46 +0100 Subject: [PATCH 1/6] Better README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c2c5987..ceaff48 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,8 @@ Let's set up your Git workflow to collaborate effectively: 2. **Git Workflow**: 1. Add your changes to Git using `git add `. - 2. Run `nix fmt` to lint your files. - 3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`. + 2. Run `nix fmt` to lint your files. This will format your files and make changes! + 3. Commit your changes and those of nix fmt with a descriptive message: `git commit -a -m "My descriptive commit message"`. 4. Make sure your branch has the latest changes from upstream by executing: ```bash git fetch && git rebase origin/main --autostash From 20e6e551d4194b0d426ee55cc8e0ccbf0744a3bc Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sun, 3 Dec 2023 18:38:58 +0100 Subject: [PATCH 2/6] Improved README --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ceaff48..3523700 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ Let's get your development environment up and running: curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install ``` +On Windows Subsystem for Linux (WSL) the installer will fail and tell you what to do. Execute the command from the error message and then afterwards execute: +```bash +sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' +``` + 2. **Install direnv**: - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command: @@ -95,12 +100,12 @@ Let's set up your Git workflow to collaborate effectively: ```bash tea login add ``` + - Go to https://gitea.gchq.icu/user/settings/applications and create token with all privileges - Fill out the prompt as follows: - URL of Gitea instance: `https://gitea.gchq.icu` - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171` - - Do you have an access token? No - - Username: YourUsername - - Password: YourPassword + - Do you have an access token? Yes + - Token: ********* - Set Optional settings: No 2. **Git Workflow**: From 80269832e6928cf140cca64f5b58c9eb982030ea Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sun, 3 Dec 2023 18:39:19 +0100 Subject: [PATCH 3/6] Improved README --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3523700..85d2b42 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Let's get your development environment up and running: ``` On Windows Subsystem for Linux (WSL) the installer will fail and tell you what to do. Execute the command from the error message and then afterwards execute: + ```bash sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' ``` @@ -105,13 +106,13 @@ Let's set up your Git workflow to collaborate effectively: - URL of Gitea instance: `https://gitea.gchq.icu` - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171` - Do you have an access token? Yes - - Token: ********* + - Token: ****\***** - Set Optional settings: No 2. **Git Workflow**: 1. Add your changes to Git using `git add `. - 2. Run `nix fmt` to lint your files. This will format your files and make changes! + 2. Run `nix fmt` to lint your files. This will format your files and make changes! 3. Commit your changes and those of nix fmt with a descriptive message: `git commit -a -m "My descriptive commit message"`. 4. Make sure your branch has the latest changes from upstream by executing: ```bash From a93e6019c5c395d031045482011d45cb3315dad0 Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sat, 9 Dec 2023 18:57:20 +0100 Subject: [PATCH 4/6] clan_cli: Added automated database population. Fixed incorrect error handling in backend --- pkgs/clan-cli/clan_cli/webui/__init__.py | 6 ++ pkgs/clan-cli/clan_cli/webui/app.py | 5 +- .../clan-cli/clan_cli/webui/error_handlers.py | 2 +- .../clan_cli/webui/routers/sql_connect.py | 55 +++++++++---------- pkgs/clan-cli/clan_cli/webui/schemas.py | 8 ++- pkgs/clan-cli/clan_cli/webui/server.py | 21 +++++++ pkgs/clan-cli/clan_cli/webui/sql_crud.py | 26 ++++----- pkgs/clan-cli/clan_cli/webui/sql_models.py | 1 + pkgs/clan-cli/tests/test_db_api.py | 32 ++++++++--- 9 files changed, 102 insertions(+), 54 deletions(-) diff --git a/pkgs/clan-cli/clan_cli/webui/__init__.py b/pkgs/clan-cli/clan_cli/webui/__init__.py index de2f5ae..4fc79ad 100644 --- a/pkgs/clan-cli/clan_cli/webui/__init__.py +++ b/pkgs/clan-cli/clan_cli/webui/__init__.py @@ -22,6 +22,12 @@ def register_parser(parser: argparse.ArgumentParser) -> None: parser.add_argument( "--host", type=str, default="localhost", help="Host to listen on" ) + parser.add_argument( + "--no-populate", + action="store_true", + help="Don't populate the database with dummy data", + default=False, + ) parser.add_argument( "--no-open", action="store_true", help="Don't open the browser", default=False ) diff --git a/pkgs/clan-cli/clan_cli/webui/app.py b/pkgs/clan-cli/clan_cli/webui/app.py index b5cb9b1..173cef8 100644 --- a/pkgs/clan-cli/clan_cli/webui/app.py +++ b/pkgs/clan-cli/clan_cli/webui/app.py @@ -35,10 +35,11 @@ def setup_app() -> FastAPI: # bind sql engine # TODO comment aut and add flag to run with pupulated data rm *.sql run pytest with marked then start clan webui # https://docs.pytest.org/en/7.1.x/example/markers.html - sql_models.Base.metadata.drop_all(engine) + # sql_models.Base.metadata.drop_all(engine) + sql_models.Base.metadata.create_all(bind=engine) - app = FastAPI(lifespan=lifespan) + app = FastAPI(lifespan=lifespan, swagger_ui_parameters={"tryItOutEnabled": True}) app.add_middleware( CORSMiddleware, allow_origins=origins, diff --git a/pkgs/clan-cli/clan_cli/webui/error_handlers.py b/pkgs/clan-cli/clan_cli/webui/error_handlers.py index 341bb2e..9948cc5 100644 --- a/pkgs/clan-cli/clan_cli/webui/error_handlers.py +++ b/pkgs/clan-cli/clan_cli/webui/error_handlers.py @@ -25,7 +25,7 @@ def sql_error_handler(request: Request, exc: SQLAlchemyError) -> JSONResponse: def clan_error_handler(request: Request, exc: ClanError) -> JSONResponse: - log.error("ClanError: %s", exc) + log.exception(exc) detail = [ { "loc": [], diff --git a/pkgs/clan-cli/clan_cli/webui/routers/sql_connect.py b/pkgs/clan-cli/clan_cli/webui/routers/sql_connect.py index 9a5c8b1..9fe73f7 100644 --- a/pkgs/clan-cli/clan_cli/webui/routers/sql_connect.py +++ b/pkgs/clan-cli/clan_cli/webui/routers/sql_connect.py @@ -1,3 +1,4 @@ +import logging import time from typing import List, Optional @@ -5,6 +6,7 @@ import httpx from fastapi import APIRouter, BackgroundTasks, Depends from sqlalchemy.orm import Session +from ...errors import ClanError from .. import sql_crud, sql_db, sql_models from ..schemas import ( Consumer, @@ -20,6 +22,8 @@ from ..tags import Tags router = APIRouter() +log = logging.getLogger(__name__) + ######################### # # @@ -141,11 +145,7 @@ def get_repository( @router.post("/api/v1/create_entity", response_model=Entity, tags=[Tags.entities]) def create_entity( entity: EntityCreate, db: Session = Depends(sql_db.get_db) -) -> EntityCreate | str: - # todo checken ob schon da ... - if sql_crud.get_entity_by_did(db, did=entity.did): - print("did already exsists") - return "Error did already exsists in db" +) -> EntityCreate: return sql_crud.create_entity(db, entity) @@ -178,7 +178,7 @@ def get_attached_entities( return entities -@router.post("/api/v1/detach") +@router.post("/api/v1/detach", response_model=Entity, tags=[Tags.entities]) async def detach( background_tasks: BackgroundTasks, entity_did: str = "did:sov:test:1234", @@ -186,13 +186,11 @@ async def detach( limit: int = 100, db: Session = Depends(sql_db.get_db), ) -> dict[str, str]: - background_tasks.add_task( - sql_crud.set_attached_by_entity_did, db, entity_did, False - ) - return {"message": "Detaching in the background"} + entity = sql_crud.set_attached_by_entity_did(db, entity_did, False) + return entity -@router.post("/api/v1/attach") +@router.post("/api/v1/attach", tags=[Tags.entities]) async def attach( background_tasks: BackgroundTasks, entity_did: str = "did:sov:test:1234", @@ -200,27 +198,28 @@ async def attach( limit: int = 100, db: Session = Depends(sql_db.get_db), ) -> dict[str, str]: + if sql_crud.get_entity_by_did(db, entity_did) is None: + raise ClanError(f"Entity with did '{entity_did}' not found") + background_tasks.add_task(attach_entity, db, entity_did) return {"message": "Attaching in the background"} -# TODO def attach_entity(db: Session, entity_did: str) -> None: db_entity = sql_crud.set_attached_by_entity_did(db, entity_did, True) try: - if db_entity is not None: - while db_entity.attached: - # query status endpoint - # https://www.python-httpx.org/ - response = httpx.get(f"http://{db_entity.ip}", timeout=2) - print(response) - # test with: - # while true ; do printf 'HTTP/1.1 200 OK\r\n\r\ncool, thanks' | nc -l -N localhost 5555 ; done - # client test (apt install python3-httpx): - # httpx http://localhost:5555 - # except not reached set false - time.sleep(1) - except Exception as e: - print(e) - if db_entity is not None: - db_entity = sql_crud.set_attached_by_entity_did(db, entity_did, False) + while db_entity.attached: + # query status endpoint + # https://www.python-httpx.org/ + response = httpx.get(f"http://{db_entity.ip}", timeout=2) + print(response) + # test with: + # while true ; do printf 'HTTP/1.1 200 OK\r\n\r\ncool, thanks' | nc -l -N localhost 5555 ; done + # client test (apt install python3-httpx): + # httpx http://localhost:5555 + # except not reached set false + time.sleep(1) + except Exception: + log.warning(f"Entity {entity_did} not reachable. Setting attached to false") + + db_entity = sql_crud.set_attached_by_entity_did(db, entity_did, False) diff --git a/pkgs/clan-cli/clan_cli/webui/schemas.py b/pkgs/clan-cli/clan_cli/webui/schemas.py index 96987f9..ccfb548 100644 --- a/pkgs/clan-cli/clan_cli/webui/schemas.py +++ b/pkgs/clan-cli/clan_cli/webui/schemas.py @@ -27,7 +27,7 @@ class ProducerBase(BaseModel): service_type: str = "3D Printing" endpoint_url: str = "http://127.0.0.1:8000" status: str = "unknown" - other: dict = {"test": "test"} + other: dict = {"action": ["register", "deregister", "delete", "create"]} class ProducerCreate(ProducerBase): @@ -91,7 +91,11 @@ class EntityBase(BaseModel): name: str = "C1" ip: str = "127.0.0.1" attached: bool = False - other: dict = {"test": "test"} + visible: bool = True + other: dict = { + "network": "Carlo's Home Network", + "roles": ["service repository", "service prosumer"], + } class EntityCreate(EntityBase): diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 66b0f39..8975166 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -105,6 +105,27 @@ def start_server(args: argparse.Namespace) -> None: if not args.no_open: Thread(target=open_browser, args=(base_url, args.sub_url)).start() + # DELETE all data from the database + from . import sql_models + from .sql_db import engine + + sql_models.Base.metadata.drop_all(engine) + + if args.no_populate is False: + test_dir = Path(__file__).parent.parent.parent / "tests" + + if not test_dir.is_dir(): + raise ClanError(f"Could not find test dir: {test_dir}") + + test_db_api = test_dir / "test_db_api.py" + if not test_db_api.is_file(): + raise ClanError(f"Could not find test db api: {test_db_api}") + + import subprocess + + cmd = ["pytest", "-s", "-n0", str(test_db_api)] + subprocess.run(cmd, check=True) + uvicorn.run( "clan_cli.webui.app:app", host=args.host, diff --git a/pkgs/clan-cli/clan_cli/webui/sql_crud.py b/pkgs/clan-cli/clan_cli/webui/sql_crud.py index d6119c9..fcc8c15 100644 --- a/pkgs/clan-cli/clan_cli/webui/sql_crud.py +++ b/pkgs/clan-cli/clan_cli/webui/sql_crud.py @@ -3,6 +3,7 @@ from typing import List, Optional from sqlalchemy.orm import Session from sqlalchemy.sql.expression import true +from ..errors import ClanError from . import schemas, sql_models ######################### @@ -153,21 +154,18 @@ def get_attached_entities( ) -# set attached -# None if did not found # Returns same entity if setting didnt changed something def set_attached_by_entity_did( db: Session, entity_did: str, value: bool -) -> Optional[sql_models.Entity]: - # ste attached to true +) -> sql_models.Entity: db_entity = get_entity_by_did(db, entity_did) - if db_entity is not None: - # db_entity.attached = Column(True) - setattr(db_entity, "attached", value) - # save changes in db - db.add(db_entity) - db.commit() - db.refresh(db_entity) - return db_entity - else: - return db_entity + if db_entity is None: + raise ClanError(f"Entity with did '{entity_did}' not found") + + setattr(db_entity, "attached", value) + + # save changes in db + db.add(db_entity) + db.commit() + db.refresh(db_entity) + return db_entity diff --git a/pkgs/clan-cli/clan_cli/webui/sql_models.py b/pkgs/clan-cli/clan_cli/webui/sql_models.py index 88231ca..68ab103 100644 --- a/pkgs/clan-cli/clan_cli/webui/sql_models.py +++ b/pkgs/clan-cli/clan_cli/webui/sql_models.py @@ -26,6 +26,7 @@ class Entity(Base): name = Column(String, index=True) ip = Column(String, index=True) attached = Column(Boolean, index=True) + visible = Column(Boolean, index=True) ## Non queryable body ## # In here we deposit: Network, Roles, Visible, etc. diff --git a/pkgs/clan-cli/tests/test_db_api.py b/pkgs/clan-cli/tests/test_db_api.py index 5f6d16e..9e24e4d 100644 --- a/pkgs/clan-cli/tests/test_db_api.py +++ b/pkgs/clan-cli/tests/test_db_api.py @@ -77,7 +77,9 @@ def test_producer(api: TestClient) -> None: "service_type": "3D Printing", "endpoint_url": "http://127.0.0.1:8000", "status": "unknown", - "other": {"test": "test"}, + "other": { + "action": ["register", "deregister", "delete", "create"] + }, "entity_did": default_entity_did, } paramter = "producer" @@ -92,7 +94,9 @@ def test_producer2(api: TestClient) -> None: "service_type": "Fax", "endpoint_url": "http://127.0.0.1:8001", "status": "unknown", - "other": {"faxen": "dicke"}, + "other": { + "action": ["register", "deregister", "delete", "create"] + }, "entity_did": default_entity_did2, } paramter = "producer" @@ -107,7 +111,9 @@ def test_producer3(api: TestClient) -> None: "service_type": "VR-Stream", "endpoint_url": "http://127.0.0.1:8002", "status": "unknown", - "other": {"oculos": "rift"}, + "other": { + "action": ["register", "deregister", "delete", "create"] + }, "entity_did": default_entity_did3, } paramter = "producer" @@ -122,7 +128,9 @@ def test_producer4(api: TestClient) -> None: "service_type": "gallary", "endpoint_url": "http://127.0.0.1:8003", "status": "unknown", - "other": {"nice": "pics"}, + "other": { + "action": ["register", "deregister", "delete", "create"] + }, "entity_did": default_entity_did4, } paramter = "producer" @@ -137,7 +145,9 @@ def test_producer5(api: TestClient) -> None: "service_type": "Game-Shop", "endpoint_url": "http://127.0.0.1:8004", "status": "unknown", - "other": {"war": "games"}, + "other": { + "action": ["register", "deregister", "delete", "create"] + }, "entity_did": default_entity_did5, } paramter = "producer" @@ -263,7 +273,11 @@ def test_entity(api: TestClient) -> None: "name": "C1", "ip": "127.0.0.1", "attached": False, - "other": {"test": "test"}, + "visible": True, + "other": { + "network": "Carlo1's Home Network", + "roles": ["service repository", "service consumer"], + }, } paramter = "entity" # get_request = "entity_did=did%3Asov%3Atest%3A1234" @@ -276,7 +290,11 @@ def test_entity2(api: TestClient) -> None: "name": "C2", "ip": "127.0.0.2", "attached": False, - "other": {"test": "test"}, + "visible": True, + "other": { + "network": "Carlo2's Home Network", + "roles": ["service repository", "service prosumer"], + }, } paramter = "entity" get_request = "entity_did=" + url.quote(default_entity_did2) From 4e0346d0a0e73c7efbf41100c40ca5590c7e7dfb Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sat, 9 Dec 2023 18:57:55 +0100 Subject: [PATCH 5/6] nix fmt --- README.md | 2 +- pkgs/clan-cli/tests/test_db_api.py | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 85d2b42..ef9cb23 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Let's set up your Git workflow to collaborate effectively: - URL of Gitea instance: `https://gitea.gchq.icu` - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171` - Do you have an access token? Yes - - Token: ****\***** + - Token: \***\*\*\*\*** - Set Optional settings: No 2. **Git Workflow**: diff --git a/pkgs/clan-cli/tests/test_db_api.py b/pkgs/clan-cli/tests/test_db_api.py index 9e24e4d..b26a65e 100644 --- a/pkgs/clan-cli/tests/test_db_api.py +++ b/pkgs/clan-cli/tests/test_db_api.py @@ -77,9 +77,7 @@ def test_producer(api: TestClient) -> None: "service_type": "3D Printing", "endpoint_url": "http://127.0.0.1:8000", "status": "unknown", - "other": { - "action": ["register", "deregister", "delete", "create"] - }, + "other": {"action": ["register", "deregister", "delete", "create"]}, "entity_did": default_entity_did, } paramter = "producer" @@ -94,9 +92,7 @@ def test_producer2(api: TestClient) -> None: "service_type": "Fax", "endpoint_url": "http://127.0.0.1:8001", "status": "unknown", - "other": { - "action": ["register", "deregister", "delete", "create"] - }, + "other": {"action": ["register", "deregister", "delete", "create"]}, "entity_did": default_entity_did2, } paramter = "producer" @@ -111,9 +107,7 @@ def test_producer3(api: TestClient) -> None: "service_type": "VR-Stream", "endpoint_url": "http://127.0.0.1:8002", "status": "unknown", - "other": { - "action": ["register", "deregister", "delete", "create"] - }, + "other": {"action": ["register", "deregister", "delete", "create"]}, "entity_did": default_entity_did3, } paramter = "producer" @@ -128,9 +122,7 @@ def test_producer4(api: TestClient) -> None: "service_type": "gallary", "endpoint_url": "http://127.0.0.1:8003", "status": "unknown", - "other": { - "action": ["register", "deregister", "delete", "create"] - }, + "other": {"action": ["register", "deregister", "delete", "create"]}, "entity_did": default_entity_did4, } paramter = "producer" @@ -145,9 +137,7 @@ def test_producer5(api: TestClient) -> None: "service_type": "Game-Shop", "endpoint_url": "http://127.0.0.1:8004", "status": "unknown", - "other": { - "action": ["register", "deregister", "delete", "create"] - }, + "other": {"action": ["register", "deregister", "delete", "create"]}, "entity_did": default_entity_did5, } paramter = "producer" From 0c08cde34c3a5295c178260af09e70cb7824e85d Mon Sep 17 00:00:00 2001 From: Luis-Hebendanz Date: Sat, 9 Dec 2023 19:10:11 +0100 Subject: [PATCH 6/6] Fixed testing issue. Improved README --- README.md | 10 ++++++---- pkgs/clan-cli/clan_cli/webui/__init__.py | 4 ++-- pkgs/clan-cli/clan_cli/webui/server.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ef9cb23..5d285c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# 1Service Aware Network Project Repo +# Service Aware Network Project Repo -Welcome to our website repository! This repo is designed to help you and your team build high-quality websites efficiently. We've carefully chosen the technologies to make development smooth and enjoyable. Here's what you can expect from this template: +Welcome to our website repository! This repo is designed to build high-quality websites efficiently. We've carefully chosen the technologies to make development smooth and enjoyable. **Frontend**: Our frontend is powered by [React NextJS](https://nextjs.org/), a popular and versatile framework for building web applications. @@ -51,6 +51,7 @@ sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' 4. **Clone the Repository and Navigate**: - Clone this repository and navigate to it. + - If you are under Windows Subystem For Linux (WSL) please clone the repository to the home folder of your Linux. Do NOT clone it onto your Windows machine! 5. **Allow .envrc**: @@ -72,9 +73,10 @@ sudo echo "experimental-features = nix-command flakes" > '/etc/nix/nix.conf' - To start the backend server, execute: ```bash - clan webui --reload --no-open --log-level debug + clan webui --reload --no-open --log-level debug --populate ``` - The server will automatically restart if any Python files change. + - The `--populate` flag will automatically populate the database with dummy data 8. **Build the Frontend**: @@ -194,8 +196,8 @@ If you need to inspect the Nix sandbox while running tests, follow these steps: 2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example: ```bash - cntr exec -w your_sandbox_name psgrep -a -x your_python_process_name + cntr attach ``` These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application. diff --git a/pkgs/clan-cli/clan_cli/webui/__init__.py b/pkgs/clan-cli/clan_cli/webui/__init__.py index 4fc79ad..1305e78 100644 --- a/pkgs/clan-cli/clan_cli/webui/__init__.py +++ b/pkgs/clan-cli/clan_cli/webui/__init__.py @@ -23,9 +23,9 @@ def register_parser(parser: argparse.ArgumentParser) -> None: "--host", type=str, default="localhost", help="Host to listen on" ) parser.add_argument( - "--no-populate", + "--populate", action="store_true", - help="Don't populate the database with dummy data", + help="Populate the database with dummy data", default=False, ) parser.add_argument( diff --git a/pkgs/clan-cli/clan_cli/webui/server.py b/pkgs/clan-cli/clan_cli/webui/server.py index 8975166..762ccb2 100644 --- a/pkgs/clan-cli/clan_cli/webui/server.py +++ b/pkgs/clan-cli/clan_cli/webui/server.py @@ -111,7 +111,7 @@ def start_server(args: argparse.Namespace) -> None: sql_models.Base.metadata.drop_all(engine) - if args.no_populate is False: + if args.populate: test_dir = Path(__file__).parent.parent.parent / "tests" if not test_dir.is_dir():