readmes added comments and some links to the openapi docs mds :)
All checks were successful
checks-impure / test (pull_request) Successful in 29s
checks / test (pull_request) Successful in 3m29s

This commit is contained in:
Georg-Stahn
2024-01-16 22:25:05 +01:00
parent 01e98d363b
commit 3052015a51
15 changed files with 215 additions and 97 deletions

View File

@@ -0,0 +1,37 @@
**init\_**.py:
```bash
usage: clan webui [-h] [--port PORT] [--host HOST] [--populate] [--emulate] [--no-open] [--dev]
[--dev-port DEV_PORT] [--dev-host DEV_HOST] [--reload]
[--log-level {critical,error,warning,info,debug,trace}]
[sub_url]
positional arguments:
sub_url Sub URL to open in the browser
options:
-h, --help show this help message and exit
--port PORT Port to listen on
--host HOST Host to listen on
--populate Populate the database with dummy data
--emulate Emulate two entities c1 and c2 + dlg and ap
--no-open Don't open the browser
--dev Run in development mode
--dev-port DEV_PORT Port to listen on for the dev server
--dev-host DEV_HOST Host to listen on
--reload Don't reload on changes
--log-level {critical,error,warning,info,debug,trace}
Log level
```
In this folder are some basic files:
- config.py
- to configer basic value for the server and the emulation
- ip/host
- ports
- emuplate_fast.py
- some api call that emulate the behavoir
- extra servers with api calls are emulated here
In the subfolder <webui> is the backend impplemented.

View File

@@ -1,14 +1,18 @@
# Imports
import argparse
import logging
import sys
from types import ModuleType
from typing import Optional
# Custom imports
from . import webui
from .custom_logger import setup_logging
# Setting up the logger
log = logging.getLogger(__name__)
# Trying to import argcomplete module, if not present, set it to None
argcomplete: Optional[ModuleType] = None
try:
import argcomplete # type: ignore[no-redef]
@@ -16,44 +20,59 @@ except ImportError:
pass
# Function to create the main argument parser
def create_parser(prog: Optional[str] = None) -> argparse.ArgumentParser:
# Creating the main argument parser with a description
parser = argparse.ArgumentParser(prog=prog, description="cLAN tool")
# Adding a debug argument to enable debug logging
parser.add_argument(
"--debug",
help="Enable debug logging",
action="store_true",
)
# Adding subparsers for different commands
subparsers = parser.add_subparsers()
# Adding a subparser for the "webui" command
parser_webui = subparsers.add_parser("webui", help="start webui")
# Registering additional arguments for the "webui" command
webui.register_parser(parser_webui)
# Using argcomplete for shell autocompletion if available
if argcomplete:
argcomplete.autocomplete(parser)
# If no command-line arguments provided, print the help message
if len(sys.argv) == 1:
parser.print_help()
return parser
# this will be the entrypoint under /bin/clan (see pyproject.toml config)
### Main entry point function
def main() -> None:
# Creating the main argument parser
parser = create_parser()
# Parsing command-line arguments
args = parser.parse_args()
# Setting up logging based on the debug flag
if args.debug:
setup_logging(logging.DEBUG)
log.debug("Debug log activated")
else:
setup_logging(logging.INFO)
# If the parsed arguments do not have the "func" attribute, exit
if not hasattr(args, "func"):
return
# Calling the function associated with the specified command
args.func(args)
# Entry point for script execution
if __name__ == "__main__":
main()

View File

@@ -1,4 +1,15 @@
# CORS configuration
cors_url = [
"http://localhost",
"http://127.0.0.1",
"http://0.0.0.0",
"http://[::]",
]
cors_ports = [2979, 3000]
# host for the server, frontend, backend and emulators
host = "127.0.0.1"
# used for emmulation and population for testing
port_dlg = 7000
port_ap = 7500
_port_client_base = 8000

View File

@@ -1,19 +1,24 @@
# Importing necessary modules and packages
import sys
import time
import urllib
from datetime import datetime
# Importing FastAPI and related components
from fastapi import FastAPI
from fastapi.responses import HTMLResponse, JSONResponse
# Importing configuration and schemas from the clan_cli package
import clan_cli.config as config
from clan_cli.webui.schemas import Resolution
# Creating FastAPI instances for different applications
app_dlg = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
app_ap = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
app_c1 = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
app_c2 = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
# List of FastAPI instances and their associated ports
apps = [
(app_dlg, config.port_dlg),
(app_ap, config.port_ap),
@@ -22,7 +27,7 @@ apps = [
]
#### HEALTHCHECK
# Healthcheck endpoints for different applications
@app_c1.get("/")
async def root_c1() -> str:
return "C1 is alive"
@@ -63,6 +68,7 @@ async def healthcheck_ap() -> str:
return "200 OK"
# Function for performing health checks on a given URL with retries
def get_health(*, url: str, max_retries: int = 20, delay: float = 0.2) -> str | None:
for attempt in range(max_retries):
try:
@@ -74,13 +80,10 @@ def get_health(*, url: str, max_retries: int = 20, delay: float = 0.2) -> str |
return None
#### CONSUME SERVICE
# TODO send_msg???
# Service consumption emulation for c1 which returns a gif1
@app_c1.get("/v1/print_daemon1", response_class=HTMLResponse)
async def consume_service_from_other_entity_c1() -> HTMLResponse:
# HTML content for the response
html_content = """
<html>
<body>
@@ -104,6 +107,7 @@ async def deregister_c1() -> JSONResponse:
@app_c2.get("/v1/print_daemon2", response_class=HTMLResponse)
async def consume_service_from_other_entity_c2() -> HTMLResponse:
# Similar HTML content for the response
html_content = """
<html>
<body>
@@ -127,7 +131,9 @@ async def deregister_c2() -> JSONResponse:
@app_ap.get("/ap_list_of_services", response_class=JSONResponse)
async def ap_list_of_services() -> JSONResponse:
# Sample list of services as a JSON response
res = [
# Service 1
{
"uuid": "bdd640fb-0667-1ad1-1c80-317fa3b1799d",
"service_name": "Carlos Printing0",
@@ -150,6 +156,7 @@ async def ap_list_of_services() -> JSONResponse:
},
"usage": [{"times_consumed": 2, "consumer_entity_did": "did:sov:test:120"}],
},
# Service 2 (similar structure)
{
"uuid": "23b8c1e9-3924-56de-3eb1-3b9046685257",
"service_name": "Carlos Printing1",
@@ -217,7 +224,6 @@ async def ap_list_of_services() -> JSONResponse:
"usage": [{"times_consumed": 2, "consumer_entity_did": "did:sov:test:120"}],
},
]
# resp = json.dumps(obj=res)
return JSONResponse(content=res, status_code=200)

View File

@@ -0,0 +1,6 @@
Here are the files found for the backend of the service.
- the backed is using a sql light db with sqlachremy
- this is done in
- sql\_\*.py, schema.py, tags.py
- subfolder: routers which also contains the apicall defenitions

View File

@@ -2,25 +2,55 @@ import argparse
import logging
from typing import Callable, NoReturn, Optional
# Get the logger for this module
log = logging.getLogger(__name__)
# Initialize variables for server startup and potential ImportError
start_server: Optional[Callable] = None
ServerImportError: Optional[ImportError] = None
# Try importing the start_server function from the server module
try:
from .server import start_server
except ImportError as e:
# If ImportError occurs, log the exception and store it in ServerImportError
log.exception(e)
ServerImportError = e
# Function to be called when FastAPI is not installed
##########################################################################################
# usage: clan webui [-h] [--port PORT] [--host HOST] [--populate] [--emulate] [--no-open] [--dev]
# [--dev-port DEV_PORT] [--dev-host DEV_HOST] [--reload]
# [--log-level {critical,error,warning,info,debug,trace}]
# [sub_url]
#
# positional arguments:
# sub_url Sub URL to open in the browser
#
# options:
# -h, --help show this help message and exit
# --port PORT Port to listen on
# --host HOST Host to listen on
# --populate Populate the database with dummy data
# --emulate Emulate two entities c1 and c2 + dlg and ap
# --no-open Don't open the browser
# --dev Run in development mode
# --dev-port DEV_PORT Port to listen on for the dev server
# --dev-host DEV_HOST Host to listen on
# --reload Don't reload on changes
# --log-level {critical,error,warning,info,debug,trace}
# Log level
##########################################################################################
def fastapi_is_not_installed(_: argparse.Namespace) -> NoReturn:
assert ServerImportError is not None
print(
f"Dependencies for the webserver is not installed. The webui command has been disabled ({ServerImportError})"
f"Dependencies for the webserver are not installed. The webui command has been disabled ({ServerImportError})"
)
exit(1)
# Function to register command-line arguments for the webserver
def register_parser(parser: argparse.ArgumentParser) -> None:
parser.add_argument("--port", type=int, default=2979, help="Port to listen on")
parser.add_argument(
@@ -69,10 +99,10 @@ def register_parser(parser: argparse.ArgumentParser) -> None:
type=str,
default="/",
nargs="?",
help="Sub url to open in the browser",
help="Sub URL to open in the browser",
)
# Set the args.func variable in args
# Set the args.func variable in args based on whether FastAPI is installed
if start_server is None:
parser.set_defaults(func=fastapi_is_not_installed)
else:

View File

@@ -1,55 +1,46 @@
# Imports
import logging
from contextlib import asynccontextmanager
from typing import Any
# import for sql
# Import FastAPI components and SQLAlchemy related modules
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.routing import APIRoute
from fastapi.staticfiles import StaticFiles
from sqlalchemy.exc import SQLAlchemyError
# Import configs
from ..config import cors_ports, cors_url
# Import custom modules and classes
from ..errors import ClanError
from . import sql_models
from .assets import asset_path
from .error_handlers import clan_error_handler, sql_error_handler
from .routers import endpoints, health, root, socket_manager2 # sql router hinzufügen
from .routers import endpoints, health, root
from .sql_db import engine
from .tags import tags_metadata
cors_url = [
"http://localhost",
"http://127.0.0.1",
"http://0.0.0.0",
"http://[::]",
]
cors_ports = [2979, 3000]
cors_whitelist = []
for u in cors_url:
for p in cors_ports:
cors_whitelist.append(f"{u}:{p}")
# Logging setup
log = logging.getLogger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI) -> Any:
await socket_manager2.brd.connect()
yield
await socket_manager2.brd.disconnect()
# Function to set up and configure the FastAPI application
def setup_app() -> FastAPI:
# bind sql engine
# TODO comment aut and add flag to run with pupulated data rm *.sql run pytest with marked then start clan webui
# https://docs.pytest.org/en/7.1.x/example/markers.html
# Uncomment the following line to drop existing tables during startup (if needed)
# sql_models.Base.metadata.drop_all(engine)
# Create tables in the database using SQLAlchemy
sql_models.Base.metadata.create_all(bind=engine)
app = FastAPI(lifespan=lifespan, swagger_ui_parameters={"tryItOutEnabled": True})
# Initialize FastAPI application with lifespan management
app = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
# Configure CORS middleware
app.add_middleware(
CORSMiddleware,
allow_origins=cors_whitelist,
@@ -58,31 +49,35 @@ def setup_app() -> FastAPI:
allow_headers=["*"],
)
# Include routers for various endpoints and components
app.include_router(health.router)
# sql methodes
app.include_router(endpoints.router)
app.include_router(socket_manager2.router)
# Needs to be last in register. Because of wildcard route
# Needs to be last in registration due to wildcard route
app.include_router(root.router)
# Add custom exception handlers
app.add_exception_handler(ClanError, clan_error_handler) # type: ignore
app.add_exception_handler(SQLAlchemyError, sql_error_handler) # type: ignore
# Mount the "static" route for serving static files
app.mount("/static", StaticFiles(directory=asset_path()), name="static")
# Add tag descriptions to the OpenAPI schema
app.openapi_tags = tags_metadata
# Assign operation IDs to API routes
for route in app.routes:
if isinstance(route, APIRoute):
route.operation_id = route.name # in this case, 'read_items'
log.debug(f"Registered route: {route}")
# Log registered exception handlers
for i in app.exception_handlers.items():
log.debug(f"Registered exception handler: {i}")
return app
# Create an instance of the FastAPI application
app = setup_app()

View File

@@ -0,0 +1 @@
In the <endpoints.py> are the api endpoints implemented which could be used of the user/s.

View File

@@ -27,11 +27,21 @@ router = APIRouter()
log = logging.getLogger(__name__)
# API Endpoints for all tables
# see the default api documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/DefaultApi.md
#########################
# #
# Service #
# #
#########################
# see the corresponding documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/Service.md
### pkgs/clan-cli/tests/openapi_client/docs/ServiceCreate.md
### pkgs/clan-cli/tests/openapi_client/docs/ServiceUsageCreate.md
### pkgs/clan-cli/tests/openapi_client/docs/ServicesApi.md
@router.post("/api/v1/service", response_model=Service, tags=[Tags.services])
def create_service(
service: ServiceCreate, db: Session = Depends(sql_db.get_db)
@@ -134,6 +144,10 @@ def delete_service(
# Entity #
# #
#########################
# see the corresponding documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/Entity.md
### pkgs/clan-cli/tests/openapi_client/docs/EntityCreate.md
### pkgs/clan-cli/tests/openapi_client/docs/EntitiesApi.md
@router.post("/api/v1/entity", response_model=Entity, tags=[Tags.entities])
def create_entity(
entity: EntityCreate, db: Session = Depends(sql_db.get_db)
@@ -298,6 +312,9 @@ def get_rpc_by_role(db: Session, role: Role, path: str) -> Any:
# Resolution #
# #
#########################
# see the corresponding documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/Resolution.md
### pkgs/clan-cli/tests/openapi_client/docs/ResolutionApi.md
@router.get(
"/api/v1/resolutions", response_model=List[Resolution], tags=[Tags.resolutions]
)
@@ -312,6 +329,8 @@ def get_all_resolutions(
# Repository #
# #
#########################
# see the corresponding documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/RepositoriesApi.md
@router.get(
"/api/v1/repositories", tags=[Tags.repositories], response_model=List[Service]
)
@@ -326,6 +345,10 @@ def get_all_repositories(
# Eventmessage #
# #
#########################
# see the corresponding documentation under:
### pkgs/clan-cli/tests/openapi_client/docs/Eventmessage.md
### pkgs/clan-cli/tests/openapi_client/docs/EventmessageCreate.md
### pkgs/clan-cli/tests/openapi_client/docs/EventmessageApi.md
@router.post(
"/api/v1/event_message", response_model=Eventmessage, tags=[Tags.eventmessages]
)

View File

@@ -1,52 +0,0 @@
# Requires: `starlette`, `uvicorn`, `jinja2`
# Run with `uvicorn example:app`
import logging
import os
import anyio
from broadcaster import Broadcast
from fastapi import APIRouter, WebSocket
from fastapi.responses import HTMLResponse
log = logging.getLogger(__name__)
router = APIRouter()
brd = Broadcast("memory://")
@router.get("/ws2_example")
async def get() -> HTMLResponse:
html = open(f"{os.getcwd()}/webui/routers/messenger.html").read()
return HTMLResponse(html)
@router.websocket("/ws2")
async def chatroom_ws(websocket: WebSocket) -> None:
await websocket.accept()
async with anyio.create_task_group() as task_group:
# run until first is complete
async def run_chatroom_ws_receiver() -> None:
await chatroom_ws_receiver(websocket=websocket)
task_group.cancel_scope.cancel()
task_group.start_soon(run_chatroom_ws_receiver)
log.warning("Started chatroom_ws_sender")
await chatroom_ws_sender(websocket)
async def chatroom_ws_receiver(websocket: WebSocket) -> None:
async for message in websocket.iter_text():
log.warning(f"Received message: {message}")
await brd.publish(channel="chatroom", message=message)
async def chatroom_ws_sender(websocket: WebSocket) -> None:
async with brd.subscribe(channel="chatroom") as subscriber:
if subscriber is None:
log.error("Subscriber is None")
return
async for event in subscriber: # type: ignore
await websocket.send_text(event.message)

View File

@@ -1,3 +1,4 @@
# Imports
import logging
from datetime import datetime
from typing import List
@@ -7,14 +8,23 @@ from pydantic import BaseModel, Field, validator
from . import sql_models
from .db_types import Role, Status
# Set logger
log = logging.getLogger(__name__)
# create basemodel
class Machine(BaseModel):
name: str
status: Status
### Create database schema for sql
# each section will represent an own table
# Entity, Service, Resolution, Eventmessages
# The relation between them is as follows:
# one Entity can have many Services
#########################
# #
# Entity #

View File

@@ -1,3 +1,4 @@
# Imports
import argparse
import logging
import multiprocessing as mp
@@ -10,7 +11,6 @@ from pathlib import Path
from threading import Thread
from typing import Iterator
# XXX: can we dynamically load this using nix develop?
import uvicorn
from pydantic import AnyUrl, IPvAnyAddress
from pydantic.tools import parse_obj_as
@@ -19,9 +19,11 @@ import clan_cli.config as config
from clan_cli.emulate_fastapi import apps, get_health
from clan_cli.errors import ClanError
# Setting up logging
log = logging.getLogger(__name__)
# Function to open the browser for a specified URL
def open_browser(base_url: AnyUrl, sub_url: str) -> None:
for i in range(5):
try:
@@ -33,6 +35,7 @@ def open_browser(base_url: AnyUrl, sub_url: str) -> None:
_open_browser(url)
# Helper function to open a web browser for a given URL using available browsers
def _open_browser(url: AnyUrl) -> subprocess.Popen:
for browser in ("firefox", "iceweasel", "iceape", "seamonkey"):
if shutil.which(browser):
@@ -52,6 +55,7 @@ def _open_browser(url: AnyUrl) -> subprocess.Popen:
raise ClanError("No browser found")
# Context manager to spawn the Node.js development server
@contextmanager
def spawn_node_dev_server(host: IPvAnyAddress, port: int) -> Iterator[None]:
log.info("Starting node dev server...")
@@ -78,6 +82,7 @@ def spawn_node_dev_server(host: IPvAnyAddress, port: int) -> Iterator[None]:
proc.terminate()
# Main function to start the server
def start_server(args: argparse.Namespace) -> None:
with ExitStack() as stack:
headers: list[tuple[str, str]] = []
@@ -115,6 +120,7 @@ def start_server(args: argparse.Namespace) -> None:
sql_models.Base.metadata.drop_all(engine)
if args.populate:
# pre populate the server with some test data
test_dir = Path(__file__).parent.parent.parent / "tests"
if not test_dir.is_dir():

View File

@@ -1,3 +1,4 @@
# Imports
from typing import List, Optional
from sqlalchemy import func
@@ -7,6 +8,8 @@ from sqlalchemy.sql.expression import true
from ..errors import ClanError
from . import schemas, sql_models
# Functions to manipulate the tables of the database
#########################
# #

View File

@@ -1,3 +1,4 @@
# Imports
from typing import Generator
from sqlalchemy import create_engine
@@ -7,6 +8,8 @@ from sqlalchemy.orm import Session, declarative_base, sessionmaker
URL = "sqlite:///./sql_app.db"
# Create db engine
engine = create_engine(URL, connect_args={"check_same_thread": False})
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

View File

@@ -4,15 +4,15 @@ from sqlalchemy.orm import relationship
from .db_types import Role
from .sql_db import Base
# Relationsship example
# https://dev.to/freddiemazzilli/flask-sqlalchemy-relationships-exploring-relationship-associations-igo
# SQLAlchemy model for the "entities" table
class Entity(Base):
__tablename__ = "entities"
## Queryable body ##
# Primary Key
did = Column(String, primary_key=True, index=True)
# Indexed Columns
name = Column(String, index=True, unique=True)
ip = Column(String, index=True)
network = Column(String, index=True)
@@ -21,67 +21,85 @@ class Entity(Base):
stop_health_task = Column(Boolean)
## Non queryable body ##
# In here we deposit: Not yet defined stuff
# JSON field for additional non-queryable data
other = Column(JSON)
## Relations ##
# One-to-Many relationship with "services" table
services = relationship("Service", back_populates="entity")
# One-to-Many relationship with "entity_roles" table
roles = relationship("EntityRoles", back_populates="entity")
# One-to-Many relationship with "service_usage" table
consumes = relationship("ServiceUsage", back_populates="consumer_entity")
# SQLAlchemy model for the "entity_roles" table
class EntityRoles(Base):
__tablename__ = "entity_roles"
## Queryable body ##
# Primary Key
id = Column(Integer, primary_key=True, autoincrement=True)
# Foreign Key
entity_did = Column(String, ForeignKey("entities.did"))
# Enum field for role
role = Column(Enum(Role), index=True, nullable=False) # type: ignore
## Relations ##
# Many-to-One relationship with "entities" table
entity = relationship("Entity", back_populates="roles")
# SQLAlchemy model for the "service_usage" table
class ServiceUsage(Base):
__tablename__ = "service_usage"
## Queryable body ##
# Primary Key
id = Column(Integer, primary_key=True, autoincrement=True)
# Foreign Key
consumer_entity_did = Column(String, ForeignKey("entities.did"))
# Many-to-One relationship with "entities" table
consumer_entity = relationship("Entity", back_populates="consumes")
times_consumed = Column(Integer, index=True, nullable=False)
service_uuid = Column(String, ForeignKey("services.uuid"))
# Many-to-One relationship with "services" table
service = relationship("Service", back_populates="usage")
# SQLAlchemy model for the "services" table
class Service(Base):
__tablename__ = "services"
# Queryable body
# Primary Key
uuid = Column(Text(length=36), primary_key=True, index=True)
service_name = Column(String, index=True)
service_type = Column(String, index=True)
endpoint_url = Column(String, index=True)
## Non queryable body ##
# In here we deposit: Action
# JSON fields for additional non-queryable data
other = Column(JSON)
status = Column(JSON, index=True)
action = Column(JSON, index=True)
## Relations ##
# One entity can have many services
# One-to-Many relationship with "entities" table
entity = relationship("Entity", back_populates="services")
entity_did = Column(String, ForeignKey("entities.did"))
# One-to-Many relationship with "service_usage" table
usage = relationship("ServiceUsage", back_populates="service")
# SQLAlchemy model for the "eventmessages" table
class Eventmessage(Base):
__tablename__ = "eventmessages"
## Queryable body ##
# Primary Key
id = Column(Integer, primary_key=True, autoincrement=True)
timestamp = Column(Integer, unique=True, index=True)
group = Column(Integer, index=True)
@@ -91,8 +109,10 @@ class Eventmessage(Base):
des_did = Column(String, index=True)
## Non queryable body ##
# In here we deposit: Network, Roles, Visible, etc.
# JSON field for additional non-queryable data
msg = Column(JSON)
## Relations ##
# One-to-Many relationship with "entities" table
# One entity can send many messages
# (Note: The comment is incomplete and can be extended based on the relationship)