update ui-assets.nix

Reviewed-on: #13

.gitea/workflows/ui_assets.yaml

@@ -35,8 +35,8 @@ jobs:
               export PATH=$PATH:$DEPS
 
               # Setup git config
-              git config --global user.email "ui-asset-bot@gchq.icu"
-              git config --global user.name "ui-asset-bot"
+              git config --global user.email "$BOT_EMAIL"
+              git config --global user.name "$BOT_NAME"
 
               ################################################
               #                                              #
@@ -66,3 +66,5 @@ jobs:
         env:
           MODIFIED_FILES: ${{ steps.changed-files.outputs.modified_files }}
           GITEA_TOKEN: ${{ secrets.BOT_ACCESS_TOKEN }}
+          BOT_NAME: "ui-asset-bot"
+          BOT_EMAIL: "ui-asset-bot@gchq.icu"

README.md

@@ -1,95 +1,211 @@
 # Website Template
 
-This repository is a template to build high quality websites as a team.
-The frontend uses [React NextJS
-](https://nextjs.org/) and the backend uses Python with the [Fastapi framework](https://fastapi.tiangolo.com/). To ensure API compatibility between frontend and backend an `openapi.json` file is generated from the Python backend code, which defines the REST API. This `openapi.json` file is then fed into [Orval](https://orval.dev/), which generates Typescript bindings for the Rest API. To ensure code correctness, we use [mypy](https://mypy-lang.org/) to ensure the Python code is correctly statically typed, and [pytest](https://docs.pytest.org/en/7.4.x/) for backend tests. A Continuos Integration (CI) Bot, verifies the code with previously mentioned Quality Assurance (QA) tools and blocks Pull requests if any errors arise.
-For dependency management we use the [Nix package manager](https://nixos.org/) to ensure reproducibility.
+Welcome to our website template repository! This template is designed to help you and your team build high-quality websites efficiently. We've carefully chosen the technologies to make development smooth and enjoyable. Here's what you can expect from this template:
 
-## Getting Started: Development Environment
+**Frontend**: Our frontend is powered by [React NextJS](https://nextjs.org/), a popular and versatile framework for building web applications.
 
-1. Install the Nix package manager by [downloading the nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or executing this command:
+**Backend**: For the backend, we use Python along with the [FastAPI framework](https://fastapi.tiangolo.com/). To ensure seamless communication between the frontend and backend, we generate an `openapi.json` file from the Python code, which defines the REST API. This file is then used with [Orval](https://orval.dev/) to generate TypeScript bindings for the REST API. We're committed to code correctness, so we use [mypy](https://mypy-lang.org/) to ensure that our Python code is statically typed correctly. For backend testing, we rely on [pytest](https://docs.pytest.org/en/7.4.x/).
+
+**Continuous Integration (CI)**: We've set up a CI bot that rigorously checks your code using the quality assurance (QA) tools mentioned above. If any errors are detected, it will block pull requests until they're resolved.
+
+**Dependency Management**: We use the [Nix package manager](https://nixos.org/) to manage dependencies and ensure reproducibility, making your development process more robust.
+
+## Supported Operating Systems
+
+- Linux
+- macOS
+
+# Getting Started with the Development Environment
+
+Let's get your development environment up and running:
+
+1. **Install Nix Package Manager**:
+
+   - You can install the Nix package manager by either [downloading the Nix installer](https://github.com/DeterminateSystems/nix-installer/releases) or running this command:
+     ```bash
+     curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+     ```
+
+2. **Install direnv**:
+
+   - Download the direnv package from [here](https://direnv.net/docs/installation.html) or run the following command:
+     ```bash
+     curl -sfL https://direnv.net/install.sh | bash
+     ```
+
+3. **Add direnv to your shell**:
+
+   - Direnv needs to [hook into your shell](https://direnv.net/docs/hook.html) to work.
+     You can do this by executing following command:
+
+   ```bash
+   echo 'eval "$(direnv hook zsh)"' >> ~/.zshrc && echo 'eval "$(direnv hook bash)"' >> ~/.bashrc && eval "$SHELL"
+   ```
+
+4. **Clone the Repository and Navigate**:
+
+   - Clone this repository and navigate to it.
+
+5. **Allow .envrc**:
+
+   - When you enter the directory, you'll receive an error message like this:
+     ```bash
+     direnv: error .envrc is blocked. Run `direnv allow` to approve its content
+     ```
+   - Execute `direnv allow` to automatically execute the shell script `.envrc` when entering the directory.
+
+6. **Build the Backend**:
+
+   - Go to the `pkgs/clan-cli` directory and execute:
+     ```bash
+     direnv allow
+     ```
+   - Wait for the backend to build.
+
+7. **Start the Backend Server**:
+
+   - To start the backend server, execute:
+     ```bash
+     clan webui --reload --no-open --log-level debug
+     ```
+   - The server will automatically restart if any Python files change.
+
+8. **Build the Frontend**:
+
+   - In a different shell, navigate to the `pkgs/ui` directory and execute:
+     ```bash
+     direnv allow
+     ```
+   - Wait for the frontend to build.
+
+9. **Start the Frontend**:
+   - To start the frontend, execute:
+     ```bash
+     npm run dev
+     ```
+   - Access the website by going to [http://localhost:3000](http://localhost:3000).
+
+# Setting Up Your Git Workflow
+
+Let's set up your Git workflow to collaborate effectively:
+
+1. **Register Your Gitea Account Locally**:
+
+   - Execute the following command to add your Gitea account locally:
+     ```bash
+     tea login add
+     ```
+   - Fill out the prompt as follows:
+     - URL of Gitea instance: `https://gitea.gchq.icu`
+     - Name of new Login [gitea.gchq.icu]: `gitea.gchq.icu:7171`
+     - Do you have an access token? No
+     - Username: YourUsername
+     - Password: YourPassword
+     - Set Optional settings: No
+
+2. **Git Workflow**:
+
+   1. Add your changes to Git using `git add <file1> <file2>`.
+   2. Run `nix fmt` to lint your files.
+   3. Commit your changes with a descriptive message: `git commit -a -m "My descriptive commit message"`.
+   4. Make sure your branch has the latest changes from upstream by executing:
+      ```bash
+      git fetch && git rebase origin/main --autostash
+      ```
+   5. Use `git status` to check for merge conflicts.
+   6. If conflicts exist, resolve them. Here's a tutorial for resolving conflicts in [VSCode](https://code.visualstudio.com/docs/sourcecontrol/overview#_merge-conflicts).
+   7. After resolving conflicts, execute `git merge --continue` and repeat step 5 until there are no conflicts.
+
+3. **Create a Pull Request**:
+
+   - To automatically open a pull request that gets merged if all tests pass, execute:
+     ```bash
+     merge-after-ci
+     ```
+
+4. **Review Your Pull Request**:
+
+   - Visit https://gitea.gchq.icu and go to the project page. Check under "Pull Requests" for any issues with your pull request.
+
+5. **Push Your Changes**:
+   - If there are issues, fix them and redo step 2. Afterward, execute:
+     ```bash
+     git push origin HEAD:YourUsername-main
+     ```
+   - This will directly push to your open pull request.
+
+# Debugging
+
+When working on the backend of your project, debugging is an essential part of the development process. Here are some methods for debugging and testing the backend of your application:
+
+## Test Backend Locally in Devshell with Breakpoints
+
+To test the backend locally in a development environment and set breakpoints for debugging, follow these steps:
+
+1. Run the following command to execute your tests and allow for debugging with breakpoints:
+   ```bash
+   pytest -n0 -s --maxfail=1
+   ```
+   You can place `breakpoint()` in your Python code where you want to trigger a breakpoint for debugging.
+
+## Test Backend Locally in a Nix Sandbox
+
+To run your backend tests in a Nix sandbox, you have two options depending on whether your test functions have been marked as impure or not:
+
+### Running Tests Marked as Impure
+
+If your test functions need to execute `nix build` and have been marked as impure because you can't execute `nix build` inside a Nix sandbox, use the following command:
 
 ```bash
-curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
+nix run .#impure-checks
 ```
 
-2. Install direnv by [downloading the direnv package](https://direnv.net/docs/installation.html) or executing this command:
+This command will run the impure test functions.
+
+### Running Pure Tests
+
+For test functions that have not been marked as impure and don't require executing `nix build`, you can use the following command:
 
 ```bash
-curl -sfL https://direnv.net/install.sh | bash
+nix build .#checks.x86_64-linux.clan-pytest --rebuild
 ```
 
-3. Clone the repository and cd into it
-4. You should see an error message reading like this:
+This command will run all pure test functions.
 
-```bash
-direnv: error .envrc is blocked. Run `direnv allow` to approve its content
-```
+### Inspecting the Nix Sandbox
 
-5. Execute `direnv allow` to allow automatically executing the shell script `.envrc` on entering the directory
-6. Go to `pkgs/clan-cli` and execute
+If you need to inspect the Nix sandbox while running tests, follow these steps:
 
-```bash
-direnv allow
-```
+1. Insert an endless sleep into your test code where you want to pause the execution. For example:
 
-Then wait for the backend to build  
-7. To start the backend server then execute:
+   ```python
+   import time
+   time.sleep(3600)  # Sleep for one hour
+   ```
 
-```
-clan webui --reload --no-open --log-level debug
-```
+2. Use `cntr` and `psgrep` to attach to the Nix sandbox. This allows you to interactively debug your code while it's paused. For example:
 
-The server will automatically restart if any Python file changes.  
-8. In a different shell go to `pkgs/ui` and execute
+   ```bash
+   cntr exec -w your_sandbox_name
+   psgrep -a -x your_python_process_name
+   ```
 
-```bash
-direnv allow
-```
+These debugging and testing methods will help you identify and fix issues in your backend code efficiently, ensuring the reliability and robustness of your application.
 
-Then wait for the frontend to build.  
-9. To start the frontend, execute:
+# Using this Template
 
-```bash
-npm run dev
-```
+To make the most of this template:
 
-Visit the website by going to [http://localhost:3000](http://localhost:3000)
+1. Set up a new Gitea account named `ui-asset-bot`. Generate an access token with all access permissions and set it under `settings/actions/secrets` as a secret called `BOT_ACCESS_TOKEN`.
 
-## Getting started: Setup Git Workflow
+   - Also, edit the file `.gitea/workflows/ui_assets.yaml` and change the `BOT_EMAIL` variable to match the email you set for that account. Gitea matches commits to accounts by their email address, so this step is essential.
 
-1. Register your Gitea account locally by executing
+2. Create a second Gitea account named `merge-bot`. Edit the file `pkgs/merge-after-ci/default.nix` if the name should be different. Under "Branches," set the main branch to be protected and add `merge-bot` to the whitelisted users for pushing. Set the unprotected file pattern to `**/ui-assets.nix`.
 
-```bash
-tea login add
-```
+   - Enable the status check for "build / test (pull_request)."
 
-You will then see a prompt, please fill it out like outlined below:
+3. Add both `merge-bot` and `ui-asset-bot` as collaborators.
+   - Set the option to "Delete pull request branch after merge by default."
+   - Also, set the default merge style to "Rebase then create merge commit."
 
-```
-? URL of Gitea instance:  https://gitea.gchq.icu
-? Name of new Login [gitea.gchq.icu]:  gitea.gchq.icu:7171
-? Do you have an access token? No
-? Username:  MyUserName
-? Password:  **********
-? Set Optional settings: No
-```
-
-2. First add your changes to git:
-   1. `git add <file1> <file2>` your changes
-   2. Execute `nix fmt` to lint your files
-   3. `git commit -a -m "My descriptive commit message"`
-3. To automatically open up a pull request, that gets merged if all tests pass execute:
-
-```bash
-merge-after-ci
-```
-
-4. Go to https://gitea.gchq.icu to the project page, and look under "Pull Requests" if there are any issues with it.
-5. If there are issues, fix them and redo step 2. Afterwards execute
-
-```
-git push origin HEAD:MyUserName-main
-```
-
-to directly push to your open pull request
+With this template, you're well-equipped to build and collaborate on high-quality websites efficiently. Happy coding!.

pkgs/clan-cli/README.md

@@ -56,15 +56,15 @@ Add this `launch.json` to your .vscode directory to have working breakpoints in
 
 ## Run locally single-threaded for debugging
 
-By default tests run in parallel using pytest-parallel.
-pytest-parallel however breaks `breakpoint()`. To disable it, use this:
+By default tests run in parallel using pytest-xdist.
+pytest-xdist however breaks `breakpoint()`. To disable it, use this:
 
 ```console
-pytest --workers "" -s
+pytest -n0 -s
 ```
 
 You can also run a single test like this:
 
 ```console
-pytest --workers "" -s tests/test_secrets_cli.py::test_users
+pytest -n0 -s tests/test_secrets_cli.py::test_users
 ```

pkgs/ui/nix/ui-assets.nix

@@ -1,5 +1,5 @@
 { fetchzip }:
 fetchzip {
-  url = "https://gitea.gchq.icu/api/packages/ui-asset-bot/generic/ui/1inlxk1i9q87bw00zwqyvpm3m0hdgdffhxlymzgi5wvqn9syd0cm/assets.tar.gz";
-  sha256 = "1inlxk1i9q87bw00zwqyvpm3m0hdgdffhxlymzgi5wvqn9syd0cm";
+  url = "https://gitea.gchq.icu/api/packages/ui-asset-bot/generic/Luis-nextjs-python-web-template/0wq3lxsnsr9jcjqfhmxvjzxr4zz46aimwj1m23n7zzc835b47d70/assets.tar.gz";
+  sha256 = "0wq3lxsnsr9jcjqfhmxvjzxr4zz46aimwj1m23n7zzc835b47d70";
 }

pkgs/ui/nix/update-ui-assets.sh

@@ -3,12 +3,21 @@ set -xeuo pipefail
 
 # GITEA_TOKEN
 if [[ -z "${GITEA_TOKEN:-}" ]]; then
-  echo "GITEA_TOKEN is not set"
-  echo "Go to https://gitea.gchq.icu/user/settings/applications and generate a token"
+  echo "GITEA_TOKEN is not set. Check if the secret BOT_ACCESS_TOKEN is set in the repository settings."
   exit 1
 fi
 
-DEPS=$(nix shell --inputs-from '.#' "nixpkgs#gnutar" "nixpkgs#curl" "nixpkgs#gzip" -c bash -c "echo \$PATH")
+if [[ -z "${BOT_NAME:-}" ]]; then
+  echo "Env var BOT_NAME is not set. Use the name of the bot user here."
+  exit 1
+fi
+
+if [[ -z "${GITHUB_SERVER_URL:-}" ]]; then
+  echo "Env var GITHUB_SERVER_URL is not set. Please use the Gitea base URL here."
+  exit 1
+fi
+
+DEPS=$(nix shell --inputs-from '.#' "nixpkgs#gnutar" "nixpkgs#gnused" "nixpkgs#curl" "nixpkgs#gzip" -c bash -c "echo \$PATH")
 export PATH=$PATH:$DEPS
 
 
@@ -22,11 +31,12 @@ nix build '.#ui' --out-link "$tmpdir/result"
 tar --transform 's,^\.,assets,' -czvf "$tmpdir/assets.tar.gz" -C "$tmpdir"/result/lib/node_modules/*/out .
 NAR_HASH=$(nix-prefetch-url --unpack file://<(cat "$tmpdir/assets.tar.gz"))
 
-owner=ui-asset-bot
-package_name=ui
+owner=$BOT_NAME
+package_name=$(echo -n "$GITHUB_REPOSITORY" | sed 's/\//-/g')
 package_version=$NAR_HASH
+baseurl=$GITHUB_SERVER_URL
 
-url="https://gitea.gchq.icu/api/packages/$owner/generic/$package_name/$package_version/assets.tar.gz"
+url="$baseurl/api/packages/$owner/generic/$package_name/$package_version/assets.tar.gz"
 set +x
 curl --upload-file "$tmpdir/assets.tar.gz" -X PUT "$url?token=$GITEA_TOKEN"
 set -x

pkgs/ui/shell.nix

@@ -25,7 +25,7 @@ pkgs.mkShell {
 
 
     # re-generate the api code 
-    rm -rf api openapi.json
+    rm -rf src/api openapi.json
     cp ${clanPkgs.clan-openapi}/openapi.json . 
     orval
   '';