refactor: always run evaluations in worker containers, add dev debugpy support

Removes single-container mode in preparation for subject-specific worker images. In dev mode, workers expose port 5678 and mount the bughog source for live debugger attachment.

Author: GJFR

.vscode/launch.json

@@ -4,6 +4,21 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "BugHog: Attach Worker",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}/bughog",
+                    "remoteRoot": "/app/bughog"
+                }
+            ]
+        },
         {
             "name": "BugHog",
             "type": "debugpy",

bughog/distribution/worker_manager.py

@@ -7,7 +7,7 @@
 import docker
 import docker.errors
 
-from bughog import config, worker
+from bughog import config
 from bughog.parameters import ExperimentParameters
 from bughog.version_control.state.base import State
 from bughog.web.clients import Clients
@@ -19,22 +19,14 @@ class WorkerManager:
     def __init__(self, subject_type: str, subject_name: str, max_nb_of_containers: int) -> None:
         self.max_nb_of_containers = max_nb_of_containers
 
-        if self.max_nb_of_containers == 1:
-            logger.info('Running in single container mode')
-        else:
-            self.container_id_pool = Queue(maxsize=self.max_nb_of_containers)
-            for i in range(self.max_nb_of_containers):
-                self.container_id_pool.put(i)
-            self.client = docker.from_env()
-            self.worker_image_ref = self.__get_worker_image_ref(subject_type, subject_name)
+        self.container_id_pool = Queue(maxsize=self.max_nb_of_containers)
+        for i in range(self.max_nb_of_containers):
+            self.container_id_pool.put(i)
+        self.client = docker.from_env()
+        self.worker_image_ref = self.__get_worker_image_ref(subject_type, subject_name)
 
     def start_experiment(self, params: ExperimentParameters, state: State, blocking_wait=True) -> None:
-        if self.max_nb_of_containers != 1:
-            return self.__run_container(params, state, blocking_wait)
-
-        # Single container mode
-        worker.run(params, state)
-        Clients.push_results_to_all()
+        return self.__run_container(params, state, blocking_wait)
 
     def __run_container(self, params: ExperimentParameters, state: State, blocking_wait=True) -> None:
         try:
@@ -71,8 +63,22 @@ def start_container_thread():
             except docker.errors.APIError:
                 logger.error('Could not consult list of active containers', exc_info=True)
 
+            debug = bool(os.getenv('DEVELOPMENT'))
             container = None
             try:
+                volumes = [
+                    os.path.join(host_pwd, '.devcontainer') + ':/app/.devcontainer:ro',
+                    os.path.join(host_pwd, '.vscode') + ':/app/.vscode:ro',
+                    os.path.join(host_pwd, 'config') + ':/app/config:ro',
+                    os.path.join(host_pwd, 'subject') + ':/app/subject:rw',
+                    os.path.join(host_pwd, 'logs') + ':/app/logs:rw',
+                    os.path.join(host_pwd, 'nginx/ssl') + ':/etc/nginx/ssl:ro',
+                ]
+                debug_kwargs = {}
+                if debug:
+                    volumes.append(os.path.join(host_pwd, 'bughog') + ':/app/bughog:rw')
+                    debug_kwargs['ports'] = {'5678/tcp': 5678}
+                    debug_kwargs['environment'] = {'DEVELOPMENT': '1'}
                 container = self.client.containers.run(
                     self.worker_image_ref,
                     name=container_name,
@@ -82,13 +88,9 @@ def start_container_thread():
                     detach=True,
                     labels=['bh_worker'],
                     command=[params.serialize(), state.serialize()],
-                    volumes=[
-                        os.path.join(host_pwd, 'config') + ':/app/config:ro',
-                        os.path.join(host_pwd, 'subject') + ':/app/subject:rw',
-                        os.path.join(host_pwd, 'logs') + ':/app/logs:rw',
-                        os.path.join(host_pwd, 'nginx/ssl') + ':/etc/nginx/ssl:ro',
-                    ],
+                    volumes=volumes,
                     tmpfs={'/memory': 'exec,size=3g,mode=1777'},
+                    **debug_kwargs,
                 )
                 result = container.wait()
                 if result['StatusCode'] != 0:

bughog/worker.py

@@ -2,6 +2,8 @@
 import os
 import sys
 
+import debugpy
+
 from bughog.config import Loggers
 from bughog.database.mongo.mongodb import MongoDB
 from bughog.evaluation.evaluation import Evaluation
@@ -19,6 +21,11 @@ def __run_by_worker() -> None:
     Should only be called by worker.
     """
     Loggers.configure_loggers()
+    if os.getenv('DEVELOPMENT'):
+        debugpy.listen(('0.0.0.0', 5678))
+        logger.info('Waiting for debugger to attach on port 5678...')
+        debugpy.wait_for_client()
+        logger.info('Debugger attached.')
     if len(sys.argv) < 3:
         logger.info('Worker did not receive enough arguments.')
         os._exit(0)

scripts/boot/worker.sh

@@ -7,5 +7,9 @@ source /app/scripts/boot/setup_environment.sh
 rm -f /tmp/Xvfb.pid
 service xvfb start
 
-uv sync --no-dev --locked
+if [ -n "$DEVELOPMENT" ]; then
+    uv sync --locked
+else
+    uv sync --no-dev --locked
+fi
 exec python3 /app/bughog/worker.py "$@"