From 276eba73c3b2cb14cf3c24ced7e9ccee64266f22 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 16:06:12 -0400 Subject: [PATCH 01/42] Extract reusable shot queue widget Add a reusable ShotQueueWidget and ShotQueueTreeView in labscript_utils.qtwidgets for queue editing and state restore. --- labscript_utils/qtwidgets/__init__.py | 4 +- labscript_utils/qtwidgets/shotqueue.py | 359 +++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 labscript_utils/qtwidgets/shotqueue.py diff --git a/labscript_utils/qtwidgets/__init__.py b/labscript_utils/qtwidgets/__init__.py index af02b0ad..5ec168d4 100644 --- a/labscript_utils/qtwidgets/__init__.py +++ b/labscript_utils/qtwidgets/__init__.py @@ -9,4 +9,6 @@ # BSD License. See the license.txt file in the root of the project # # for the full license. # # # -##################################################################### \ No newline at end of file +##################################################################### + +from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py new file mode 100644 index 00000000..5b2cacc3 --- /dev/null +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -0,0 +1,359 @@ +##################################################################### +# # +# shotqueue.py # +# # +# Copyright 2026, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import os + +from qtutils.qt.QtCore import * +from qtutils.qt.QtGui import * +from qtutils.qt.QtWidgets import * +from qtutils.qt.QtCore import pyqtSignal as Signal + + +FILEPATH_COLUMN = 0 +__all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] + + +class ShotQueueTreeView(QTreeView): + """Generic queue view with delete-key handling and shot-file drops.""" + + deleteRequested = Signal() + filesDropped = Signal(list) + + def __init__(self, parent=None, accepted_extensions=None): + QTreeView.__init__(self, parent) + self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self.header().setStretchLastSection(True) + self.setAutoScroll(False) + self.setAcceptDrops(True) + self.setDragEnabled(True) + self.setDropIndicatorShown(True) + self.setDragDropMode(QAbstractItemView.DragDrop) + self.setDefaultDropAction(Qt.MoveAction) + self.setSelectionBehavior(QAbstractItemView.SelectRows) + self.setSelectionMode(QAbstractItemView.ExtendedSelection) + self.setRootIsDecorated(False) + self.setUniformRowHeights(True) + + def accepted_extensions(self): + return tuple(self._accepted_extensions) + + def set_accepted_extensions(self, accepted_extensions): + self._accepted_extensions = self._normalise_extensions(accepted_extensions) + + def keyPressEvent(self, event): + if event.key() == Qt.Key_Delete: + event.accept() + self.deleteRequested.emit() + return + QTreeView.keyPressEvent(self, event) + + def dragEnterEvent(self, event): + if self._event_has_acceptable_urls(event): + event.setDropAction(Qt.CopyAction) + event.accept() + return + QTreeView.dragEnterEvent(self, event) + + def dragMoveEvent(self, event): + if self._event_has_acceptable_urls(event): + event.setDropAction(Qt.CopyAction) + event.accept() + return + QTreeView.dragMoveEvent(self, event) + + def dropEvent(self, event): + if self._event_has_acceptable_urls(event): + paths = self._paths_from_urls(event.mimeData().urls()) + if paths: + event.setDropAction(Qt.CopyAction) + event.accept() + self.filesDropped.emit(paths) + return + QTreeView.dropEvent(self, event) + + def _event_has_acceptable_urls(self, event): + if not event.mimeData().hasUrls(): + return False + return bool(self._paths_from_urls(event.mimeData().urls())) + + def _paths_from_urls(self, urls): + paths = [] + for url in urls: + if not url.isLocalFile(): + continue + path = os.path.abspath(str(url.toLocalFile())) + if self._is_accepted_path(path): + paths.append(path) + return paths + + def _is_accepted_path(self, path): + return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) + + def _normalise_extensions(self, accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + return tuple(extension.lower() for extension in accepted_extensions) + + +class ShotQueueWidget(QWidget): + """Reusable single-column shot queue editor widget.""" + + queueChanged = Signal() + selectionChanged = Signal(list) + filesAdded = Signal(list) + + def __init__( + self, + parent=None, + accepted_extensions=None, + file_dialog_filter='Shot files (*.h5 *.hdf5)', + allow_duplicates=False, + column_title='Filepath', + ): + QWidget.__init__(self, parent) + self.accepted_extensions = self._normalise_extensions(accepted_extensions) + self.file_dialog_filter = file_dialog_filter + self.allow_duplicates = allow_duplicates + self.last_opened_shots_folder = '' + + self.queue_model = QStandardItemModel(self) + self.queue_model.setHorizontalHeaderItem(FILEPATH_COLUMN, QStandardItem(column_title)) + + self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) + self.queue_view.setModel(self.queue_model) + + self.add_button = QToolButton(self) + self.add_button.setText('Add') + self.delete_button = QToolButton(self) + self.delete_button.setText('Delete') + self.clear_button = QToolButton(self) + self.clear_button.setText('Clear') + self.move_top_button = QToolButton(self) + self.move_top_button.setText('Top') + self.move_up_button = QToolButton(self) + self.move_up_button.setText('Up') + self.move_down_button = QToolButton(self) + self.move_down_button.setText('Down') + self.move_bottom_button = QToolButton(self) + self.move_bottom_button.setText('Bottom') + + button_layout = QHBoxLayout() + button_layout.setContentsMargins(0, 0, 0, 0) + button_layout.addWidget(self.add_button) + button_layout.addWidget(self.delete_button) + button_layout.addWidget(self.clear_button) + button_layout.addStretch(1) + button_layout.addWidget(self.move_top_button) + button_layout.addWidget(self.move_up_button) + button_layout.addWidget(self.move_down_button) + button_layout.addWidget(self.move_bottom_button) + + layout = QVBoxLayout(self) + layout.setContentsMargins(0, 0, 0, 0) + layout.addWidget(self.queue_view) + layout.addLayout(button_layout) + + self.add_button.clicked.connect(self.prompt_for_files) + self.delete_button.clicked.connect(self.remove_selected) + self.clear_button.clicked.connect(self.clear) + self.move_top_button.clicked.connect(self.move_top) + self.move_up_button.clicked.connect(self.move_up) + self.move_down_button.clicked.connect(self.move_down) + self.move_bottom_button.clicked.connect(self.move_bottom) + self.queue_view.deleteRequested.connect(self.remove_selected) + self.queue_view.filesDropped.connect(self.add_files) + self.queue_model.rowsInserted.connect(self._on_queue_changed) + self.queue_model.rowsRemoved.connect(self._on_queue_changed) + self.queue_model.modelReset.connect(self._on_queue_changed) + self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) + + self._update_button_states() + + def files(self): + return [ + self.queue_model.item(row, FILEPATH_COLUMN).text() + for row in range(self.queue_model.rowCount()) + ] + + def selected_files(self): + return [self.queue_model.item(row, FILEPATH_COLUMN).text() for row in self.selected_rows()] + + def selected_rows(self): + return sorted(index.row() for index in self.queue_view.selectionModel().selectedRows()) + + def select_paths(self, paths): + if isinstance(paths, str): + paths = [paths] + path_lookup = {os.path.abspath(str(path)) for path in paths} + rows = [] + for row in range(self.queue_model.rowCount()): + item = self.queue_model.item(row, FILEPATH_COLUMN) + if item.text() in path_lookup: + rows.append(row) + self._select_rows(rows) + + def set_files(self, paths): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + self.append(paths) + + def append(self, paths): + paths = self._prepare_paths(paths) + if not paths: + return [] + for path in paths: + self.queue_model.appendRow(self._create_row(path)) + self.filesAdded.emit(paths) + return paths + + def prepend(self, path): + paths = self._prepare_paths([path]) + if not paths: + return [] + self.queue_model.insertRow(0, self._create_row(paths[0])) + self.filesAdded.emit(paths) + self._select_rows([0]) + return paths + + def add_files(self, paths): + added = self.append(paths) + if added: + self.last_opened_shots_folder = os.path.dirname(added[-1]) + first_new_row = self.queue_model.rowCount() - len(added) + self._select_rows(range(first_new_row, self.queue_model.rowCount())) + return added + + def prompt_for_files(self): + file_names = QFileDialog.getOpenFileNames( + self, + 'Select shot file(s) to add to the queue', + self.last_opened_shots_folder, + self.file_dialog_filter, + ) + if isinstance(file_names, tuple): + file_names, _ = file_names + return self.add_files(file_names) + + def remove_selected(self): + for row in reversed(self.selected_rows()): + self.queue_model.removeRow(row) + + def clear(self): + if self.queue_model.rowCount(): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + + def move_up(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[0] == 0: + return + for row_index in selected_rows: + self.queue_model.insertRow(row_index - 1, self.queue_model.takeRow(row_index)) + self._select_rows([row - 1 for row in selected_rows]) + + def move_down(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: + return + for row_index in reversed(selected_rows): + self.queue_model.insertRow(row_index + 1, self.queue_model.takeRow(row_index)) + self._select_rows([row + 1 for row in selected_rows]) + + def move_top(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[0] == 0: + return + rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] + for offset, row_items in enumerate(reversed(rows)): + self.queue_model.insertRow(offset, row_items) + self._select_rows(range(len(selected_rows))) + + def move_bottom(self): + selected_rows = self.selected_rows() + if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: + return + rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] + start_row = self.queue_model.rowCount() + for row_items in reversed(rows): + self.queue_model.appendRow(row_items) + self._select_rows(range(start_row, self.queue_model.rowCount())) + + def is_in_queue(self, path): + path = os.path.abspath(str(path)) + return bool(self.queue_model.findItems(path, column=FILEPATH_COLUMN)) + + def get_save_data(self): + return { + 'files_queued': self.files(), + 'last_opened_shots_folder': self.last_opened_shots_folder, + } + + def restore_save_data(self, data): + self.last_opened_shots_folder = data.get('last_opened_shots_folder', '') + self.set_files(data.get('files_queued', [])) + + def _create_row(self, path): + item = QStandardItem(path) + item.setToolTip(path) + item.setEditable(False) + return [item] + + def _prepare_paths(self, paths): + if isinstance(paths, str): + paths = [paths] + existing = set(self.files()) + pending = set() + prepared = [] + for path in paths: + path = os.path.abspath(str(path)) + if not self._is_accepted_path(path): + continue + if not self.allow_duplicates and (path in existing or path in pending): + continue + pending.add(path) + prepared.append(path) + return prepared + + def _is_accepted_path(self, path): + return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) + + def _on_queue_changed(self, *args): + self._update_button_states() + self.queueChanged.emit() + + def _on_selection_changed(self, *args): + self._update_button_states() + self.selectionChanged.emit(self.selected_files()) + + def _update_button_states(self): + row_count = self.queue_model.rowCount() + selected_rows = self.selected_rows() + has_selection = bool(selected_rows) + self.delete_button.setEnabled(has_selection) + self.clear_button.setEnabled(bool(row_count)) + self.move_top_button.setEnabled(has_selection and selected_rows[0] > 0) + self.move_up_button.setEnabled(has_selection and selected_rows[0] > 0) + self.move_bottom_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) + self.move_down_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) + + def _select_rows(self, rows): + rows = list(rows) + selection_model = self.queue_view.selectionModel() + selection_model.clearSelection() + for row in rows: + index = self.queue_model.index(row, FILEPATH_COLUMN) + selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) + if rows: + self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) + + def _normalise_extensions(self, accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + return tuple(extension.lower() for extension in accepted_extensions) From cbbcc4f0c36e0c1b9905212ff26fc8c877998f2c Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 16:59:40 -0400 Subject: [PATCH 02/42] Fix shot queue widget edge cases Normalize accepted_extensions consistently so a single string like '.h5' is treated as one extension instead of being iterated character by character. Make labscript_utils.qtwidgets lazily expose the new shot queue classes so importing the package no longer eagerly imports Qt-dependent code. --- labscript_utils/qtwidgets/__init__.py | 21 ++++++++++++++++++++- labscript_utils/qtwidgets/shotqueue.py | 25 +++++++++++-------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/labscript_utils/qtwidgets/__init__.py b/labscript_utils/qtwidgets/__init__.py index 5ec168d4..3f90f1c0 100644 --- a/labscript_utils/qtwidgets/__init__.py +++ b/labscript_utils/qtwidgets/__init__.py @@ -11,4 +11,23 @@ # # ##################################################################### -from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget +__all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] + + +def __getattr__(name): + if name not in __all__: + raise AttributeError("module %r has no attribute %r" % (__name__, name)) + + from .shotqueue import FILEPATH_COLUMN, ShotQueueTreeView, ShotQueueWidget + + exports = { + 'FILEPATH_COLUMN': FILEPATH_COLUMN, + 'ShotQueueTreeView': ShotQueueTreeView, + 'ShotQueueWidget': ShotQueueWidget, + } + globals().update(exports) + return exports[name] + + +def __dir__(): + return sorted(list(globals().keys()) + __all__) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 5b2cacc3..df95b23b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -22,6 +22,14 @@ __all__ = ['FILEPATH_COLUMN', 'ShotQueueTreeView', 'ShotQueueWidget'] +def _normalise_extensions(accepted_extensions): + if accepted_extensions is None: + accepted_extensions = ('.h5', '.hdf5') + elif isinstance(accepted_extensions, str): + accepted_extensions = (accepted_extensions,) + return tuple(extension.lower() for extension in accepted_extensions) + + class ShotQueueTreeView(QTreeView): """Generic queue view with delete-key handling and shot-file drops.""" @@ -30,7 +38,7 @@ class ShotQueueTreeView(QTreeView): def __init__(self, parent=None, accepted_extensions=None): QTreeView.__init__(self, parent) - self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self._accepted_extensions = _normalise_extensions(accepted_extensions) self.header().setStretchLastSection(True) self.setAutoScroll(False) self.setAcceptDrops(True) @@ -47,7 +55,7 @@ def accepted_extensions(self): return tuple(self._accepted_extensions) def set_accepted_extensions(self, accepted_extensions): - self._accepted_extensions = self._normalise_extensions(accepted_extensions) + self._accepted_extensions = _normalise_extensions(accepted_extensions) def keyPressEvent(self, event): if event.key() == Qt.Key_Delete: @@ -98,12 +106,6 @@ def _paths_from_urls(self, urls): def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) - def _normalise_extensions(self, accepted_extensions): - if accepted_extensions is None: - accepted_extensions = ('.h5', '.hdf5') - return tuple(extension.lower() for extension in accepted_extensions) - - class ShotQueueWidget(QWidget): """Reusable single-column shot queue editor widget.""" @@ -120,7 +122,7 @@ def __init__( column_title='Filepath', ): QWidget.__init__(self, parent) - self.accepted_extensions = self._normalise_extensions(accepted_extensions) + self.accepted_extensions = _normalise_extensions(accepted_extensions) self.file_dialog_filter = file_dialog_filter self.allow_duplicates = allow_duplicates self.last_opened_shots_folder = '' @@ -352,8 +354,3 @@ def _select_rows(self, rows): selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) if rows: self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) - - def _normalise_extensions(self, accepted_extensions): - if accepted_extensions is None: - accepted_extensions = ('.h5', '.hdf5') - return tuple(extension.lower() for extension in accepted_extensions) From e44fbe1c417a9cc0ed604e797534c7df7d112496 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:32:28 -0400 Subject: [PATCH 03/42] Make excepthook warning hook idempotent --- labscript_utils/excepthook/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labscript_utils/excepthook/__init__.py b/labscript_utils/excepthook/__init__.py index 9ac5975d..c6551fc8 100644 --- a/labscript_utils/excepthook/__init__.py +++ b/labscript_utils/excepthook/__init__.py @@ -28,6 +28,7 @@ class l: logger = None child_processes = [] +_original_showwarning = warnings.showwarning def install_thread_excepthook(): @@ -83,12 +84,11 @@ def tkhandler(exceptclass, exception, exec_info, reraise=True): def logwarning(message, category, filename, lineno, file=None, line=None): logmessage = warnings.formatwarning(message, category, filename, lineno, line) l.logger.warn(logmessage) - warnings._showwarning(message, category, filename, lineno, file, line) + _original_showwarning(message, category, filename, lineno, file, line) def set_logger(logger): l.logger = logger - warnings._showwarning = warnings.showwarning warnings.showwarning = logwarning # Check for tkinter availability. Tkinter is frustratingly not available From b522ed3ea3e1a5208c6100101a34933630dcf193 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:34:14 -0400 Subject: [PATCH 04/42] Fix SecureSocket detection in ls_zprocess Context --- labscript_utils/ls_zprocess.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/labscript_utils/ls_zprocess.py b/labscript_utils/ls_zprocess.py index 4d9e9575..29c94d52 100644 --- a/labscript_utils/ls_zprocess.py +++ b/labscript_utils/ls_zprocess.py @@ -257,7 +257,7 @@ def socket(self, socket_type, socket_class=None, **kwargs): # be a SecureSocket. If caller has explicitly requested a different socket type # (e.g since pyzmq 25, ThreadAuthenticator sets up an internal socket by calling # `Context.socket(..., socket_class=zmq.Socket)), then don't.` - if socket_class is None or issubclass(socket_class, SecureContext): + if socket_class is None or issubclass(socket_class, SecureSocket): config = get_config() kwargs['allow_insecure'] = config['allow_insecure'] return SecureContext.socket(self, socket_type=socket_type, **kwargs) @@ -383,4 +383,3 @@ def ensure_connected_to_zlog(): else: client.ping() _connected_to_zlog = True - From 502563328a5d320c71b8efec2bee0008fa6240fa Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 17 Mar 2026 19:35:04 -0400 Subject: [PATCH 05/42] Support NumPy scalar JSON serialization --- labscript_utils/properties.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/labscript_utils/properties.py b/labscript_utils/properties.py index 8c2402fe..d31e7245 100644 --- a/labscript_utils/properties.py +++ b/labscript_utils/properties.py @@ -62,6 +62,10 @@ def _default(o): # Workaround for https://bugs.python.org/issue24313 if isinstance(o, np.integer): return int(o) + if isinstance(o, np.floating): + return float(o) + if isinstance(o, np.bool_): + return bool(o) raise TypeError From 5ccbfba447e006feecc21fa7d2ce796f0c7629b9 Mon Sep 17 00:00:00 2001 From: spielman Date: Wed, 18 Mar 2026 09:39:43 -0400 Subject: [PATCH 06/42] Configure shared Qt application defaults --- .../default_profile/labconfig/example.ini | 2 ++ labscript_utils/splash.py | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini index 7f219dc1..94a6c16f 100644 --- a/labscript_profile/default_profile/labconfig/example.ini +++ b/labscript_profile/default_profile/labconfig/example.ini @@ -16,6 +16,8 @@ connection_table_py = %(labscriptlib)s\connection_table.py [servers] zlock = localhost runmanager = localhost +blacs = localhost +lyse = localhost [ports] BLACS = 42517 diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 51c56864..a80ab333 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,32 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + + +def get_qapplication(argv=None): + qapplication = QtWidgets.QApplication.instance() + if qapplication is None: + qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) + return configure_qapplication(qapplication) + + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -50,9 +76,7 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = QtWidgets.QApplication.instance() - if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + self.qapplication = get_qapplication() super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 805d94e5b1a57e2804bae8e3e694119035bf4884 Mon Sep 17 00:00:00 2001 From: spielman Date: Wed, 25 Mar 2026 13:55:30 -0400 Subject: [PATCH 07/42] Fix TOML globals profile bootstrap --- labscript_profile/create.py | 8 +++++--- labscript_profile/default_profile/labconfig/example.ini | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index 38a67734..c36d4e4c 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -4,6 +4,7 @@ import configparser from pathlib import Path from subprocess import check_output +import h5py from labscript_profile import LABSCRIPT_SUITE_PROFILE, default_labconfig_path import argparse @@ -84,8 +85,9 @@ def compile_connection_table(): output_h5_path = os.path.expandvars(config['paths']['connection_table_h5']) # create output directory, if needed Path(output_h5_path).parent.mkdir(parents=True, exist_ok=True) - # compile the h5 file - runmanager.new_globals_file(output_h5_path) + # Create a fresh HDF5 target for the compile output. + with h5py.File(output_h5_path, 'w'): + pass def dummy_callback(success): pass @@ -164,4 +166,4 @@ def create_profile(apparatus_name = None, compile_table = False): if compile_table: print('Compiling the example connection table') - compile_connection_table() \ No newline at end of file + compile_connection_table() diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini index 94a6c16f..c959a192 100644 --- a/labscript_profile/default_profile/labconfig/example.ini +++ b/labscript_profile/default_profile/labconfig/example.ini @@ -43,7 +43,7 @@ save_git_info = False [BLACS/plugins] connection_table = True -connection_table.hashable_types = ['.py', '.txt', '.ini', '.json'] +connection_table.hashable_types = ['.py', '.txt', '.toml', '.ini', '.json'] connection_table.polling_interval = 1.0 delete_repeated_shots = False From dee0033cf8b3b09d774bac4f0caf31fbb6baaac5 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 13:36:41 -0400 Subject: [PATCH 08/42] Migrate shared config handling to TOML --- NEWS.md | 9 +- README.md | 6 + docs/source/labconfig.rst | 16 +- labscript_profile/__init__.py | 130 ++++++++-- labscript_profile/create.py | 57 +++-- .../default_profile/labconfig/example.ini | 66 ------ .../default_profile/labconfig/example.toml | 64 +++++ labscript_profile/toml_config.py | 80 +++++++ labscript_utils/filewatcher.py | 2 +- labscript_utils/labconfig.py | 222 +++++++++++++----- pyproject.toml | 2 + 11 files changed, 484 insertions(+), 170 deletions(-) delete mode 100644 labscript_profile/default_profile/labconfig/example.ini create mode 100644 labscript_profile/default_profile/labconfig/example.toml create mode 100644 labscript_profile/toml_config.py diff --git a/NEWS.md b/NEWS.md index 7b5fa25b..353814b5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,10 @@ +## Unreleased + +- TOML-backed labconfig and app-config writing now depends on `tomli-w`. The package + dependency is declared in `labscript-utils`, but existing editable or conda + environments may need that dependency installed explicitly when upgrading. + + ## [2.15.0] - 2019-12-04 This release includes one bugfix, one enhancement, one update for compatibility with a @@ -60,4 +67,4 @@ compatibility with a newer version of a library. ([PR #91](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/91)) - Compatibility with `importlib_metadata` >= 0.21. Contributed by Chris Billington - ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88)) \ No newline at end of file + ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88)) diff --git a/README.md b/README.md index eb133843..f63c5b26 100644 --- a/README.md +++ b/README.md @@ -19,3 +19,9 @@ Shared modules used by the [*labscript suite*](https://github.com/labscript-suit ## Installation labscript-utils is distributed as a Python package on [PyPI](https://pypi.org/user/labscript-suite) and [Anaconda Cloud](https://anaconda.org/labscript-suite), and should be installed with other components of the _labscript suite_. Please see the [installation guide](https://docs.labscriptsuite.org/en/latest/installation) for details. + +TOML-backed configuration support depends on `tomli-w` at runtime. This dependency is +declared in `labscript-utils` package metadata, so fresh installs should receive it +automatically. Existing editable or conda environments created before this dependency +was added may need `conda install tomli-w` or an updated `labscript-utils` +installation. diff --git a/docs/source/labconfig.rst b/docs/source/labconfig.rst index fa727640..f342a17b 100644 --- a/docs/source/labconfig.rst +++ b/docs/source/labconfig.rst @@ -1,19 +1,19 @@ -The *labconfig.ini* file +The *labconfig.toml* file ======================== -The `labconfig.ini` file is a global configuration file for your **labscript-suite** installation. +The `labconfig.toml` file is a global configuration file for your **labscript-suite** installation. It contains configurable settings that govern how the individual components of the suite operate. The name of this file must be the host computer's system name. -So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.ini`. +So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.toml`. This file should be located in the `labscript-suite` directory in the user space, in the `labconfig` subdirectory. When :doc:`installing the **labscript-suite** for the first time `, running the `labscript-profile-create` command will automatically generate the `labscript-suite` user space directory in the correct place -and generate a `labconfig.ini` file for use on your system. -By editing the `ini` file named after your system, you can update the configuration settings of your **labscript-suite** installation. +and generate a `labconfig.toml` file for use on your system. +By editing the TOML file named after your system, you can update the configuration settings of your **labscript-suite** installation. -The Default *labconfig.ini* ---------------------------- +The Default *labconfig.toml* +---------------------------- Below is a copy of the default lab configuration if you were to install the **labscript-suite** today. @@ -23,5 +23,5 @@ Below is a copy of the default lab configuration if you were to install the **la Instead, if keys are missing from your local profile, default behavior will be assumed. To implement the added functionality, you will need to manually add/change the keys in your local labconfig. -.. include:: ../../labscript_profile/default_profile/labconfig/example.ini +.. include:: ../../labscript_profile/default_profile/labconfig/example.toml :code: diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 55571f78..a45a5744 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -1,12 +1,14 @@ import site import sys import os -from configparser import ConfigParser, NoSectionError, NoOptionError +import configparser +from ast import literal_eval from pathlib import Path from subprocess import check_output import socket from getpass import getuser +from .toml_config import TomlConfigParser, dump_toml_file, load_toml_file # The contents of this file are imported every time the Python interpreter starts up, # owing to our custom .pth file that runs the below two functions. This ensures that @@ -36,6 +38,13 @@ def hostname(): def default_labconfig_path(): + if LABSCRIPT_SUITE_PROFILE is None: + return None + return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.toml' + + +def legacy_labconfig_path(): + """Temporary legacy INI labconfig location. Slated for removal soon.""" if LABSCRIPT_SUITE_PROFILE is None: return None return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.ini' @@ -47,17 +56,110 @@ def add_userlib_and_pythonlib(): re-implements finding and reading the config file so as to not import labscript_utils, since we dont' want to import something like labscript_utils every time the interpreter starts up""" - labconfig = default_labconfig_path() - if labconfig is not None and labconfig.exists(): - # str() below is for py36 compat, where ConfigParser can't deal with Path objs - config = ConfigParser( - defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)} - ) - config.read(labconfig) - for option in ['userlib', 'pythonlib']: - try: - paths = config.get('DEFAULT', option).split(',') - except (NoSectionError, NoOptionError): - paths = [] - for path in paths: + labconfig = ensure_labconfig() + if labconfig is None: + return + if not labconfig.exists(): + return + config = TomlConfigParser(defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)}) + config.read_toml(labconfig) + for option in ['userlib', 'pythonlib']: + try: + paths = config.get('DEFAULT', option) + except (configparser.NoSectionError, configparser.NoOptionError): + paths = '' + if paths: + for path in paths.split(','): site.addsitedir(path) + + +def ensure_labconfig(): + """Return the TOML labconfig path, converting a legacy INI file if needed. + + The legacy conversion path is temporary and slated for removal soon. + """ + labconfig = default_labconfig_path() + if labconfig is None: + return None + if labconfig.exists(): + return labconfig + legacy = legacy_labconfig_path() + if legacy is None or not legacy.exists(): + return labconfig + _convert_legacy_labconfig(legacy, labconfig) + return labconfig + + +def _example_labconfig_data(): + if LABSCRIPT_SUITE_PROFILE is not None: + profile_example = LABSCRIPT_SUITE_PROFILE / 'labconfig' / 'example.toml' + if profile_example.exists(): + return load_toml_file(profile_example) + bundled_example = Path(__file__).resolve().parent / 'default_profile' / 'labconfig' / 'example.toml' + if bundled_example.exists(): + return load_toml_file(bundled_example) + return {} + + +def _coerce_legacy_value(value, template_value): + if template_value is None: + return value + if isinstance(template_value, bool): + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in configparser.ConfigParser.BOOLEAN_STATES: + return configparser.ConfigParser.BOOLEAN_STATES[lowered] + return value + if isinstance(template_value, int): + try: + return int(value) + except (TypeError, ValueError): + return value + if isinstance(template_value, float): + try: + return float(value) + except (TypeError, ValueError): + return value + if isinstance(template_value, list): + if not isinstance(value, str): + return value + try: + parsed = literal_eval(value) + except (SyntaxError, ValueError): + return value + if not isinstance(parsed, (list, tuple)): + return value + if not template_value: + return list(parsed) + template_item = template_value[0] + return [_coerce_legacy_value(item, template_item) for item in parsed] + return value + + +def _convert_legacy_labconfig(legacy_path, toml_path): + """Temporary legacy migration path from INI to TOML. Slated for removal soon.""" + config = configparser.ConfigParser(interpolation=None) + config.read(legacy_path) + template_data = _example_labconfig_data() + default_template = template_data.get('DEFAULT', {}) + data = { + 'DEFAULT': { + key: _coerce_legacy_value(value, default_template.get(key)) + for key, value in config.defaults().items() + } + } + for section in config.sections(): + section_items = dict(config._sections[section]) + section_items.pop('__name__', None) + autoload = section_items.get('autoload_config_file') + if isinstance(autoload, str) and autoload.lower().endswith('.ini'): + section_items['autoload_config_file'] = autoload[:-4] + '.toml' + section_template = template_data.get(section, {}) + data[section] = { + key: _coerce_legacy_value(value, section_template.get(key)) + for key, value in section_items.items() + } + toml_path.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(toml_path, data) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index c36d4e4c..2ca7e46c 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -1,17 +1,31 @@ import sys import os import shutil -import configparser from pathlib import Path from subprocess import check_output import h5py -from labscript_profile import LABSCRIPT_SUITE_PROFILE, default_labconfig_path +from labscript_profile import ( + LABSCRIPT_SUITE_PROFILE, + default_labconfig_path, + legacy_labconfig_path, +) import argparse +from labscript_profile.toml_config import dump_toml_file, load_toml_file _here = os.path.dirname(os.path.abspath(__file__)) DEFAULT_PROFILE_CONTENTS = os.path.join(_here, 'default_profile') +def _replace_backslashes(value): + if isinstance(value, dict): + return {key: _replace_backslashes(child) for key, child in value.items()} + if isinstance(value, list): + return [_replace_backslashes(child) for child in value] + if isinstance(value, str): + return value.replace('\\', os.path.sep) + return value + + def make_shared_secret(directory): """Create a new zprocess shared secret file in the given directory and return its filepath""" @@ -32,37 +46,33 @@ def make_labconfig_file(apparatus_name = None): Overrides the default apparatus name with the provided one if not None """ - source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.ini') + source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.toml') target_path = default_labconfig_path() - if os.path.exists(target_path): + legacy_path = legacy_labconfig_path() + if os.path.exists(target_path) or (legacy_path is not None and os.path.exists(legacy_path)): raise FileExistsError(target_path) - with open(source_path) as infile, open(target_path, 'w') as outfile: - data = infile.read() - data = data.replace('\\', os.path.sep) - outfile.write(data) - - # Now change some things about it: - config = configparser.ConfigParser(interpolation=None) - config.read(target_path) + config = load_toml_file(source_path) + if os.path.sep != '\\': + config = _replace_backslashes(config) if sys.platform == 'linux': - config.set('programs', 'text_editor', 'gedit') + config['programs']['text_editor'] = 'gedit' elif sys.platform == 'darwin': - config.set('programs', 'text_editor', 'open') - config.set('programs', 'text_editor_arguments', '-a TextEdit {file}') + config['programs']['text_editor'] = 'open' + config['programs']['text_editor_arguments'] = '-a TextEdit {file}' if sys.platform != 'win32': - config.set('programs', 'hdf5_viewer', 'hdfview') - config.set('DEFAULT', 'shared_drive', '$HOME/labscript_shared') + config['programs']['hdf5_viewer'] = 'hdfview' + config['DEFAULT']['shared_drive'] = '$HOME/labscript_shared' shared_secret = make_shared_secret(target_path.parent) shared_secret_entry = Path( '%(labscript_suite)s', shared_secret.relative_to(LABSCRIPT_SUITE_PROFILE) ) - config.set('security', 'shared_secret', str(shared_secret_entry)) + config['security']['shared_secret'] = str(shared_secret_entry) if apparatus_name is not None: print(f'\tSetting apparatus name to \'{apparatus_name}\'') - config.set('DEFAULT', 'apparatus_name', apparatus_name) + config['DEFAULT']['apparatus_name'] = apparatus_name - with open(target_path, 'w') as f: - config.write(f) + target_path.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(target_path, config) def compile_connection_table(): """Compile the connection table defined in the labconfig file @@ -76,8 +86,9 @@ def compile_connection_table(): # if runmanager doesn't import, skip compilation return - config = configparser.ConfigParser(defaults = {'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)}) - config.read(default_labconfig_path()) + from labscript_utils.labconfig import LabConfig + + config = LabConfig() # The path to the user's connection_table.py script script_path = os.path.expandvars(config['paths']['connection_table_py']) diff --git a/labscript_profile/default_profile/labconfig/example.ini b/labscript_profile/default_profile/labconfig/example.ini deleted file mode 100644 index c959a192..00000000 --- a/labscript_profile/default_profile/labconfig/example.ini +++ /dev/null @@ -1,66 +0,0 @@ -[DEFAULT] -apparatus_name = example_apparatus -shared_drive = C: -experiment_shot_storage = %(shared_drive)s\Experiments\%(apparatus_name)s -userlib= %(labscript_suite)s\userlib -pythonlib = %(userlib)s\pythonlib -labscriptlib = %(userlib)s\labscriptlib\%(apparatus_name)s -analysislib = %(userlib)s\analysislib\%(apparatus_name)s -app_saved_configs = %(labscript_suite)s\app_saved_configs\%(apparatus_name)s -user_devices = user_devices - -[paths] -connection_table_h5 = %(experiment_shot_storage)s\connection_table.h5 -connection_table_py = %(labscriptlib)s\connection_table.py - -[servers] -zlock = localhost -runmanager = localhost -blacs = localhost -lyse = localhost - -[ports] -BLACS = 42517 -lyse = 42519 -runviewer = 42521 -runmanager = 42523 -zlock = 7339 -zlog = 7340 -zprocess_remote = 7341 - -[timeouts] -communication_timeout = 60 - -[programs] -text_editor = %%PROGRAMFILES%%\Sublime Text 3\sublime_text.exe -text_editor_arguments = {file} -hdf5_viewer = %%LOCALAPPDATA%%\HDF_Group\HDFView\3.1.0\hdfview.bat -hdf5_viewer_arguments = {file} - -[labscript] -save_hg_info = False -save_git_info = False - -[BLACS/plugins] -connection_table = True -connection_table.hashable_types = ['.py', '.txt', '.toml', '.ini', '.json'] -connection_table.polling_interval = 1.0 - -delete_repeated_shots = False -general = True -memory = False -progress_bar = False -theme = True - - -[lyse] -autoload_config_file = %(app_saved_configs)s\lyse\lyse.ini -integer_indexing = False - -[runmanager] -autoload_config_file = %(app_saved_configs)s\runmanager\runmanager.ini -output_folder_format = %%Y\%%m\%%d\{sequence_index:04d} -filename_prefix_format = %%Y-%%m-%%d_{sequence_index:04d}_{script_basename} - -[security] -shared_secret = %(labscript_suite)s\labconfig\zpsecret-b810f83f.key diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml new file mode 100644 index 00000000..8e368358 --- /dev/null +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -0,0 +1,64 @@ +[DEFAULT] +apparatus_name = 'example_apparatus' +shared_drive = 'C:' +experiment_shot_storage = '%(shared_drive)s\Experiments\%(apparatus_name)s' +userlib = '%(labscript_suite)s\userlib' +pythonlib = '%(userlib)s\pythonlib' +labscriptlib = '%(userlib)s\labscriptlib\%(apparatus_name)s' +analysislib = '%(userlib)s\analysislib\%(apparatus_name)s' +app_saved_configs = '%(labscript_suite)s\app_saved_configs\%(apparatus_name)s' +user_devices = 'user_devices' + +[paths] +connection_table_h5 = '%(experiment_shot_storage)s\connection_table.h5' +connection_table_py = '%(labscriptlib)s\connection_table.py' + +[servers] +zlock = 'localhost' +runmanager = 'localhost' +blacs = 'localhost' +lyse = 'localhost' + +[ports] +BLACS = 42517 +lyse = 42519 +runviewer = 42521 +runmanager = 42523 +zlock = 7339 +zlog = 7340 +zprocess_remote = 7341 + +[timeouts] +communication_timeout = 60 + +[programs] +text_editor = '%%PROGRAMFILES%%\Sublime Text 3\sublime_text.exe' +text_editor_arguments = '{file}' +hdf5_viewer = '%%LOCALAPPDATA%%\HDF_Group\HDFView\3.1.0\hdfview.bat' +hdf5_viewer_arguments = '{file}' + +[labscript] +save_hg_info = false +save_git_info = false + +["BLACS/plugins"] +connection_table = true +"connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] +"connection_table.polling_interval" = 1.0 +delete_repeated_shots = false +general = true +memory = false +progress_bar = false +theme = true + +[lyse] +autoload_config_file = '%(app_saved_configs)s\lyse\lyse.toml' +integer_indexing = false + +[runmanager] +autoload_config_file = '%(app_saved_configs)s\runmanager\runmanager.toml' +output_folder_format = '%%Y\%%m\%%d\{sequence_index:04d}' +filename_prefix_format = '%%Y-%%m-%%d_{sequence_index:04d}_{script_basename}' + +[security] +shared_secret = '%(labscript_suite)s\labconfig\zpsecret-b810f83f.key' diff --git a/labscript_profile/toml_config.py b/labscript_profile/toml_config.py new file mode 100644 index 00000000..67eb7f1e --- /dev/null +++ b/labscript_profile/toml_config.py @@ -0,0 +1,80 @@ +import configparser + +try: + import tomllib +except ImportError: + import tomli as tomllib + +import tomli_w + + +def load_toml_file(filename): + with open(filename, 'rb') as f: + return tomllib.load(f) + + +def dump_toml_file(filename, data): + with open(filename, 'wb') as f: + tomli_w.dump(data, f) + + +class TomlBasicInterpolation(configparser.BasicInterpolation): + """Basic interpolation that leaves non-string TOML values alone.""" + + def before_get(self, parser, section, option, value, defaults): + if not isinstance(value, str): + return value + return super().before_get(parser, section, option, value, defaults) + + +class TomlConfigParser(configparser.ConfigParser): + """ConfigParser variant that preserves native TOML values.""" + + def __init__(self, *args, **kwargs): + kwargs.setdefault('interpolation', TomlBasicInterpolation()) + super().__init__(*args, **kwargs) + + def set(self, section, option, value=None): + if not section or section == self.default_section: + sectdict = self._defaults + else: + try: + sectdict = self._sections[section] + except KeyError: + raise configparser.NoSectionError(section) from None + sectdict[self.optionxform(option)] = value + + def getboolean( + self, section, option, *, raw=False, vars=None, fallback=configparser._UNSET + ): + value = self.get(section, option, raw=raw, vars=vars, fallback=fallback) + if value is fallback: + return value + if isinstance(value, bool): + return value + return self._convert_to_boolean(value) + + def read_toml(self, filename): + data = load_toml_file(filename) + if not isinstance(data, dict): + raise TypeError("Top-level TOML document must be a mapping of sections") + for section_name, section in data.items(): + if not isinstance(section, dict): + raise TypeError(f"Section {section_name!r} must contain key/value pairs") + section_name = str(section_name) + if section_name != self.default_section and not self.has_section(section_name): + self.add_section(section_name) + for option, value in section.items(): + self.set(section_name, str(option), value) + return [str(filename)] + + def to_toml_dict(self): + data = {} + if self.defaults(): + data[self.default_section] = dict(self.defaults()) + for section in self.sections(): + data[section] = dict(self._sections[section]) + return data + + def write_toml(self, filename): + dump_toml_file(filename, self.to_toml_dict()) diff --git a/labscript_utils/filewatcher.py b/labscript_utils/filewatcher.py index 4b6f84b2..edc081a8 100644 --- a/labscript_utils/filewatcher.py +++ b/labscript_utils/filewatcher.py @@ -268,7 +268,7 @@ def callback(name, modified, event=None): callback, files=test_files[0], folders=test_folder, - hashable_types=['.py', '.ini', '.txt'], + hashable_types=['.py', '.toml', '.txt'], interval=2, ) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index d110fe82..be4e60e9 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -10,31 +10,45 @@ # for the full license. # # # ##################################################################### -import os import configparser +import os +import warnings from ast import literal_eval -from pprint import pformat from pathlib import Path -import warnings from labscript_utils import dedent -from labscript_profile import default_labconfig_path, LABSCRIPT_SUITE_PROFILE +from labscript_profile import ( + LABSCRIPT_SUITE_PROFILE, + default_labconfig_path, + ensure_labconfig, +) +from labscript_profile.toml_config import ( + TomlBasicInterpolation, + TomlConfigParser, + dump_toml_file, + load_toml_file, +) + default_config_path = default_labconfig_path() -class EnvInterpolation(configparser.BasicInterpolation): - """Interpolation which expands environment variables in values, - by post-filtering BasicInterpolation.before_get()""" +class EnvInterpolation(TomlBasicInterpolation): + """Interpolation that expands environment variables after TOML/basic interpolation.""" def before_get(self, *args): value = super(EnvInterpolation, self).before_get(*args) - return os.path.expandvars(value) + if isinstance(value, str): + return os.path.expandvars(value) + return value + +class LabConfig(TomlConfigParser): + """TOML labconfig reader with the small ConfigParser-like API used by the suite.""" -class LabConfig(configparser.ConfigParser): NoOptionError = configparser.NoOptionError NoSectionError = configparser.NoSectionError + DuplicateSectionError = configparser.DuplicateSectionError def __init__( self, config_path=default_config_path, required_params=None, defaults=None, @@ -43,12 +57,20 @@ def __init__( required_params = {} if defaults is None: defaults = {} - # str() below is for py36 compat, where ConfigParser can't deal with Path objs + else: + defaults = dict(defaults) defaults['labscript_suite'] = str(LABSCRIPT_SUITE_PROFILE) if isinstance(config_path, list): - self.config_path = config_path[0] + config_paths = list(config_path) + if config_paths: + if config_paths[0] == default_config_path: + config_paths[0] = ensure_labconfig() + self.config_path = config_paths[0] + else: + self.config_path = None else: - self.config_path = config_path + self.config_path = ensure_labconfig() if config_path == default_config_path else config_path + config_paths = [self.config_path] self.file_format = "" for section, options in required_params.items(): @@ -56,15 +78,12 @@ def __init__( for option in options: self.file_format += "%s = \n" % option - # Load the config file - configparser.ConfigParser.__init__( + TomlConfigParser.__init__( self, defaults=defaults, interpolation=EnvInterpolation() ) - # read all files in the config path if it is a list (self.config_path only - # contains one string): - self.read(config_path) + for path in config_paths: + self._read_path(path) - # Rename experiment_name to apparatus_name and raise a DeprectionWarning experiment_name = self.get("DEFAULT", "experiment_name", fallback=None) if experiment_name: msg = """The experiment_name keyword has been renamed apparatus_name in @@ -76,57 +95,146 @@ def __init__( your labconfig. Please omit the deprecate experiment_name keyword.""" raise Exception(dedent(msg)) - else: - self.set("DEFAULT", "apparatus_name", experiment_name) + self.set("DEFAULT", "apparatus_name", experiment_name) try: for section, options in required_params.items(): for option in options: self.get(section, option) - except configparser.NoOptionError: + except (configparser.NoOptionError, configparser.NoSectionError): msg = f"""The experiment configuration file located at {config_path} does not have the required keys. Make sure the config file contains the following structure:\n{self.file_format}""" raise Exception(dedent(msg)) + def _read_path(self, path): + """Read a TOML labconfig, or temporarily fall back to legacy INI.""" + if path is None: + return + path = Path(path) + if not path.exists(): + return + if path.suffix.lower() == '.toml': + self.read_toml(path) + return + if path.suffix.lower() == '.ini': + msg = """INI labconfig support is deprecated and slated for removal soon. + Convert this file to TOML.""" + warnings.warn(dedent(msg), FutureWarning) + self.read(path) + return + msg = f"Unsupported labconfig format for {path}. Expected a .toml or .ini file." + raise RuntimeError(msg) + + +def _resolve_appconfig_load_path(filename): + path = Path(filename) + suffix = path.suffix.lower() + if suffix == '.ini': + if path.exists(): + return path + toml_path = path.with_suffix('.toml') + if toml_path.exists(): + return toml_path + return path + if suffix == '.toml': + if path.exists(): + return path + legacy_path = path.with_suffix('.ini') + if legacy_path.exists(): + return legacy_path + return path + toml_path = path.with_suffix('.toml') + if toml_path.exists(): + return toml_path + legacy_path = path.with_suffix('.ini') + if legacy_path.exists(): + return legacy_path + return toml_path + + +def _resolve_appconfig_save_path(filename): + return Path(filename).with_suffix('.toml') + + +def _to_toml_compatible(value, location='value'): + if isinstance(value, dict): + converted = {} + for key, child in value.items(): + if not isinstance(key, str): + msg = f"{location} contains a non-string dict key {key!r}" + raise TypeError(msg) + converted[key] = _to_toml_compatible(child, f"{location}.{key}") + return converted + if isinstance(value, (list, tuple)): + return [ + _to_toml_compatible(child, f"{location}[{index}]") + for index, child in enumerate(value) + ] + if isinstance(value, (str, bool, int, float)): + return value + msg = f"{location} value {value!r} is not representable in TOML app config" + raise TypeError(msg) + + +def _load_legacy_appconfig_value(value): + """Temporary legacy INI app-config decoder. Slated for removal soon.""" + if not isinstance(value, str): + return value + try: + return literal_eval(value) + except (ValueError, SyntaxError): + return value + def save_appconfig(filename, data): - """Save a dictionary as an ini file. The keys of the dictionary comprise the section - names, and the values must themselves be dictionaries for the names and values - within each section. All section values will be converted to strings with - pprint.pformat().""" - # Error checking - for section_name, section in data.items(): - for name, value in section.items(): - try: - valid = value == literal_eval(pformat(value)) - except (ValueError, SyntaxError): - valid = False - if not valid: - msg = f"{section_name}/{name} value {value} not a Python built-in type" - raise TypeError(msg) + """Save a dictionary as a TOML app config.""" data = { - section_name: {name: pformat(value) for name, value in section.items()} + section_name: { + name: _to_toml_compatible(value, f"{section_name}/{name}") + for name, value in section.items() + } for section_name, section in data.items() } - c = configparser.ConfigParser(interpolation=None) - c.optionxform = str # preserve case - c.read_dict(data) - Path(filename).parent.mkdir(parents=True, exist_ok=True) - with open(filename, 'w') as f: - c.write(f) - - -def load_appconfig(filename): - """Load an .ini file and return a dictionary of its contents. All values will be - converted to Python objects with ast.literal_eval(). All keys will be lowercase - regardless of the written contents on the .ini file.""" - c = configparser.ConfigParser(interpolation=None) - c.optionxform = str # preserve case - # No file? No config - don't crash. - if Path(filename).exists(): - c.read(filename) - return { - section_name: {name: literal_eval(value) for name, value in section.items()} - for section_name, section in c.items() - } + filename = _resolve_appconfig_save_path(filename) + filename.parent.mkdir(parents=True, exist_ok=True) + dump_toml_file(filename, data) + return str(filename) + + +def load_appconfig(filename, return_save_path=False): + """Load a TOML app config, or a legacy INI app config if required. + + If a legacy INI file is loaded, a sibling TOML file is immediately written and can be + returned as the canonical save path via ``return_save_path=True``. Legacy INI support + here is temporary and slated for removal soon. + """ + requested_path = Path(filename) + filename = _resolve_appconfig_load_path(filename) + save_path = _resolve_appconfig_save_path(requested_path) + if filename.suffix.lower() == '.ini': + c = configparser.ConfigParser(interpolation=None) + c.optionxform = str + if filename.exists(): + c.read(filename) + data = { + section_name: { + name: _load_legacy_appconfig_value(value) + for name, value in c.items(section_name, raw=True) + } + for section_name in c.sections() + } + if filename.exists(): + save_path = Path(save_appconfig(save_path, data)) + else: + raw = load_toml_file(filename) if filename.exists() else {} + data = { + section_name: dict(section.items()) + for section_name, section in raw.items() + if section_name != 'DEFAULT' and isinstance(section, dict) + } + if filename.suffix.lower() == '.toml': + save_path = filename + if return_save_path: + return data, str(save_path) + return data diff --git a/pyproject.toml b/pyproject.toml index 53ec1e59..fed85e39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ dependencies = [ "scipy", "zprocess>=2.18.0", "setuptools_scm>=4.1.0", + "tomli>=1.1.0; python_version < '3.11'", + "tomli-w>=1.2.0", ] dynamic = ["version"] From 806fab58ee817d7b43dfe9c7550fa564c522409b Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 13:58:46 -0400 Subject: [PATCH 09/42] Normalize labconfig TOML section names --- labscript_profile/__init__.py | 9 ++- labscript_profile/create.py | 4 +- .../default_profile/labconfig/example.toml | 6 +- labscript_profile/toml_config.py | 75 ++++++++++++++++++- labscript_utils/labconfig.py | 4 +- 5 files changed, 87 insertions(+), 11 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index a45a5744..07def1ec 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -143,21 +143,22 @@ def _convert_legacy_labconfig(legacy_path, toml_path): config = configparser.ConfigParser(interpolation=None) config.read(legacy_path) template_data = _example_labconfig_data() - default_template = template_data.get('DEFAULT', {}) + default_template = template_data.get('default', {}) data = { - 'DEFAULT': { + 'default': { key: _coerce_legacy_value(value, default_template.get(key)) for key, value in config.defaults().items() } } for section in config.sections(): + canonical_section = section.lower() section_items = dict(config._sections[section]) section_items.pop('__name__', None) autoload = section_items.get('autoload_config_file') if isinstance(autoload, str) and autoload.lower().endswith('.ini'): section_items['autoload_config_file'] = autoload[:-4] + '.toml' - section_template = template_data.get(section, {}) - data[section] = { + section_template = template_data.get(canonical_section, {}) + data[canonical_section] = { key: _coerce_legacy_value(value, section_template.get(key)) for key, value in section_items.items() } diff --git a/labscript_profile/create.py b/labscript_profile/create.py index 2ca7e46c..b1b3bff1 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -61,7 +61,7 @@ def make_labconfig_file(apparatus_name = None): config['programs']['text_editor_arguments'] = '-a TextEdit {file}' if sys.platform != 'win32': config['programs']['hdf5_viewer'] = 'hdfview' - config['DEFAULT']['shared_drive'] = '$HOME/labscript_shared' + config['default']['shared_drive'] = '$HOME/labscript_shared' shared_secret = make_shared_secret(target_path.parent) shared_secret_entry = Path( '%(labscript_suite)s', shared_secret.relative_to(LABSCRIPT_SUITE_PROFILE) @@ -69,7 +69,7 @@ def make_labconfig_file(apparatus_name = None): config['security']['shared_secret'] = str(shared_secret_entry) if apparatus_name is not None: print(f'\tSetting apparatus name to \'{apparatus_name}\'') - config['DEFAULT']['apparatus_name'] = apparatus_name + config['default']['apparatus_name'] = apparatus_name target_path.parent.mkdir(parents=True, exist_ok=True) dump_toml_file(target_path, config) diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml index 8e368358..2c5c47b8 100644 --- a/labscript_profile/default_profile/labconfig/example.toml +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -1,4 +1,4 @@ -[DEFAULT] +[default] apparatus_name = 'example_apparatus' shared_drive = 'C:' experiment_shot_storage = '%(shared_drive)s\Experiments\%(apparatus_name)s' @@ -20,7 +20,7 @@ blacs = 'localhost' lyse = 'localhost' [ports] -BLACS = 42517 +blacs = 42517 lyse = 42519 runviewer = 42521 runmanager = 42523 @@ -41,7 +41,7 @@ hdf5_viewer_arguments = '{file}' save_hg_info = false save_git_info = false -["BLACS/plugins"] +["blacs/plugins"] connection_table = true "connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] "connection_table.polling_interval" = 1.0 diff --git a/labscript_profile/toml_config.py b/labscript_profile/toml_config.py index 67eb7f1e..e5c6d756 100644 --- a/labscript_profile/toml_config.py +++ b/labscript_profile/toml_config.py @@ -31,10 +31,64 @@ class TomlConfigParser(configparser.ConfigParser): """ConfigParser variant that preserves native TOML values.""" def __init__(self, *args, **kwargs): + self.lowercase_keys_and_sections = kwargs.pop('lowercase_keys_and_sections', True) + if self.lowercase_keys_and_sections: + kwargs.setdefault('default_section', 'default') kwargs.setdefault('interpolation', TomlBasicInterpolation()) super().__init__(*args, **kwargs) + if not self.lowercase_keys_and_sections: + self.optionxform = str + + def _canonical_section_name(self, section): + if section is None: + return None + if not self.lowercase_keys_and_sections: + return section + return str(section).lower() + + def _normalize_loaded_sections(self): + if not self.lowercase_keys_and_sections: + return + renamed_sections = {} + migrated_defaults = {} + for name, section_data in self._sections.items(): + canonical = self._canonical_section_name(name) + if canonical == self.default_section: + migrated_defaults.update(section_data) + continue + existing = renamed_sections.get(canonical) + if existing is not None and existing is not section_data: + raise configparser.DuplicateSectionError(canonical) + renamed_sections[canonical] = section_data + self._defaults.update(migrated_defaults) + self._sections = renamed_sections + self._proxies = {self.default_section: configparser.SectionProxy(self, self.default_section)} + for canonical in renamed_sections: + self._proxies[canonical] = configparser.SectionProxy(self, canonical) + + def add_section(self, section): + return super().add_section(self._canonical_section_name(section)) + + def has_section(self, section): + return super().has_section(self._canonical_section_name(section)) + + def options(self, section): + return super().options(self._canonical_section_name(section)) + + def has_option(self, section, option): + return super().has_option(self._canonical_section_name(section), option) + + def remove_section(self, section): + return super().remove_section(self._canonical_section_name(section)) + + def remove_option(self, section, option): + return super().remove_option(self._canonical_section_name(section), option) + + def __getitem__(self, key): + return super().__getitem__(self._canonical_section_name(key)) def set(self, section, option, value=None): + section = self._canonical_section_name(section) if not section or section == self.default_section: sectdict = self._defaults else: @@ -54,6 +108,25 @@ def getboolean( return value return self._convert_to_boolean(value) + def get(self, section, option, *, raw=False, vars=None, fallback=configparser._UNSET): + return super().get( + self._canonical_section_name(section), + option, + raw=raw, + vars=vars, + fallback=fallback, + ) + + def items(self, section=configparser._UNSET, raw=False, vars=None): + if section is configparser._UNSET: + return super().items(section=section, raw=raw, vars=vars) + return super().items(self._canonical_section_name(section), raw=raw, vars=vars) + + def read(self, filenames, encoding=None): + read_ok = super().read(filenames, encoding=encoding) + self._normalize_loaded_sections() + return read_ok + def read_toml(self, filename): data = load_toml_file(filename) if not isinstance(data, dict): @@ -61,7 +134,7 @@ def read_toml(self, filename): for section_name, section in data.items(): if not isinstance(section, dict): raise TypeError(f"Section {section_name!r} must contain key/value pairs") - section_name = str(section_name) + section_name = self._canonical_section_name(section_name) if section_name != self.default_section and not self.has_section(section_name): self.add_section(section_name) for option, value in section.items(): diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index be4e60e9..6587eb11 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -79,7 +79,9 @@ def __init__( self.file_format += "%s = \n" % option TomlConfigParser.__init__( - self, defaults=defaults, interpolation=EnvInterpolation() + self, + defaults=defaults, + interpolation=EnvInterpolation(), ) for path in config_paths: self._read_path(path) From c24a4fed85986c66dc858d4665f3793485dd6c1a Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 19 Mar 2026 15:19:22 -0400 Subject: [PATCH 10/42] Updated example.toml. --- labscript_profile/default_profile/labconfig/example.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_profile/default_profile/labconfig/example.toml b/labscript_profile/default_profile/labconfig/example.toml index 2c5c47b8..f655e501 100644 --- a/labscript_profile/default_profile/labconfig/example.toml +++ b/labscript_profile/default_profile/labconfig/example.toml @@ -43,7 +43,7 @@ save_git_info = false ["blacs/plugins"] connection_table = true -"connection_table.hashable_types" = ['.py', '.txt', '.toml', '.json'] +"connection_table.hashable_types" = ['.py', '.txt', '.ini', '.toml', '.json'] "connection_table.polling_interval" = 1.0 delete_repeated_shots = false general = true From 7a623bab4857b078db2b415683df9aea8c9d4a92 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 20 Mar 2026 10:23:49 -0400 Subject: [PATCH 11/42] Refine TOML labconfig normalization --- labscript_profile/__init__.py | 2 +- labscript_utils/device_registry/_device_registry.py | 2 +- labscript_utils/labconfig.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 07def1ec..1d05b9a4 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -65,7 +65,7 @@ def add_userlib_and_pythonlib(): config.read_toml(labconfig) for option in ['userlib', 'pythonlib']: try: - paths = config.get('DEFAULT', option) + paths = config.get('default', option) except (configparser.NoSectionError, configparser.NoOptionError): paths = '' if paths: diff --git a/labscript_utils/device_registry/_device_registry.py b/labscript_utils/device_registry/_device_registry.py index e5a122a8..0e2348a0 100644 --- a/labscript_utils/device_registry/_device_registry.py +++ b/labscript_utils/device_registry/_device_registry.py @@ -74,7 +74,7 @@ def _get_device_dirs(): """Return the directory of labscript_devices, and the folders containing submodules of any packages listed in the user_devices labconfig setting""" try: - user_devices = LabConfig().get('DEFAULT', 'user_devices') + user_devices = LabConfig().get('default', 'user_devices') except (LabConfig.NoOptionError, LabConfig.NoSectionError): user_devices = 'user_devices' # Split on commas, remove whitespace: diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 6587eb11..86bb7101 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -86,21 +86,22 @@ def __init__( for path in config_paths: self._read_path(path) - experiment_name = self.get("DEFAULT", "experiment_name", fallback=None) + experiment_name = self.get("default", "experiment_name", fallback=None) if experiment_name: msg = """The experiment_name keyword has been renamed apparatus_name in labscript_utils 3.0, and will be removed in a future version. Please update your labconfig to use the apparatus_name keyword.""" warnings.warn(dedent(msg), FutureWarning) - if self.get("DEFAULT", "apparatus_name", fallback=None): + if self.get("default", "apparatus_name", fallback=None): msg = """You have defined both experiment_name and apparatus_name in your labconfig. Please omit the deprecate experiment_name keyword.""" raise Exception(dedent(msg)) - self.set("DEFAULT", "apparatus_name", experiment_name) + self.set("default", "apparatus_name", experiment_name) try: for section, options in required_params.items(): + section = self._canonical_section_name(section) for option in options: self.get(section, option) except (configparser.NoOptionError, configparser.NoSectionError): @@ -233,7 +234,7 @@ def load_appconfig(filename, return_save_path=False): data = { section_name: dict(section.items()) for section_name, section in raw.items() - if section_name != 'DEFAULT' and isinstance(section, dict) + if section_name != 'default' and isinstance(section, dict) } if filename.suffix.lower() == '.toml': save_path = filename From 11e77a8a5957d000c0a7caea37f3e525b4d586b5 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 20 Mar 2026 10:50:34 -0400 Subject: [PATCH 12/42] Fix zlock launch with TOML port values --- labscript_utils/zlock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/zlock.py b/labscript_utils/zlock.py index 5878997f..baac341b 100644 --- a/labscript_utils/zlock.py +++ b/labscript_utils/zlock.py @@ -40,7 +40,7 @@ def main(): '-m', 'zprocess.zlock', '--port', - config['zlock_port'], + str(config['zlock_port']), '-l', LOG_PATH, ] From c172cae8d3fb9928dd90dda7624ca9757ef7ee03 Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 21 Mar 2026 19:01:02 -0400 Subject: [PATCH 13/42] Prefer canonical TOML over stale legacy app configs --- labscript_profile/__init__.py | 8 +++++--- labscript_profile/create.py | 1 + labscript_utils/labconfig.py | 34 +++++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/labscript_profile/__init__.py b/labscript_profile/__init__.py index 1d05b9a4..f2b12580 100644 --- a/labscript_profile/__init__.py +++ b/labscript_profile/__init__.py @@ -42,7 +42,7 @@ def default_labconfig_path(): return None return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.toml' - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def legacy_labconfig_path(): """Temporary legacy INI labconfig location. Slated for removal soon.""" if LABSCRIPT_SUITE_PROFILE is None: @@ -83,6 +83,7 @@ def ensure_labconfig(): return None if labconfig.exists(): return labconfig + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy = legacy_labconfig_path() if legacy is None or not legacy.exists(): return labconfig @@ -100,7 +101,7 @@ def _example_labconfig_data(): return load_toml_file(bundled_example) return {} - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _coerce_legacy_value(value, template_value): if template_value is None: return value @@ -137,7 +138,7 @@ def _coerce_legacy_value(value, template_value): return [_coerce_legacy_value(item, template_item) for item in parsed] return value - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _convert_legacy_labconfig(legacy_path, toml_path): """Temporary legacy migration path from INI to TOML. Slated for removal soon.""" config = configparser.ConfigParser(interpolation=None) @@ -155,6 +156,7 @@ def _convert_legacy_labconfig(legacy_path, toml_path): section_items = dict(config._sections[section]) section_items.pop('__name__', None) autoload = section_items.get('autoload_config_file') + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if isinstance(autoload, str) and autoload.lower().endswith('.ini'): section_items['autoload_config_file'] = autoload[:-4] + '.toml' section_template = template_data.get(canonical_section, {}) diff --git a/labscript_profile/create.py b/labscript_profile/create.py index b1b3bff1..dcea2d11 100644 --- a/labscript_profile/create.py +++ b/labscript_profile/create.py @@ -48,6 +48,7 @@ def make_labconfig_file(apparatus_name = None): source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.toml') target_path = default_labconfig_path() + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = legacy_labconfig_path() if os.path.exists(target_path) or (legacy_path is not None and os.path.exists(legacy_path)): raise FileExistsError(target_path) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index 86bb7101..971bd4cd 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -120,6 +120,7 @@ def _read_path(self, path): if path.suffix.lower() == '.toml': self.read_toml(path) return + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if path.suffix.lower() == '.ini': msg = """INI labconfig support is deprecated and slated for removal soon. Convert this file to TOML.""" @@ -133,16 +134,18 @@ def _read_path(self, path): def _resolve_appconfig_load_path(filename): path = Path(filename) suffix = path.suffix.lower() + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if suffix == '.ini': - if path.exists(): - return path toml_path = path.with_suffix('.toml') if toml_path.exists(): return toml_path + if path.exists(): + return path return path if suffix == '.toml': if path.exists(): return path + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = path.with_suffix('.ini') if legacy_path.exists(): return legacy_path @@ -150,6 +153,7 @@ def _resolve_appconfig_load_path(filename): toml_path = path.with_suffix('.toml') if toml_path.exists(): return toml_path + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. legacy_path = path.with_suffix('.ini') if legacy_path.exists(): return legacy_path @@ -159,6 +163,23 @@ def _resolve_appconfig_load_path(filename): def _resolve_appconfig_save_path(filename): return Path(filename).with_suffix('.toml') +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. +def backup_legacy_config(path): + """Rename a migrated legacy config file to ``.old`` without clobbering backups.""" + path = Path(path) + if not path.exists(): + return None + backup_path = Path(str(path) + '.old') + index = 1 + while backup_path.exists(): + backup_path = Path(str(path) + '.old' + str(index)) + index += 1 + try: + path.rename(backup_path) + except OSError: + return None + return backup_path + def _to_toml_compatible(value, location='value'): if isinstance(value, dict): @@ -179,7 +200,7 @@ def _to_toml_compatible(value, location='value'): msg = f"{location} value {value!r} is not representable in TOML app config" raise TypeError(msg) - +# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. def _load_legacy_appconfig_value(value): """Temporary legacy INI app-config decoder. Slated for removal soon.""" if not isinstance(value, str): @@ -209,12 +230,14 @@ def load_appconfig(filename, return_save_path=False): """Load a TOML app config, or a legacy INI app config if required. If a legacy INI file is loaded, a sibling TOML file is immediately written and can be - returned as the canonical save path via ``return_save_path=True``. Legacy INI support - here is temporary and slated for removal soon. + returned as the canonical save path via ``return_save_path=True``. After successful + conversion, the legacy INI file is moved aside to ``.old`` so future loads prefer the + canonical TOML file. Legacy INI support here is temporary and slated for removal soon. """ requested_path = Path(filename) filename = _resolve_appconfig_load_path(filename) save_path = _resolve_appconfig_save_path(requested_path) + # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED. if filename.suffix.lower() == '.ini': c = configparser.ConfigParser(interpolation=None) c.optionxform = str @@ -229,6 +252,7 @@ def load_appconfig(filename, return_save_path=False): } if filename.exists(): save_path = Path(save_appconfig(save_path, data)) + backup_legacy_config(filename) else: raw = load_toml_file(filename) if filename.exists() else {} data = { From 5725e3e306832fb7597245cb24ad1dc274059a8c Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 09:14:10 -0400 Subject: [PATCH 14/42] Set macOS app icons from splash path --- labscript_utils/splash.py | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 51c56864..94b4e960 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,39 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + icon_path = qapplication.property('_labscript_icon_path') + if icon_path: + icon = QtGui.QIcon(icon_path) + if not icon.isNull(): + qapplication.setWindowIcon(icon) + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + elif qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + + +def get_qapplication(argv=None): + qapplication = QtWidgets.QApplication.instance() + if qapplication is None: + qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) + return configure_qapplication(qapplication) + + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -50,9 +83,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = QtWidgets.QApplication.instance() - if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + self.qapplication = get_qapplication() + self.qapplication.setProperty('_labscript_icon_path', imagepath) + configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 89fdbdec69467514bc0142abdf9ed48beeafbbb7 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 09:22:56 -0400 Subject: [PATCH 15/42] Inline redundant splash QApplication helper --- labscript_utils/splash.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 94b4e960..00890695 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -65,13 +65,6 @@ def configure_qapplication(qapplication): return qapplication -def get_qapplication(argv=None): - qapplication = QtWidgets.QApplication.instance() - if qapplication is None: - qapplication = QtWidgets.QApplication(sys.argv if argv is None else argv) - return configure_qapplication(qapplication) - - class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -83,7 +76,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath): - self.qapplication = get_qapplication() + self.qapplication = QtWidgets.QApplication.instance() + if self.qapplication is None: + self.qapplication = QtWidgets.QApplication(sys.argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) configure_qapplication(self.qapplication) super().__init__() From 454570c12748266641f4f3ea86523b05e9644cb4 Mon Sep 17 00:00:00 2001 From: spielman Date: Thu, 26 Mar 2026 17:02:09 -0400 Subject: [PATCH 16/42] Add shared indexed filepath helper --- labscript_utils/file_utils.py | 41 +++++++++++++++++++++++++++++++++++ labscript_utils/shot_utils.py | 3 ++- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 labscript_utils/file_utils.py diff --git a/labscript_utils/file_utils.py b/labscript_utils/file_utils.py new file mode 100644 index 00000000..57a2ef85 --- /dev/null +++ b/labscript_utils/file_utils.py @@ -0,0 +1,41 @@ +##################################################################### +# # +# file_utils.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### + +import os +import re + + +def next_available_indexed_filepath(filepath, suffix_format, start=0): + """Return the first non-existent filepath created by appending a formatted suffix. + + ``suffix_format`` is formatted with a single ``index`` keyword argument and is + inserted between the filepath stem and its extension. If filepath already ends + with a suffix matching the pattern, that suffix is stripped and the search begins + at the next higher index.""" + basename, ext = os.path.splitext(filepath) + suffix_pattern = re.escape(suffix_format) + suffix_pattern = re.sub( + r'\\\{index(?::[^}]*)?\\\}', + lambda match: r'(?P\d+)', + suffix_pattern, + ) + match = re.search(suffix_pattern + r'$', basename) + if match is not None: + basename = basename[:match.start()] + start = max(start, int(match.group('index')) + 1) + index = start + while True: + candidate = basename + suffix_format.format(index=index) + ext + if not os.path.exists(candidate): + return candidate, index + index += 1 diff --git a/labscript_utils/shot_utils.py b/labscript_utils/shot_utils.py index aaccced9..987abed7 100644 --- a/labscript_utils/shot_utils.py +++ b/labscript_utils/shot_utils.py @@ -14,6 +14,7 @@ import h5py import numpy as np + def get_shot_globals(filepath): """Returns the evaluated globals for a shot, for use by labscript or lyse. Simple dictionary access as in dict(h5py.File(filepath).attrs) would be fine @@ -34,4 +35,4 @@ def get_shot_globals(filepath): if isinstance(value, bytes): value = value.decode() params[name] = value - return params \ No newline at end of file + return params From 8bf4faab0a1d1db57adf859a8f5c1c7486e5190e Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 11:23:59 -0400 Subject: [PATCH 17/42] Make indexed filepath start explicit --- labscript_utils/file_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/labscript_utils/file_utils.py b/labscript_utils/file_utils.py index 57a2ef85..ee032183 100644 --- a/labscript_utils/file_utils.py +++ b/labscript_utils/file_utils.py @@ -15,13 +15,14 @@ import re -def next_available_indexed_filepath(filepath, suffix_format, start=0): +def next_available_indexed_filepath(filepath, suffix_format, start=None): """Return the first non-existent filepath created by appending a formatted suffix. ``suffix_format`` is formatted with a single ``index`` keyword argument and is inserted between the filepath stem and its extension. If filepath already ends - with a suffix matching the pattern, that suffix is stripped and the search begins - at the next higher index.""" + with a suffix matching the pattern, that suffix is stripped. If ``start`` is + ``None``, the search begins at the next higher index from that suffix. Otherwise + ``start`` is used exactly as provided.""" basename, ext = os.path.splitext(filepath) suffix_pattern = re.escape(suffix_format) suffix_pattern = re.sub( @@ -32,7 +33,10 @@ def next_available_indexed_filepath(filepath, suffix_format, start=0): match = re.search(suffix_pattern + r'$', basename) if match is not None: basename = basename[:match.start()] - start = max(start, int(match.group('index')) + 1) + if start is None: + start = int(match.group('index')) + 1 + if start is None: + start = 0 index = start while True: candidate = basename + suffix_format.format(index=index) + ext From d87ad88588ed7c375d84a960bebcb5c27ab40926 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 11:55:10 -0400 Subject: [PATCH 18/42] Simplify shared shot queue controls --- labscript_utils/qtwidgets/shotqueue.py | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index df95b23b..30c8c54c 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -132,6 +132,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -139,14 +140,6 @@ def __init__( self.delete_button.setText('Delete') self.clear_button = QToolButton(self) self.clear_button.setText('Clear') - self.move_top_button = QToolButton(self) - self.move_top_button.setText('Top') - self.move_up_button = QToolButton(self) - self.move_up_button.setText('Up') - self.move_down_button = QToolButton(self) - self.move_down_button.setText('Down') - self.move_bottom_button = QToolButton(self) - self.move_bottom_button.setText('Bottom') button_layout = QHBoxLayout() button_layout.setContentsMargins(0, 0, 0, 0) @@ -154,10 +147,6 @@ def __init__( button_layout.addWidget(self.delete_button) button_layout.addWidget(self.clear_button) button_layout.addStretch(1) - button_layout.addWidget(self.move_top_button) - button_layout.addWidget(self.move_up_button) - button_layout.addWidget(self.move_down_button) - button_layout.addWidget(self.move_bottom_button) layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) @@ -167,10 +156,6 @@ def __init__( self.add_button.clicked.connect(self.prompt_for_files) self.delete_button.clicked.connect(self.remove_selected) self.clear_button.clicked.connect(self.clear) - self.move_top_button.clicked.connect(self.move_top) - self.move_up_button.clicked.connect(self.move_up) - self.move_down_button.clicked.connect(self.move_down) - self.move_bottom_button.clicked.connect(self.move_bottom) self.queue_view.deleteRequested.connect(self.remove_selected) self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) @@ -340,10 +325,6 @@ def _update_button_states(self): has_selection = bool(selected_rows) self.delete_button.setEnabled(has_selection) self.clear_button.setEnabled(bool(row_count)) - self.move_top_button.setEnabled(has_selection and selected_rows[0] > 0) - self.move_up_button.setEnabled(has_selection and selected_rows[0] > 0) - self.move_bottom_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) - self.move_down_button.setEnabled(has_selection and selected_rows[-1] < row_count - 1) def _select_rows(self, rows): rows = list(rows) From 7c13f18ac876f7d788e2ac5787d3e9cd18f42bf5 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:00:28 -0400 Subject: [PATCH 19/42] Use context-menu deletion in shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 30c8c54c..46e4c87b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -64,6 +64,17 @@ def keyPressEvent(self, event): return QTreeView.keyPressEvent(self, event) + def contextMenuEvent(self, event): + menu = QMenu(self) + delete_action = menu.addAction('Delete selected rows') + delete_action.setEnabled(bool(self.selectionModel().selectedRows())) + chosen_action = menu.exec_(event.globalPos()) + if chosen_action is delete_action: + self.deleteRequested.emit() + event.accept() + return + QTreeView.contextMenuEvent(self, event) + def dragEnterEvent(self, event): if self._event_has_acceptable_urls(event): event.setDropAction(Qt.CopyAction) @@ -136,16 +147,10 @@ def __init__( self.add_button = QToolButton(self) self.add_button.setText('Add') - self.delete_button = QToolButton(self) - self.delete_button.setText('Delete') - self.clear_button = QToolButton(self) - self.clear_button.setText('Clear') button_layout = QHBoxLayout() button_layout.setContentsMargins(0, 0, 0, 0) button_layout.addWidget(self.add_button) - button_layout.addWidget(self.delete_button) - button_layout.addWidget(self.clear_button) button_layout.addStretch(1) layout = QVBoxLayout(self) @@ -154,8 +159,6 @@ def __init__( layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) - self.delete_button.clicked.connect(self.remove_selected) - self.clear_button.clicked.connect(self.clear) self.queue_view.deleteRequested.connect(self.remove_selected) self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) @@ -320,11 +323,7 @@ def _on_selection_changed(self, *args): self.selectionChanged.emit(self.selected_files()) def _update_button_states(self): - row_count = self.queue_model.rowCount() - selected_rows = self.selected_rows() - has_selection = bool(selected_rows) - self.delete_button.setEnabled(has_selection) - self.clear_button.setEnabled(bool(row_count)) + pass def _select_rows(self, rows): rows = list(rows) From 29999395b9cf3a2a0f2d1bd962650ce0ee75adea Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:02:45 -0400 Subject: [PATCH 20/42] Handle macOS delete key in shot queue --- labscript_utils/qtwidgets/shotqueue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 46e4c87b..68a48f3b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -58,7 +58,7 @@ def set_accepted_extensions(self, accepted_extensions): self._accepted_extensions = _normalise_extensions(accepted_extensions) def keyPressEvent(self, event): - if event.key() == Qt.Key_Delete: + if event.key() in (Qt.Key_Delete, Qt.Key_Backspace): event.accept() self.deleteRequested.emit() return From 5876c9b87760719752aaa5384c87e7f061e7fcf4 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:10:21 -0400 Subject: [PATCH 21/42] Move shot queue column support into shared widget --- labscript_utils/qtwidgets/shotqueue.py | 89 +++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 68a48f3b..08d92775 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -118,7 +118,7 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self._accepted_extensions) class ShotQueueWidget(QWidget): - """Reusable single-column shot queue editor widget.""" + """Reusable shot queue editor widget.""" queueChanged = Signal() selectionChanged = Signal(list) @@ -131,19 +131,31 @@ def __init__( file_dialog_filter='Shot files (*.h5 *.hdf5)', allow_duplicates=False, column_title='Filepath', + column_titles=None, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) self.file_dialog_filter = file_dialog_filter self.allow_duplicates = allow_duplicates self.last_opened_shots_folder = '' + if column_titles is None: + column_titles = [column_title] + self.column_titles = list(column_titles) self.queue_model = QStandardItemModel(self) - self.queue_model.setHorizontalHeaderItem(FILEPATH_COLUMN, QStandardItem(column_title)) + self.queue_model.setColumnCount(len(self.column_titles)) + for column, title in enumerate(self.column_titles): + self.queue_model.setHorizontalHeaderItem(column, QStandardItem(title)) self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + self.queue_view.header().setStretchLastSection(False) + self.queue_view.header().setSectionResizeMode(FILEPATH_COLUMN, QHeaderView.Stretch) + for column in range(1, len(self.column_titles)): + self.queue_view.header().setSectionResizeMode( + column, QHeaderView.ResizeToContents + ) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -170,12 +182,12 @@ def __init__( def files(self): return [ - self.queue_model.item(row, FILEPATH_COLUMN).text() + self._queue_path_for_row(row) for row in range(self.queue_model.rowCount()) ] def selected_files(self): - return [self.queue_model.item(row, FILEPATH_COLUMN).text() for row in self.selected_rows()] + return [self._queue_path_for_row(row) for row in self.selected_rows()] def selected_rows(self): return sorted(index.row() for index in self.queue_view.selectionModel().selectedRows()) @@ -186,8 +198,7 @@ def select_paths(self, paths): path_lookup = {os.path.abspath(str(path)) for path in paths} rows = [] for row in range(self.queue_model.rowCount()): - item = self.queue_model.item(row, FILEPATH_COLUMN) - if item.text() in path_lookup: + if self._queue_path_for_row(row) in path_lookup: rows.append(row) self._select_rows(rows) @@ -277,7 +288,10 @@ def move_bottom(self): def is_in_queue(self, path): path = os.path.abspath(str(path)) - return bool(self.queue_model.findItems(path, column=FILEPATH_COLUMN)) + return any( + self._queue_path_for_row(row) == path + for row in range(self.queue_model.rowCount()) + ) def get_save_data(self): return { @@ -289,11 +303,70 @@ def restore_save_data(self, data): self.last_opened_shots_folder = data.get('last_opened_shots_folder', '') self.set_files(data.get('files_queued', [])) + def set_row_infos(self, row_infos): + self.queue_model.removeRows(0, self.queue_model.rowCount()) + if not row_infos: + return + for row_info in row_infos: + self.queue_model.appendRow(self._create_row_from_info(row_info)) + def _create_row(self, path): item = QStandardItem(path) item.setToolTip(path) item.setEditable(False) - return [item] + item.setData(path, Qt.UserRole) + return [item] + self._create_padding_items(self.queue_model.columnCount() - 1) + + def _create_row_from_info(self, row_info): + if not isinstance(row_info, dict): + return self._create_row(str(row_info)) + path = os.path.abspath(str(row_info['path'])) + label = row_info.get('label', os.path.basename(path)) + tooltip = row_info.get('tooltip', path) + columns = list(row_info.get('columns', [])) + + item = QStandardItem(label) + item.setEditable(False) + item.setToolTip(tooltip) + item.setData(path, Qt.UserRole) + row_items = [item] + + for column_info in columns[: self.queue_model.columnCount() - 1]: + if isinstance(column_info, dict): + text = str(column_info.get('text', '')) + column_tooltip = column_info.get('tooltip', '') + alignment = column_info.get( + 'alignment', Qt.AlignLeft | Qt.AlignVCenter + ) + else: + text = str(column_info) + column_tooltip = '' + alignment = Qt.AlignLeft | Qt.AlignVCenter + column_item = QStandardItem(text) + column_item.setEditable(False) + column_item.setToolTip(column_tooltip) + column_item.setTextAlignment(alignment) + row_items.append(column_item) + + row_items.extend( + self._create_padding_items(self.queue_model.columnCount() - len(row_items)) + ) + return row_items + + def _create_padding_items(self, count): + items = [] + for _ in range(max(0, count)): + item = QStandardItem('') + item.setEditable(False) + items.append(item) + return items + + def _queue_path_for_row(self, row): + item = self.queue_model.item(row, FILEPATH_COLUMN) + path = item.data(Qt.UserRole) + if path is None: + path = item.text() + return os.path.abspath(str(path)) def _prepare_paths(self, paths): if isinstance(paths, str): From a66065477b7a5122cb36e99f4379462333d2998d Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:12:43 -0400 Subject: [PATCH 22/42] Support path column selection in shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 87 ++++++++++++++------------ 1 file changed, 47 insertions(+), 40 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 08d92775..0183b187 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -132,6 +132,7 @@ def __init__( allow_duplicates=False, column_title='Filepath', column_titles=None, + path_column=FILEPATH_COLUMN, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) @@ -141,6 +142,7 @@ def __init__( if column_titles is None: column_titles = [column_title] self.column_titles = list(column_titles) + self.path_column = int(path_column) self.queue_model = QStandardItemModel(self) self.queue_model.setColumnCount(len(self.column_titles)) @@ -149,13 +151,15 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) self.queue_view.header().setStretchLastSection(False) - self.queue_view.header().setSectionResizeMode(FILEPATH_COLUMN, QHeaderView.Stretch) - for column in range(1, len(self.column_titles)): - self.queue_view.header().setSectionResizeMode( - column, QHeaderView.ResizeToContents + for column in range(len(self.column_titles)): + resize_mode = ( + QHeaderView.Stretch + if column == self.path_column + else QHeaderView.ResizeToContents ) + self.queue_view.header().setSectionResizeMode(column, resize_mode) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -311,11 +315,13 @@ def set_row_infos(self, row_infos): self.queue_model.appendRow(self._create_row_from_info(row_info)) def _create_row(self, path): - item = QStandardItem(path) - item.setToolTip(path) - item.setEditable(False) - item.setData(path, Qt.UserRole) - return [item] + self._create_padding_items(self.queue_model.columnCount() - 1) + row_items = self._create_padding_items(self.queue_model.columnCount()) + row_items[self.path_column] = self._create_display_item( + path, + tooltip=path, + path=path, + ) + return row_items def _create_row_from_info(self, row_info): if not isinstance(row_info, dict): @@ -324,45 +330,46 @@ def _create_row_from_info(self, row_info): label = row_info.get('label', os.path.basename(path)) tooltip = row_info.get('tooltip', path) columns = list(row_info.get('columns', [])) - - item = QStandardItem(label) - item.setEditable(False) - item.setToolTip(tooltip) - item.setData(path, Qt.UserRole) - row_items = [item] - - for column_info in columns[: self.queue_model.columnCount() - 1]: - if isinstance(column_info, dict): - text = str(column_info.get('text', '')) - column_tooltip = column_info.get('tooltip', '') - alignment = column_info.get( - 'alignment', Qt.AlignLeft | Qt.AlignVCenter - ) - else: - text = str(column_info) - column_tooltip = '' - alignment = Qt.AlignLeft | Qt.AlignVCenter - column_item = QStandardItem(text) - column_item.setEditable(False) - column_item.setToolTip(column_tooltip) - column_item.setTextAlignment(alignment) - row_items.append(column_item) - - row_items.extend( - self._create_padding_items(self.queue_model.columnCount() - len(row_items)) + row_items = self._create_padding_items(self.queue_model.columnCount()) + row_items[self.path_column] = self._create_display_item( + label, + tooltip=tooltip, + path=path, ) + extra_columns = [i for i in range(self.queue_model.columnCount()) if i != self.path_column] + for column_index, column_info in zip(extra_columns, columns): + row_items[column_index] = self._create_display_item_from_info(column_info) return row_items def _create_padding_items(self, count): items = [] for _ in range(max(0, count)): - item = QStandardItem('') - item.setEditable(False) - items.append(item) + items.append(self._create_display_item('')) return items + def _create_display_item(self, text, tooltip='', alignment=None, path=None): + item = QStandardItem(str(text)) + item.setEditable(False) + item.setToolTip(tooltip) + if alignment is not None: + item.setTextAlignment(alignment) + if path is not None: + item.setData(os.path.abspath(str(path)), Qt.UserRole) + return item + + def _create_display_item_from_info(self, column_info): + if isinstance(column_info, dict): + text = column_info.get('text', '') + tooltip = column_info.get('tooltip', '') + alignment = column_info.get('alignment', Qt.AlignLeft | Qt.AlignVCenter) + else: + text = column_info + tooltip = '' + alignment = Qt.AlignLeft | Qt.AlignVCenter + return self._create_display_item(text, tooltip=tooltip, alignment=alignment) + def _queue_path_for_row(self, row): - item = self.queue_model.item(row, FILEPATH_COLUMN) + item = self.queue_model.item(row, self.path_column) path = item.data(Qt.UserRole) if path is None: path = item.text() From 9e581dbc1892cb6399c75f40bc2400a7c5696585 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:16:40 -0400 Subject: [PATCH 23/42] Use external scrollbar for shot queue widget --- labscript_utils/qtwidgets/shotqueue.py | 43 +++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 0183b187..99d553c8 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -151,7 +151,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff) self.queue_view.header().setStretchLastSection(False) for column in range(len(self.column_titles)): resize_mode = ( @@ -160,6 +160,7 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) + self.queue_scrollbar = QScrollBar(Qt.Vertical, self) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -169,9 +170,15 @@ def __init__( button_layout.addWidget(self.add_button) button_layout.addStretch(1) + queue_layout = QHBoxLayout() + queue_layout.setContentsMargins(0, 0, 0, 0) + queue_layout.setSpacing(0) + queue_layout.addWidget(self.queue_view) + queue_layout.addWidget(self.queue_scrollbar) + layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) - layout.addWidget(self.queue_view) + layout.addLayout(queue_layout) layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) @@ -181,7 +188,12 @@ def __init__( self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) + self.queue_view.verticalScrollBar().rangeChanged.connect(self._sync_vertical_scrollbar) + self.queue_view.verticalScrollBar().valueChanged.connect(self.queue_scrollbar.setValue) + self.queue_scrollbar.valueChanged.connect(self.queue_view.verticalScrollBar().setValue) + self._sync_vertical_scrollbar() + self._resize_auxiliary_columns() self._update_button_states() def files(self): @@ -216,6 +228,7 @@ def append(self, paths): return [] for path in paths: self.queue_model.appendRow(self._create_row(path)) + self._resize_auxiliary_columns() self.filesAdded.emit(paths) return paths @@ -224,6 +237,7 @@ def prepend(self, path): if not paths: return [] self.queue_model.insertRow(0, self._create_row(paths[0])) + self._resize_auxiliary_columns() self.filesAdded.emit(paths) self._select_rows([0]) return paths @@ -310,9 +324,11 @@ def restore_save_data(self, data): def set_row_infos(self, row_infos): self.queue_model.removeRows(0, self.queue_model.rowCount()) if not row_infos: + self._resize_auxiliary_columns() return for row_info in row_infos: self.queue_model.appendRow(self._create_row_from_info(row_info)) + self._resize_auxiliary_columns() def _create_row(self, path): row_items = self._create_padding_items(self.queue_model.columnCount()) @@ -375,6 +391,22 @@ def _queue_path_for_row(self, row): path = item.text() return os.path.abspath(str(path)) + def _resize_auxiliary_columns(self): + header = self.queue_view.header() + for column, title in enumerate(self.column_titles): + if column == self.path_column: + continue + title_width = self.fontMetrics().horizontalAdvance(title) + 24 + content_width = max(0, self.queue_view.sizeHintForColumn(column)) + 12 + header.resizeSection(column, max(title_width, content_width)) + + def _sync_vertical_scrollbar(self, minimum=0, maximum=0): + tree_scrollbar = self.queue_view.verticalScrollBar() + self.queue_scrollbar.setRange(minimum, maximum) + self.queue_scrollbar.setPageStep(tree_scrollbar.pageStep()) + self.queue_scrollbar.setSingleStep(tree_scrollbar.singleStep()) + self.queue_scrollbar.setValue(tree_scrollbar.value()) + def _prepare_paths(self, paths): if isinstance(paths, str): paths = [paths] @@ -395,6 +427,9 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) def _on_queue_changed(self, *args): + self._resize_auxiliary_columns() + tree_scrollbar = self.queue_view.verticalScrollBar() + self._sync_vertical_scrollbar(tree_scrollbar.minimum(), tree_scrollbar.maximum()) self._update_button_states() self.queueChanged.emit() @@ -410,7 +445,7 @@ def _select_rows(self, rows): selection_model = self.queue_view.selectionModel() selection_model.clearSelection() for row in rows: - index = self.queue_model.index(row, FILEPATH_COLUMN) + index = self.queue_model.index(row, self.path_column) selection_model.select(index, QItemSelectionModel.Select | QItemSelectionModel.Rows) if rows: - self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], FILEPATH_COLUMN)) + self.queue_view.setCurrentIndex(self.queue_model.index(rows[0], self.path_column)) From fe14e619000cd080e65558d9390466df99f31719 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:19:15 -0400 Subject: [PATCH 24/42] Reserve corner space for shot queue scrollbar --- labscript_utils/qtwidgets/shotqueue.py | 42 +++----------------------- 1 file changed, 5 insertions(+), 37 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 99d553c8..140e3339 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -151,7 +151,7 @@ def __init__( self.queue_view = ShotQueueTreeView(self, accepted_extensions=self.accepted_extensions) self.queue_view.setModel(self.queue_model) - self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + self.queue_view.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.queue_view.header().setStretchLastSection(False) for column in range(len(self.column_titles)): resize_mode = ( @@ -160,7 +160,9 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) - self.queue_scrollbar = QScrollBar(Qt.Vertical, self) + self.scrollbar_corner = QWidget(self.queue_view) + self.scrollbar_corner.setFixedWidth(self.queue_view.verticalScrollBar().sizeHint().width()) + self.queue_view.setCornerWidget(self.scrollbar_corner) self.add_button = QToolButton(self) self.add_button.setText('Add') @@ -170,15 +172,9 @@ def __init__( button_layout.addWidget(self.add_button) button_layout.addStretch(1) - queue_layout = QHBoxLayout() - queue_layout.setContentsMargins(0, 0, 0, 0) - queue_layout.setSpacing(0) - queue_layout.addWidget(self.queue_view) - queue_layout.addWidget(self.queue_scrollbar) - layout = QVBoxLayout(self) layout.setContentsMargins(0, 0, 0, 0) - layout.addLayout(queue_layout) + layout.addWidget(self.queue_view) layout.addLayout(button_layout) self.add_button.clicked.connect(self.prompt_for_files) @@ -188,12 +184,7 @@ def __init__( self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) self.queue_view.selectionModel().selectionChanged.connect(self._on_selection_changed) - self.queue_view.verticalScrollBar().rangeChanged.connect(self._sync_vertical_scrollbar) - self.queue_view.verticalScrollBar().valueChanged.connect(self.queue_scrollbar.setValue) - self.queue_scrollbar.valueChanged.connect(self.queue_view.verticalScrollBar().setValue) - self._sync_vertical_scrollbar() - self._resize_auxiliary_columns() self._update_button_states() def files(self): @@ -228,7 +219,6 @@ def append(self, paths): return [] for path in paths: self.queue_model.appendRow(self._create_row(path)) - self._resize_auxiliary_columns() self.filesAdded.emit(paths) return paths @@ -237,7 +227,6 @@ def prepend(self, path): if not paths: return [] self.queue_model.insertRow(0, self._create_row(paths[0])) - self._resize_auxiliary_columns() self.filesAdded.emit(paths) self._select_rows([0]) return paths @@ -324,11 +313,9 @@ def restore_save_data(self, data): def set_row_infos(self, row_infos): self.queue_model.removeRows(0, self.queue_model.rowCount()) if not row_infos: - self._resize_auxiliary_columns() return for row_info in row_infos: self.queue_model.appendRow(self._create_row_from_info(row_info)) - self._resize_auxiliary_columns() def _create_row(self, path): row_items = self._create_padding_items(self.queue_model.columnCount()) @@ -391,22 +378,6 @@ def _queue_path_for_row(self, row): path = item.text() return os.path.abspath(str(path)) - def _resize_auxiliary_columns(self): - header = self.queue_view.header() - for column, title in enumerate(self.column_titles): - if column == self.path_column: - continue - title_width = self.fontMetrics().horizontalAdvance(title) + 24 - content_width = max(0, self.queue_view.sizeHintForColumn(column)) + 12 - header.resizeSection(column, max(title_width, content_width)) - - def _sync_vertical_scrollbar(self, minimum=0, maximum=0): - tree_scrollbar = self.queue_view.verticalScrollBar() - self.queue_scrollbar.setRange(minimum, maximum) - self.queue_scrollbar.setPageStep(tree_scrollbar.pageStep()) - self.queue_scrollbar.setSingleStep(tree_scrollbar.singleStep()) - self.queue_scrollbar.setValue(tree_scrollbar.value()) - def _prepare_paths(self, paths): if isinstance(paths, str): paths = [paths] @@ -427,9 +398,6 @@ def _is_accepted_path(self, path): return os.path.isfile(path) and path.lower().endswith(self.accepted_extensions) def _on_queue_changed(self, *args): - self._resize_auxiliary_columns() - tree_scrollbar = self.queue_view.verticalScrollBar() - self._sync_vertical_scrollbar(tree_scrollbar.minimum(), tree_scrollbar.maximum()) self._update_button_states() self.queueChanged.emit() From 94d9d2e90b5d6adc49ba86aa843cad36d029a213 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:21:44 -0400 Subject: [PATCH 25/42] Remove shot queue scrollbar corner workaround --- labscript_utils/qtwidgets/shotqueue.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index 140e3339..f2f01c0b 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -160,9 +160,6 @@ def __init__( else QHeaderView.ResizeToContents ) self.queue_view.header().setSectionResizeMode(column, resize_mode) - self.scrollbar_corner = QWidget(self.queue_view) - self.scrollbar_corner.setFixedWidth(self.queue_view.verticalScrollBar().sizeHint().width()) - self.queue_view.setCornerWidget(self.scrollbar_corner) self.add_button = QToolButton(self) self.add_button.setText('Add') From c515061415e3d3f1b268d243b5d94dc34b26967c Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 6 Apr 2026 12:24:46 -0400 Subject: [PATCH 26/42] Make shot queue default hookups optional --- labscript_utils/qtwidgets/shotqueue.py | 47 +++++--------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/labscript_utils/qtwidgets/shotqueue.py b/labscript_utils/qtwidgets/shotqueue.py index f2f01c0b..239dbfa1 100644 --- a/labscript_utils/qtwidgets/shotqueue.py +++ b/labscript_utils/qtwidgets/shotqueue.py @@ -133,6 +133,9 @@ def __init__( column_title='Filepath', column_titles=None, path_column=FILEPATH_COLUMN, + connect_add_button=True, + connect_delete_requested=True, + connect_files_dropped=True, ): QWidget.__init__(self, parent) self.accepted_extensions = _normalise_extensions(accepted_extensions) @@ -174,9 +177,12 @@ def __init__( layout.addWidget(self.queue_view) layout.addLayout(button_layout) - self.add_button.clicked.connect(self.prompt_for_files) - self.queue_view.deleteRequested.connect(self.remove_selected) - self.queue_view.filesDropped.connect(self.add_files) + if connect_add_button: + self.add_button.clicked.connect(self.prompt_for_files) + if connect_delete_requested: + self.queue_view.deleteRequested.connect(self.remove_selected) + if connect_files_dropped: + self.queue_view.filesDropped.connect(self.add_files) self.queue_model.rowsInserted.connect(self._on_queue_changed) self.queue_model.rowsRemoved.connect(self._on_queue_changed) self.queue_model.modelReset.connect(self._on_queue_changed) @@ -255,41 +261,6 @@ def clear(self): if self.queue_model.rowCount(): self.queue_model.removeRows(0, self.queue_model.rowCount()) - def move_up(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[0] == 0: - return - for row_index in selected_rows: - self.queue_model.insertRow(row_index - 1, self.queue_model.takeRow(row_index)) - self._select_rows([row - 1 for row in selected_rows]) - - def move_down(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: - return - for row_index in reversed(selected_rows): - self.queue_model.insertRow(row_index + 1, self.queue_model.takeRow(row_index)) - self._select_rows([row + 1 for row in selected_rows]) - - def move_top(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[0] == 0: - return - rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] - for offset, row_items in enumerate(reversed(rows)): - self.queue_model.insertRow(offset, row_items) - self._select_rows(range(len(selected_rows))) - - def move_bottom(self): - selected_rows = self.selected_rows() - if not selected_rows or selected_rows[-1] == self.queue_model.rowCount() - 1: - return - rows = [self.queue_model.takeRow(row_index) for row_index in reversed(selected_rows)] - start_row = self.queue_model.rowCount() - for row_items in reversed(rows): - self.queue_model.appendRow(row_items) - self._select_rows(range(start_row, self.queue_model.rowCount())) - def is_in_queue(self, path): path = os.path.abspath(str(path)) return any( From 65f8ccde64110dd1e5c59a56e76e3e6d91c1173b Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 11:32:34 -0400 Subject: [PATCH 27/42] Add lookup formatter for runmanager templates --- labscript_utils/lookup_format.py | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 labscript_utils/lookup_format.py diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py new file mode 100644 index 00000000..a649cc28 --- /dev/null +++ b/labscript_utils/lookup_format.py @@ -0,0 +1,147 @@ +##################################################################### +# # +# lookup_format.py # +# # +# Copyright 2026, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import ast +import string + + +class InvalidLookupFormatField(ValueError): + """Raised when a lookup-format field is not valid lookup-only syntax.""" + + +def format_lookup_string(template, context, dt=None, preserve_unresolved=False): + """Format a template using optional ``strftime`` plus lookup-only fields. + + Args: + template (str): Format template. + context (dict): Top-level lookup context. + dt (datetime, optional): Datetime used for a first ``strftime`` pass. + preserve_unresolved (bool): If True, unresolved lookups are left intact. + """ + if dt is not None: + template = dt.strftime(template) + formatter = _LookupFormatter(context, preserve_unresolved=preserve_unresolved) + return formatter.format(template) + + +class _LookupFormatter(object): + def __init__(self, context, preserve_unresolved=False): + self.context = context + self.preserve_unresolved = preserve_unresolved + self.formatter = string.Formatter() + + def format(self, template): + parts = [] + for literal_text, field_name, format_spec, conversion in self.formatter.parse(template): + parts.append(literal_text) + if field_name is None: + continue + placeholder = self._reconstruct_placeholder( + field_name, conversion, format_spec + ) + try: + value = self._resolve_field(field_name) + if conversion: + value = self.formatter.convert_field(value, conversion) + if format_spec: + format_spec = self.format(format_spec) + parts.append(self.formatter.format_field(value, format_spec)) + except KeyError: + if self.preserve_unresolved: + parts.append(placeholder) + else: + raise + return ''.join(parts) + + def _resolve_field(self, field_name): + root_name, lookups = _parse_lookup_field(field_name) + try: + value = self.context[root_name] + except KeyError: + raise KeyError(root_name) + for lookup in lookups: + try: + value = value[lookup] + except KeyError: + raise + return value + + @staticmethod + def _reconstruct_placeholder(field_name, conversion, format_spec): + placeholder = '{' + field_name + if conversion: + placeholder += '!' + conversion + if format_spec: + placeholder += ':' + format_spec + placeholder += '}' + return placeholder + + +def _parse_lookup_field(field_name): + field_name = str(field_name) + length = len(field_name) + index = 0 + + root_name, index = _parse_identifier(field_name, index) + lookups = [] + + while index < length: + if field_name[index] != '[': + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + closing_index = field_name.find(']', index + 1) + if closing_index == -1: + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + lookup_text = field_name[index + 1 : closing_index].strip() + if not lookup_text: + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + lookups.append(_parse_lookup_key(lookup_text)) + index = closing_index + 1 + + return root_name, lookups + + +def _parse_identifier(field_name, index): + if index >= len(field_name): + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + + start = index + first = field_name[index] + if not (first.isalpha() or first == '_'): + msg = "Invalid lookup-format field %r" % field_name + raise InvalidLookupFormatField(msg) + index += 1 + + while index < len(field_name): + char = field_name[index] + if char.isalnum() or char == '_': + index += 1 + continue + break + + return field_name[start:index], index + + +def _parse_lookup_key(lookup_text): + if lookup_text[0] in ('"', "'"): + try: + return ast.literal_eval(lookup_text) + except (SyntaxError, ValueError) as exc: + msg = "Invalid lookup-format key %r" % lookup_text + raise InvalidLookupFormatField(msg) from exc + try: + return int(lookup_text) + except ValueError: + return lookup_text From d0175eb984d3129c575159c7de10d18e4f69c5ca Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 11:58:15 -0400 Subject: [PATCH 28/42] Tighten lookup format unresolved handling --- labscript_utils/lookup_format.py | 42 ++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/labscript_utils/lookup_format.py b/labscript_utils/lookup_format.py index a649cc28..51d1cb73 100644 --- a/labscript_utils/lookup_format.py +++ b/labscript_utils/lookup_format.py @@ -18,7 +18,21 @@ class InvalidLookupFormatField(ValueError): """Raised when a lookup-format field is not valid lookup-only syntax.""" -def format_lookup_string(template, context, dt=None, preserve_unresolved=False): +class _UnresolvedLookupError(LookupError): + """Raised when a lookup could not be resolved from the provided context.""" + + def __init__(self, root_name): + LookupError.__init__(self, root_name) + self.root_name = root_name + + +def format_lookup_string( + template, + context, + dt=None, + preserve_unresolved=False, + preserve_unresolved_roots=(), +): """Format a template using optional ``strftime`` plus lookup-only fields. Args: @@ -26,17 +40,29 @@ def format_lookup_string(template, context, dt=None, preserve_unresolved=False): context (dict): Top-level lookup context. dt (datetime, optional): Datetime used for a first ``strftime`` pass. preserve_unresolved (bool): If True, unresolved lookups are left intact. + preserve_unresolved_roots (iterable): Root names that may be preserved when a + lookup below them is unresolved. """ if dt is not None: template = dt.strftime(template) - formatter = _LookupFormatter(context, preserve_unresolved=preserve_unresolved) + formatter = _LookupFormatter( + context, + preserve_unresolved=preserve_unresolved, + preserve_unresolved_roots=preserve_unresolved_roots, + ) return formatter.format(template) class _LookupFormatter(object): - def __init__(self, context, preserve_unresolved=False): + def __init__( + self, + context, + preserve_unresolved=False, + preserve_unresolved_roots=(), + ): self.context = context self.preserve_unresolved = preserve_unresolved + self.preserve_unresolved_roots = set(preserve_unresolved_roots) self.formatter = string.Formatter() def format(self, template): @@ -55,8 +81,8 @@ def format(self, template): if format_spec: format_spec = self.format(format_spec) parts.append(self.formatter.format_field(value, format_spec)) - except KeyError: - if self.preserve_unresolved: + except _UnresolvedLookupError as exc: + if self.preserve_unresolved or exc.root_name in self.preserve_unresolved_roots: parts.append(placeholder) else: raise @@ -67,12 +93,12 @@ def _resolve_field(self, field_name): try: value = self.context[root_name] except KeyError: - raise KeyError(root_name) + raise _UnresolvedLookupError(root_name) for lookup in lookups: try: value = value[lookup] - except KeyError: - raise + except (IndexError, KeyError, TypeError): + raise _UnresolvedLookupError(root_name) return value @staticmethod From f1d4a9885e731be5a27fca83ba3119d74f1b7796 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 7 Apr 2026 13:02:31 -0400 Subject: [PATCH 29/42] Show active config path in window title --- labscript_utils/labconfig.py | 71 ++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/labscript_utils/labconfig.py b/labscript_utils/labconfig.py index d110fe82..72f61eab 100644 --- a/labscript_utils/labconfig.py +++ b/labscript_utils/labconfig.py @@ -13,6 +13,7 @@ import os import configparser from ast import literal_eval +from getpass import getuser from pprint import pformat from pathlib import Path import warnings @@ -23,6 +24,76 @@ default_config_path = default_labconfig_path() +def format_path_for_display(path): + """Return an absolute path with the user's home abbreviated for display.""" + absolute_path = os.path.abspath(os.fspath(path)) + try: + home_path = str(Path("~" + getuser()).expanduser()) + except Exception: + home_path = str(Path.home()) + + normalized_path = os.path.normcase(os.path.normpath(absolute_path)) + normalized_home = os.path.normcase(os.path.normpath(home_path)) + if normalized_path == normalized_home: + return '%USERPROFILE%' if os.name == 'nt' else '~' + + home_prefix = normalized_home + os.path.sep + if normalized_path.startswith(home_prefix): + relative_path = os.path.relpath(absolute_path, home_path) + prefix = '%USERPROFILE%' if os.name == 'nt' else '~' + separator = '\\' if os.name == 'nt' else '/' + return prefix + separator + relative_path.replace(os.path.sep, separator) + + return absolute_path + + +def get_default_appconfig_file( + exp_config, app_name, config_filename, ensure_directory=False +): + try: + default_path = os.path.join(exp_config.get('DEFAULT', 'app_saved_configs'), app_name) + except (LabConfig.NoOptionError, LabConfig.NoSectionError): + exp_config.set( + 'DEFAULT', + 'app_saved_configs', + os.path.join( + '%(labscript_suite)s', 'userlib', 'app_saved_configs', + '%(apparatus_name)s' + ), + ) + default_path = os.path.join( + exp_config.get('DEFAULT', 'app_saved_configs'), app_name + ) + if ensure_directory and not os.path.exists(default_path): + os.makedirs(default_path) + return os.path.join(default_path, config_filename) + + +class LabscriptApplication(object): + app_name = None + default_config_filename = None + + def init_config_window_title(self): + self.base_window_title = self.ui.windowTitle().split(' - ', 1)[0] + + def get_default_config_file(self, ensure_directory=False): + if self.app_name is None or self.default_config_filename is None: + raise NotImplementedError( + 'LabscriptApplication requires app_name and default_config_filename' + ) + return get_default_appconfig_file( + self.exp_config, + self.app_name, + self.default_config_filename, + ensure_directory=ensure_directory, + ) + + def set_config_window_title(self, filename): + self.ui.setWindowTitle( + '{} - {}'.format(self.base_window_title, format_path_for_display(filename)) + ) + + class EnvInterpolation(configparser.BasicInterpolation): """Interpolation which expands environment variables in values, by post-filtering BasicInterpolation.before_get()""" From 465943d1eaae217394043a0f1d44c52d6e56f714 Mon Sep 17 00:00:00 2001 From: spielman Date: Fri, 24 Apr 2026 22:05:46 -0400 Subject: [PATCH 30/42] Add shared plugin framework --- labscript_utils/plugins.py | 333 +++++++++++++++++++++++++++++++++++++ tests/test_plugins.py | 228 +++++++++++++++++++++++++ 2 files changed, 561 insertions(+) create mode 100644 labscript_utils/plugins.py create mode 100644 tests/test_plugins.py diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py new file mode 100644 index 00000000..4321ccb0 --- /dev/null +++ b/labscript_utils/plugins.py @@ -0,0 +1,333 @@ +##################################################################### +# # +# plugins.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### +import importlib +import logging +import os +from types import MethodType + + +DEFAULT_PRIORITY = 10 + +__all__ = [ + 'DEFAULT_PRIORITY', + 'Callback', + 'callback', + 'BasePlugin', + 'MenuBuilder', + 'PluginManager', +] + + +class Callback(object): + """Class wrapping a callable. At present only differs from a regular + function in that it has a "priority" attribute - lower numbers means + higher priority. If there are multiple callbacks triggered by the same + event, they will be returned in order of priority by get_callbacks""" + def __init__(self, func, priority=DEFAULT_PRIORITY): + self.priority = priority + self.func = func + + def __get__(self, instance, class_): + """Make sure our callable binds like an instance method. Otherwise + __call__ doesn't get the instance argument.""" + if instance is None: + return self + else: + return MethodType(self, instance) + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + +class callback(object): + """Decorator to turn a function into a Callback object. Presently + optional, and only required if the callback needs to have a non-default + priority set""" + # Instantiate the decorator: + def __init__(self, priority=DEFAULT_PRIORITY): + self.priority = priority + + # Call the decorator + def __call__(self, func): + return Callback(func, self.priority) + + +class BasePlugin(object): + """Default no-op implementation of the existing plugin interface.""" + def __init__(self, initial_settings): + self.initial_settings = initial_settings + self.menu = None + self.notifications = {} + + def get_menu_class(self): + return None + + def get_notification_classes(self): + return [] + + def get_setting_classes(self): + return [] + + def get_callbacks(self): + return None + + def get_tab_classes(self): + return {} + + def tabs_created(self, tabs): + pass + + def set_menu_instance(self, menu): + self.menu = menu + + def set_notification_instances(self, notifications): + self.notifications = notifications + + def plugin_setup_complete(self, data=None): + pass + + def get_save_data(self): + return {} + + def close(self): + pass + + +class MenuBuilder(object): + """Build menus from the nested dictionary format used by BLACS plugins.""" + def __init__(self, icon_factory=None): + self.icon_factory = icon_factory + + def create_menu(self, parent, menu_parameters): + if 'name' in menu_parameters: + if 'menu_items' in menu_parameters: + child = parent.addMenu(menu_parameters['name']) + for child_menu_params in menu_parameters['menu_items']: + self.create_menu(child, child_menu_params) + else: + if 'icon' in menu_parameters and self.icon_factory is not None: + child = parent.addAction( + self.icon_factory(menu_parameters['icon']), + menu_parameters['name'], + ) + else: + child = parent.addAction(menu_parameters['name']) + + if 'action' in menu_parameters: + child.triggered.connect(menu_parameters['action']) + + elif 'separator' in menu_parameters: + parent.addSeparator() + + +class PluginManager(object): + """Manage discovery and setup of plugins using the existing plugin API.""" + def __init__( + self, + plugin_package, + plugins_dir, + config, + config_section, + default_plugins=(), + logger=None, + ): + self.plugin_package = plugin_package + self.plugins_dir = plugins_dir + self.config = config + self.config_section = config_section + self.default_plugins = set(default_plugins) + self.logger = logger or logging.getLogger(__name__) + self.modules = {} + self.plugins = {} + + def discover_modules(self): + if not self.config.has_section(self.config_section): + self.config.add_section(self.config_section) + + configured_plugins = set( + name for name, val in self.config.items(self.config_section) + ) + + modules = {} + for module_name in os.listdir(self.plugins_dir): + module_path = os.path.join(self.plugins_dir, module_name) + if not os.path.isdir(module_path) or module_name == '__pycache__': + continue + + # If this is a new plugin, add it to the config. + if module_name not in configured_plugins: + self.config.set( + self.config_section, + module_name, + str(module_name in self.default_plugins), + ) + configured_plugins.add(module_name) + + # Only load activated plugins. + if self.config.getboolean(self.config_section, module_name): + try: + module = importlib.import_module( + self.plugin_package + '.' + module_name + ) + except Exception: + self.logger.exception( + "Could not import plugin '%s'. Skipping." % module_name + ) + else: + modules[module_name] = module + + self.modules = modules + return modules + + def instantiate_plugins(self, plugin_settings=None): + if plugin_settings is None: + plugin_settings = {} + + plugins = {} + for module_name, module in self.modules.items(): + try: + plugins[module_name] = module.Plugin( + plugin_settings[module_name] + if module_name in plugin_settings else {} + ) + except Exception: + self.logger.exception( + "Could not instantiate plugin '%s'. Skipping" % module_name + ) + + self.plugins = plugins + return plugins + + def create_tabs(self, tab_widget, settings_dict, tablist, settings, tab_data): + for module_name, plugin in self.plugins.items(): + try: + if hasattr(plugin, 'get_tab_classes'): + tab_dict = {} + + for tab_name, TabClass in plugin.get_tab_classes().items(): + settings_key = "{}: {}".format(module_name, tab_name) + settings_dict.setdefault(settings_key, {"tab_name": tab_name}) + settings_dict[settings_key]["front_panel_settings"] = ( + settings[settings_key] if settings_key in settings else {} + ) + settings_dict[settings_key]["saved_data"] = ( + tab_data[settings_key]['data'] + if settings_key in tab_data else {} + ) + + tablist[settings_key] = TabClass( + tab_widget, + settings_dict[settings_key], + ) + tab_dict[tab_name] = tablist[settings_key] + + if hasattr(plugin, 'tabs_created'): + plugin.tabs_created(tab_dict) + + except Exception: + self.logger.exception( + "Could not instantiate tab for plugin '%s'. Skipping" + % module_name + ) + + def setup_plugins(self, data, notifications, menu_builder, menubar): + settings_pages = [] + settings_callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + # Setup settings page. + settings_pages.extend(plugin.get_setting_classes()) + + # Setup menu. + if plugin.get_menu_class(): + # Must store a reference or else the methods called when + # the menu actions are triggered will be garbage collected. + menu = plugin.get_menu_class()(data) + menu_builder.create_menu(menubar, menu.get_menu_items()) + plugin.set_menu_instance(menu) + + # Setup notifications. + plugin_notifications = {} + for notification_class in plugin.get_notification_classes(): + notifications.add_notification(notification_class) + plugin_notifications[notification_class] = ( + notifications.get_instance(notification_class) + ) + plugin.set_notification_instances(plugin_notifications) + + # Register callbacks. + callbacks = plugin.get_callbacks() + # Save the settings_changed callback in a separate list for + # setting up later. + if isinstance(callbacks, dict) and 'settings_changed' in callbacks: + settings_callbacks.append(callbacks['settings_changed']) + + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." % module_name + ) + + return settings_pages, settings_callbacks + + def setup_complete(self, data): + for module_name, plugin in self.plugins.items(): + try: + plugin.plugin_setup_complete(data) + except Exception: + self.logger.exception( + "Error in plugin_setup_complete() for plugin '%s'. " + "Trying again with old call signature..." % module_name + ) + # Backwards compatibility for old plugins. + try: + plugin.plugin_setup_complete() + self.logger.warning( + "Plugin '%s' using old API. Please update " + "Plugin.plugin_setup_complete method to accept a " + "dictionary of blacs_data as the only argument." + % module_name + ) + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." + % module_name + ) + + def get_callbacks(self, name): + """Return all callbacks for a particular name, in priority order.""" + callbacks = [] + + for plugin in self.plugins.values(): + try: + plugin_callbacks = plugin.get_callbacks() + if plugin_callbacks is not None: + if name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) + except Exception: + self.logger.exception('Error getting callbacks from %s.' % str(plugin)) + + callbacks.sort( + key=lambda callback: getattr(callback, 'priority', DEFAULT_PRIORITY) + ) + return callbacks + + def close_plugins(self): + for module_name, plugin in self.plugins.items(): + try: + plugin.close() + except Exception as e: + self.logger.error( + 'Could not close plugin %s. Error was: %s' + % (module_name, str(e)) + ) diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 00000000..026057b1 --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,228 @@ +import logging +import sys +import uuid +from pathlib import Path +from types import SimpleNamespace + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from labscript_utils.plugins import ( + BasePlugin, + Callback, + MenuBuilder, + PluginManager, + callback, +) + + +class FakeConfig(object): + def __init__(self): + self.sections = {} + + def has_section(self, name): + return name in self.sections + + def add_section(self, name): + self.sections[name] = {} + + def items(self, name): + return list(self.sections[name].items()) + + def set(self, section, option, value): + self.sections[section][option] = value + + def getboolean(self, section, option): + return self.sections[section][option].lower() == 'true' + + +def test_callback_binds_as_method_and_keeps_priority(): + class Example(object): + @callback(priority=5) + def method(self, value): + return self, value + + example = Example() + + assert isinstance(Example.__dict__['method'], Callback) + assert Example.__dict__['method'].priority == 5 + assert example.method('value') == (example, 'value') + + +def test_base_plugin_defaults_are_no_ops(): + plugin = BasePlugin({'x': 1}) + + assert plugin.initial_settings == {'x': 1} + assert plugin.get_menu_class() is None + assert plugin.get_notification_classes() == [] + assert plugin.get_setting_classes() == [] + assert plugin.get_callbacks() is None + assert plugin.get_tab_classes() == {} + assert plugin.get_save_data() == {} + + +def make_plugin_package(tmp_path, modules): + package = 'plugin_test_' + uuid.uuid4().hex + package_dir = tmp_path / package + plugins_dir = package_dir / 'plugins' + plugins_dir.mkdir(parents=True) + (package_dir / '__init__.py').write_text('') + (plugins_dir / '__init__.py').write_text('') + for name, source in modules.items(): + module_dir = plugins_dir / name + module_dir.mkdir() + (module_dir / '__init__.py').write_text(source) + sys.path.insert(0, str(tmp_path)) + return package + '.plugins', plugins_dir + + +def test_discovery_defaults_config_and_imports_only_enabled_plugins(tmp_path): + plugin_package, plugins_dir = make_plugin_package( + tmp_path, + { + 'enabled': 'class Plugin(object):\n pass\n', + 'disabled': 'raise RuntimeError("should not import")\n', + }, + ) + config = FakeConfig() + manager = PluginManager( + plugin_package, + str(plugins_dir), + config, + 'app/plugins', + ['enabled'], + ) + + modules = manager.discover_modules() + + assert set(modules) == {'enabled'} + assert config.sections['app/plugins']['enabled'] == 'True' + assert config.sections['app/plugins']['disabled'] == 'False' + + +def test_instantiate_plugins_uses_saved_settings(): + class Plugin(object): + def __init__(self, initial_settings): + self.initial_settings = initial_settings + + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + ) + manager.modules = {'plugin': SimpleNamespace(Plugin=Plugin)} + + plugins = manager.instantiate_plugins({'plugin': {'answer': 42}}) + + assert plugins['plugin'].initial_settings == {'answer': 42} + + +def test_get_callbacks_sorts_by_priority_and_logs_errors(caplog): + class Slow(object): + def get_callbacks(self): + return {'event': self.on_event} + + @callback(priority=20) + def on_event(self): + pass + + class Fast(object): + def get_callbacks(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + class Broken(object): + def get_callbacks(self): + raise RuntimeError('broken') + + logger = logging.getLogger('test.plugins') + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + manager.plugins = { + 'slow': Slow(), + 'broken': Broken(), + 'fast': Fast(), + } + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + callbacks = manager.get_callbacks('event') + + assert [callback.priority for callback in callbacks] == [5, 20] + assert 'Error getting callbacks from' in caplog.text + + +class FakeSignal(object): + def __init__(self): + self.callbacks = [] + + def connect(self, callback): + self.callbacks.append(callback) + + +class FakeAction(object): + def __init__(self, name, icon=None): + self.name = name + self.icon = icon + self.triggered = FakeSignal() + + +class FakeMenu(object): + def __init__(self, name='root'): + self.name = name + self.items = [] + + def addMenu(self, name): + menu = FakeMenu(name) + self.items.append(('menu', menu)) + return menu + + def addAction(self, *args): + if len(args) == 1: + action = FakeAction(args[0]) + else: + action = FakeAction(args[1], icon=args[0]) + self.items.append(('action', action)) + return action + + def addSeparator(self): + self.items.append(('separator', None)) + + +def test_menu_builder_creates_nested_menus_icons_actions_and_separators(): + root = FakeMenu() + called = [] + + def action(): + called.append(True) + + MenuBuilder(icon_factory=lambda name: 'icon:' + name).create_menu( + root, + { + 'name': 'Top', + 'menu_items': [ + {'name': 'Plain', 'action': action}, + {'separator': True}, + {'name': 'Icon', 'icon': 'resource', 'action': action}, + ], + }, + ) + + top = root.items[0][1] + plain = top.items[0][1] + icon = top.items[2][1] + + assert root.items[0][0] == 'menu' + assert plain.name == 'Plain' + assert plain.triggered.callbacks == [action] + assert top.items[1] == ('separator', None) + assert icon.name == 'Icon' + assert icon.icon == 'icon:resource' + assert icon.triggered.callbacks == [action] From 3cef7b3da44d406b0534e6aefbab978535ab4315 Mon Sep 17 00:00:00 2001 From: spielman Date: Sat, 25 Apr 2026 12:10:49 -0400 Subject: [PATCH 31/42] More generalized plugins. --- labscript_utils/plugins.py | 1030 ++++++++++++++++++++++++++++++++++-- tests/test_plugins.py | 216 +++++++- 2 files changed, 1195 insertions(+), 51 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 4321ccb0..982b32e2 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -10,6 +10,581 @@ # for the full license. # # # ##################################################################### + +"""Plugin discovery, instantiation, and callback plumbing for labscript apps. + +This module is the small framework layer that labscript-suite applications use +to discover plugins, instantiate them, collect app-neutral contributions, and +ask them for callbacks at runtime. The application owns the actual settings +storage, notification manager, menu objects, UI containers, and event firing. +``PluginManager`` only calls plugin hooks and routes contributions to contexts +the application registered explicitly. + +The usual package layout is:: + + myapp/ + plugins/ + __init__.py + example/ + __init__.py # defines class Plugin + +The module name becomes the plugin name. ``PluginManager`` looks for a module +attribute named ``Plugin`` and passes the saved per-plugin settings dictionary +to ``Plugin(initial_settings)``. Existing plugins do not have to inherit +``BasePlugin``; they only need to provide the same methods by duck typing. + +Configuration behavior is intentionally conservative: + +* Missing plugin entries are added to the config when discovered. +* Names listed in ``default_plugins`` are written as enabled by default. +* Only enabled modules are imported. +* Import or instantiation failures are logged and skipped so one broken plugin + does not stop the application from starting. + +``Callback`` and ``callback`` wrap event handlers with priority metadata. +Lower priority numbers run first. ``Callback`` behaves like a descriptor, so +decorated methods still bind to plugin instances normally when accessed through +an instance. + +Two menu paths are supported, intentionally with different contracts: + +* ``MenuBuilder`` is the legacy nested-dictionary renderer used by existing + BLACS-style plugins. It consumes dictionaries with ``name``, ``menu_items``, + ``icon``, ``action``, and ``separator`` keys and renders them immediately + into application menu objects. +* ``MenuContext`` is the shared contribution path for future applications. It + collects actions with explicit ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action`` metadata, then renders all plugins together after + contribution routing is complete. + +``labscript_utils.plugins`` does not know whether a plugin UI belongs in a +tab, MDI subwindow, dock widget, dialog, fixed frame, or plugin-owned modal +container. Plugins declare a named ``context`` and applications register +objects that implement ``add(plugin_name, contribution, data)``. That is the +full shared UI contract. + +Minimal application integration example +--------------------------------------- + +The application owns plugin settings storage, concrete UI objects, and context +implementations. A compact integration with legacy settings/notifications plus +new contribution routing looks like this:: + + from labscript_utils.plugins import MenuBuilder, MenuContext, PluginManager + + + class App(object): + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.plugin_manager = PluginManager( + plugin_package='myapp.plugins', + plugins_dir='/path/to/myapp/plugins', + config=self.config, + config_section='myapp/plugins', + default_plugins=('example',), + logger=self.logger, + ) + + self.plugin_manager.discover_modules() + + plugin_settings = self.load_plugin_settings() + self.plugin_manager.instantiate_plugins(plugin_settings) + + menu_builder = MenuBuilder(icon_factory=self.icon_factory) + settings_pages, settings_callbacks = self.plugin_manager.setup_plugins( + data=self.app_data, + notifications=self.notification_manager, + menu_builder=menu_builder, + menubar=self.menubar, + ) + + menus = MenuContext(icon_factory=self.icon_factory, logger=self.logger) + menus.register_location('file', self.file_menu) + menus.register_location('window', self.window_menu) + + self.plugin_manager.register_context('menus', menus) + self.plugin_manager.register_context('mdi', MDIContext(self.mdi_area)) + self.plugin_manager.register_context( + 'dialogs', + DialogContext(parent=self.main_window), + ) + self.plugin_manager.setup_contexts(self.app_data) + menus.render() + + self.settings_widget = self.build_settings_ui(settings_pages) + for callback in settings_callbacks: + self.register_settings_callback(callback) + + self.plugin_manager.setup_complete(self.app_data) + + def load_plugin_settings(self): + # App-owned storage. Return a mapping from plugin name to dict. + return { + 'example': {'enabled': True}, + } + + def build_settings_ui(self, settings_pages): + # App-owned settings UI. Each item in settings_pages is a plugin + # contributed settings page class. + return settings_pages + + def register_settings_callback(self, callback): + # App-owned settings wiring. + self.settings_changed_callbacks.append(callback) + + def fire_event(self, name): + # App-owned event firing. PluginManager only returns callbacks. + for callback in self.plugin_manager.get_callbacks(name): + callback() + + def close_plugins(self): + self.plugin_manager.close_plugins() + + +Minimal plugin package example +------------------------------ + +This example shows the shared plugin surface. The plugin contributes a legacy +menu, a contribution-based menu action, a settings page, a notification, an MDI +window request, callbacks, ``setup_complete()`` handling, save data, and +shutdown cleanup. Methods can return empty lists or dictionaries when a feature +is not used:: + + from labscript_utils.plugins import BasePlugin, callback + + + class SettingsPage(object): + pass + + + class Notification(object): + pass + + + class DataBrowser(object): + def __init__(self, parent, services): + self.parent = parent + self.services = services + + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Legacy Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Legacy Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.saved_state = initial_settings + self.app_data = None + + def get_menu_class(self): + return Menu + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'path': ('Examples',), + 'group': 'open', + 'order': 20, + 'name': 'Open Data Browser', + 'shortcut': 'Ctrl+D', + 'icon': 'folder-open', + 'action': self.open_data_browser, + }, + ] + + def get_ui_contributions(self): + return [ + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + ] + + def get_notification_classes(self): + return [Notification] + + def get_setting_classes(self): + return [SettingsPage] + + def get_callbacks(self): + return { + 'settings_changed': self.on_settings_changed, + 'shot_complete': self.on_shot_complete, + } + + def plugin_setup_complete(self, data): + self.app_data = data + + def get_save_data(self): + return {'saved_state': self.saved_state} + + def close(self): + pass + + def open_data_browser(self): + pass + + @callback(priority=5) + def on_settings_changed(self): + pass + + @callback(priority=20) + def on_shot_complete(self): + pass + + +Feature excerpts and expected UI +-------------------------------- + +BasePlugin subclass with saved ``initial_settings`` and ``get_save_data()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The constructor receives the per-plugin saved settings dictionary. Store it if +you need to restore state later. ``get_save_data()`` should return the +serializable data you want written back for the next application start:: + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.initial_settings = initial_settings + + def get_save_data(self): + return { + 'window_geometry': self.window_geometry, + 'last_path': self.last_path, + } + +Settings page contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_setting_classes()`` returns the page classes the application should add +to its settings UI. The application owns the widget construction and storage:: + + class Plugin(BasePlugin): + def get_setting_classes(self): + return [SettingsPage] + + class SettingsPage(object): + pass + +Expected UI result: the application shows one settings page for each returned +class, using the app's own settings shell and persistence rules. + +Legacy menu skeleton with MenuBuilder +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_class()`` returns a class whose instance exposes +``get_menu_items()``. The nested dictionaries are translated into the app's +menu objects by ``MenuBuilder``. It expects menus and actions to support +``addMenu()``, ``addAction()``, and ``addSeparator()``:: + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + +Expected UI result: the app user sees a top-level menu labeled ``Example``, +with an ``Open`` action, a separator, and a ``Settings`` action. If the app +supplies ``icon_factory``, the ``Settings`` action can also show an icon. + +Contribution menu skeleton with MenuContext +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_contributions()`` returns dictionaries routed to the application +``menus`` context. ``MenuContext`` understands these keys: + +* ``location``: app-registered top-level menu id, such as ``file`` or + ``window`` +* ``path``: optional submenu path below that location; defaults to ``()`` +* ``group``: separator group within the same location/path; defaults to + ``None`` +* ``order``: numeric order within the group; defaults to ``DEFAULT_PRIORITY`` +* ``name``: action text +* ``action``: callable connected to the action's ``triggered`` signal +* ``shortcut``, ``icon``, ``checkable``, and ``enabled``: optional action + properties applied when the concrete action object supports them + +File-dialog-style plugin example:: + + class FileDialogsPlugin(BasePlugin): + def __init__(self, initial_settings): + super(FileDialogsPlugin, self).__init__(initial_settings) + self.services = None + + def plugin_setup_complete(self, data): + self.services = data['project_services'] + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'group': 'project', + 'order': 10, + 'name': 'New Project', + 'shortcut': 'Ctrl+N', + 'action': self.new_project, + }, + { + 'location': 'file', + 'group': 'project', + 'order': 20, + 'name': 'Open Project...', + 'shortcut': 'Ctrl+O', + 'action': self.open_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 10, + 'name': 'Save', + 'shortcut': 'Ctrl+S', + 'action': self.save_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 20, + 'name': 'Save As...', + 'shortcut': 'Ctrl+Shift+S', + 'action': self.save_project_as, + }, + { + 'location': 'file', + 'group': 'application', + 'order': 100, + 'name': 'Quit', + 'shortcut': 'Ctrl+Q', + 'action': self.quit_application, + }, + ] + + def new_project(self): + self.services.new_project_dialog.open() + + def open_project(self): + self.services.load_project_dialog.open() + + def save_project(self): + self.services.save_project_command.execute() + + def save_project_as(self): + self.services.save_as_project_dialog.open() + + def quit_application(self): + self.services.quit_command.execute() + +Expected UI result: the application renders the actions in its registered +``file`` menu, creates separators between the ``project``, ``save``, and +``application`` groups, and applies shortcuts without the shared framework +knowing what a project is. + +App-neutral UI homing with contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_ui_contributions()`` returns dictionaries with a required ``context``. +The rest of the dictionary is application-defined. This is enough to support +known containers, deferred containers, and plugin-provided containers without +adding type-specific methods such as ``create_tabs()`` or +``create_mdi_windows()`` to ``PluginManager``:: + + class KnownContainerContext(object): + def __init__(self, tab_widget): + self.tab_widget = tab_widget + + def add(self, plugin_name, contribution, data): + widget = contribution['factory']( + self.tab_widget, + data['settings'][contribution['key']], + ) + self.tab_widget.addTab(widget, contribution['title']) + + + class DeferredMDIContext(object): + def __init__(self, mdi_area): + self.mdi_area = mdi_area + self.openers = {} + + def add(self, plugin_name, contribution, data): + key = contribution['key'] + + def open_window(): + widget = contribution['factory'](parent=self.mdi_area, data=data) + self.mdi_area.addSubWindow(widget) + widget.show() + return widget + + self.openers[key] = open_window + + + class DialogContext(object): + def __init__(self, parent): + self.parent = parent + self.dialogs = {} + + def add(self, plugin_name, contribution, data): + self.dialogs[contribution['key']] = contribution['factory']( + parent=self.parent, + services=data['services'], + ) + + + class Plugin(BasePlugin): + def get_ui_contributions(self): + return [ + { + 'context': 'tabs', + 'key': 'overview', + 'title': 'Overview', + 'factory': OverviewWidget, + }, + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + { + 'context': 'dialogs', + 'key': 'open_project', + 'title': 'Open Project', + 'factory': OpenProjectDialog, + }, + ] + +Expected UI result: each application-owned context decides how to instantiate, +store, parent, restore, defer, activate, or display its contribution. Unknown +contexts are logged and skipped by ``PluginManager.setup_contexts()``. + +Notification contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_notification_classes()`` returns notification classes. The application +owns the notification manager and decides how the resulting widgets are shown:: + + class Plugin(BasePlugin): + def get_notification_classes(self): + return [Notification] + + class Notification(object): + pass + +Expected UI result: the application creates whatever notification widgets or +panels it normally uses for those classes and inserts them into its own +notification area. + +Callbacks with priorities +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``@callback(priority=...)`` wraps a method in ``Callback`` metadata. Lower +numbers run first when the application asks ``PluginManager.get_callbacks()``:: + + class SlowPlugin(BasePlugin): + def get_callbacks(self): + return { + 'shot_complete': self.on_slow, + } + + @callback(priority=20) + def on_slow(self): + pass + + class FastPlugin(BasePlugin): + def get_callbacks(self): + return { + 'shot_complete': self.on_fast, + } + + @callback(priority=5) + def on_fast(self): + pass + +Expected runtime order: when the application fires ``shot_complete`` and calls +``plugin_manager.get_callbacks('shot_complete')``, the priority 5 callback +runs before the priority 20 callback, even if the plugins were discovered in a +different order. + +``setup_complete()`` for app data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup_complete()`` is called after the application has finished startup. +Use it to access app-owned state that was not available during construction, +such as the main window, settings object, registered services, or concrete +context objects:: + + class Plugin(BasePlugin): + def plugin_setup_complete(self, data): + self.app = data['app'] + self.settings = data['settings'] + self.services = data['services'] + +The ``data`` dictionary is app-defined. Old plugins may still define +``plugin_setup_complete(self)`` without the ``data`` argument, and the manager +keeps that compatibility path. + +Shutdown cleanup with ``close()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``close()`` is the plugin shutdown hook. Use it to stop timers, threads, +listeners, or any other plugin-owned resources:: + + class Plugin(BasePlugin): + def close(self): + self.worker.stop() + self.worker.wait() + +Expected behavior: the application calls each plugin's ``close()`` method +during shutdown so the plugin can release resources before exit. +""" + import importlib import logging import os @@ -24,22 +599,27 @@ 'callback', 'BasePlugin', 'MenuBuilder', + 'MenuContext', 'PluginManager', ] class Callback(object): - """Class wrapping a callable. At present only differs from a regular - function in that it has a "priority" attribute - lower numbers means - higher priority. If there are multiple callbacks triggered by the same - event, they will be returned in order of priority by get_callbacks""" + """Wrap a callable with priority metadata and method-style binding. + + ``priority`` is used when ``PluginManager.get_callbacks()`` gathers all + callbacks for a named event. Lower numbers run first. + + The descriptor protocol is implemented so that a ``Callback`` stored as a + class attribute binds like an instance method when accessed from a plugin + instance. This keeps decorated methods usable without extra wrapper code. + """ def __init__(self, func, priority=DEFAULT_PRIORITY): self.priority = priority self.func = func def __get__(self, instance, class_): - """Make sure our callable binds like an instance method. Otherwise - __call__ doesn't get the instance argument.""" + """Bind to ``instance`` the same way a normal function descriptor does.""" if instance is None: return self else: @@ -50,9 +630,13 @@ def __call__(self, *args, **kwargs): class callback(object): - """Decorator to turn a function into a Callback object. Presently - optional, and only required if the callback needs to have a non-default - priority set""" + """Decorator that turns a function into a :class:`Callback`. + + The decorator is optional. A plain method can still be returned from + ``get_callbacks()`` if it already behaves like a callback. Use the + decorator when you want to attach a non-default priority or make the + callback object explicit in the class body. + """ # Instantiate the decorator: def __init__(self, priority=DEFAULT_PRIORITY): self.priority = priority @@ -63,58 +647,149 @@ def __call__(self, func): class BasePlugin(object): - """Default no-op implementation of the existing plugin interface.""" + """Reference implementation of the plugin interface. + + Subclass this when you want a concrete starting point, but it is not + required. The manager only relies on the methods defined here; a plugin + can also satisfy the contract by duck typing. + + ``initial_settings`` contains the saved configuration for the plugin. + """ def __init__(self, initial_settings): + """Store the plugin's saved settings. + + Applications pass the dictionary returned by the previous + ``get_save_data()`` call, or an empty dictionary for a first run. + Subclasses commonly keep this dictionary and use it during + ``plugin_setup_complete()`` to restore UI state or background state. + """ self.initial_settings = initial_settings self.menu = None self.notifications = {} def get_menu_class(self): + """Return a menu class for this plugin, or ``None``. + + The class is constructed as ``MenuClass(data)`` during + ``PluginManager.setup_plugins()``. Its instance should provide + ``get_menu_items()``, returning the nested menu dictionary consumed by + :class:`MenuBuilder`. + """ return None def get_notification_classes(self): + """Return notification classes contributed by this plugin. + + The application-specific notification manager constructs the + notifications. Instances are later passed back to + ``set_notification_instances()`` in a dictionary keyed by notification + class. + """ return [] def get_setting_classes(self): + """Return settings-page classes contributed by this plugin. + + The application owns the settings UI and storage. The manager only + collects these classes and returns them to the application from + ``setup_plugins()``. + """ return [] def get_callbacks(self): + """Return callbacks keyed by event name, or ``None``. + + A typical return value is ``{'shot_complete': self.on_shot_complete}``. + Values may be plain callables or :class:`Callback` instances created + with the :class:`callback` decorator. The manager sorts callbacks for a + given event by their ``priority`` attribute. + """ return None - def get_tab_classes(self): - return {} + def get_ui_contributions(self): + """Return app-context UI contributions. - def tabs_created(self, tabs): - pass + Return an iterable of dictionaries. Each contribution has a + ``context`` key naming an application-registered context. The + application owns what that context means: a known tab area, a deferred + MDI workspace, a dialog registry, a fixed frame, or another + app-specific host. + """ + return [] + + def get_menu_contributions(self): + """Return shared menu contributions. + + Return an iterable of dictionaries. This is the app-neutral menu path. + Existing BLACS-style plugins can continue using ``get_menu_class()`` + and :class:`MenuBuilder`; new apps can register a ``menus`` context and + route these dictionaries through :class:`MenuContext`. + """ + return [] def set_menu_instance(self, menu): + """Receive the menu instance constructed for this plugin.""" self.menu = menu def set_notification_instances(self, notifications): + """Receive notification instances constructed for this plugin.""" self.notifications = notifications def plugin_setup_complete(self, data=None): + """Run after the application has finished plugin setup. + + ``data`` is an application-defined dictionary. For BLACS it contains + references such as the main UI, experiment config, plugin mapping, and + settings object. Older plugins may define this method without a + ``data`` argument; ``PluginManager.setup_complete()`` keeps that + compatibility behavior. + """ pass def get_save_data(self): + """Return serializable plugin state for the next application start.""" return {} def close(self): + """Clean up resources owned by the plugin during application shutdown.""" pass class MenuBuilder(object): - """Build menus from the nested dictionary format used by BLACS plugins.""" + """Build menus from the nested dictionary format used by plugins. + + Each menu description is a dictionary. Supported keys are: + + * ``name``: text for a menu or action + * ``menu_items``: child menu descriptions, making a submenu + * ``icon``: icon name passed to ``icon_factory`` for actions + * ``action``: callable connected to the action's ``triggered`` signal + * ``separator``: a truthy sentinel that inserts a separator + + ``icon_factory`` is optional and should return a Qt icon object or other + value accepted by ``addAction``. Supplying it keeps this module free of a + direct Qt dependency. + """ def __init__(self, icon_factory=None): + """Create a menu builder. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` before adding an icon-bearing menu + action. Applications using Qt typically pass ``QIcon``. + """ self.icon_factory = icon_factory def create_menu(self, parent, menu_parameters): + """Recursively build menus and actions from a nested menu dictionary.""" if 'name' in menu_parameters: if 'menu_items' in menu_parameters: child = parent.addMenu(menu_parameters['name']) for child_menu_params in menu_parameters['menu_items']: self.create_menu(child, child_menu_params) else: + # ``icon_factory`` stays outside this module so labscript-utils + # does not need to import a Qt binding directly. if 'icon' in menu_parameters and self.icon_factory is not None: child = parent.addAction( self.icon_factory(menu_parameters['icon']), @@ -130,8 +805,160 @@ def create_menu(self, parent, menu_parameters): parent.addSeparator() +class MenuContext(object): + """Collect and render contribution-based menu actions. + + Applications register stable top-level menu locations, such as ``file`` or + ``tools``. Plugins contribute action dictionaries that request a location, + optional submenu path, separator group, order, action text, and callback. + Rendering is deferred until all plugins have contributed so ordering and + group separators can be computed across plugins. + """ + def __init__(self, icon_factory=None, logger=None): + """Create a menu context. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` for icon-bearing actions. + logger (logging.Logger, optional): Logger used for skipped or + malformed menu contributions. + """ + self.icon_factory = icon_factory + self.logger = logger or logging.getLogger(__name__) + self.locations = {} + self.contributions = [] + + def register_location(self, name, menu): + """Register an application-owned menu object as a stable location.""" + self.locations[name] = menu + + def add(self, plugin_name, contribution, data): + """Collect one menu contribution for later rendering. + + ``data`` is accepted for the generic context contract. Menu + contributions already contain their callbacks, so this context does not + need to inspect app data directly. + """ + self.contributions.append((plugin_name, contribution)) + + def render(self): + """Render all collected menu contributions into registered locations.""" + grouped = {} + group_orders = {} + + for plugin_name, contribution in self.contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a dictionary. " + "Skipping." % plugin_name + ) + continue + if 'location' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing location. " + "Skipping." % plugin_name + ) + continue + if 'name' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing name. " + "Skipping." % plugin_name + ) + continue + + location = contribution['location'] + if location not in self.locations: + self.logger.error( + "Menu contribution from plugin '%s' requested unknown " + "location '%s'. Skipping." % (plugin_name, location) + ) + continue + + path = contribution.get('path', ()) + if isinstance(path, str): + path = (path,) + else: + path = tuple(path) + + key = (location, path) + group = contribution.get('group', None) + if key not in group_orders: + group_orders[key] = {} + if group not in group_orders[key]: + group_orders[key][group] = len(group_orders[key]) + + grouped.setdefault(key, []).append((plugin_name, contribution)) + + menus = {} + for name, menu in self.locations.items(): + menus[(name, ())] = menu + + for key in sorted(grouped): + location, path = key + parent_path = () + for submenu_name in path: + submenu_path = parent_path + (submenu_name,) + submenu_key = (location, submenu_path) + if submenu_key not in menus: + menus[submenu_key] = menus[(location, parent_path)].addMenu( + submenu_name + ) + parent_path = submenu_path + menu = menus[(location, path)] + + contributions = sorted( + grouped[key], + key=lambda item: ( + group_orders[key][item[1].get('group', None)], + item[1].get('order', DEFAULT_PRIORITY), + item[0], + item[1]['name'], + ), + ) + + previous_group = None + for index, (plugin_name, contribution) in enumerate(contributions): + group = contribution.get('group', None) + if index and group != previous_group: + menu.addSeparator() + previous_group = group + + name = contribution['name'] + icon = contribution.get('icon', None) + if icon is not None and self.icon_factory is not None: + action = menu.addAction(self.icon_factory(icon), name) + else: + action = menu.addAction(name) + + callback = contribution.get('action', None) + if callback is not None: + action.triggered.connect(callback) + + shortcut = contribution.get('shortcut', None) + if shortcut is not None and hasattr(action, 'setShortcut'): + action.setShortcut(shortcut) + + checkable = contribution.get('checkable', False) + if hasattr(action, 'setCheckable'): + action.setCheckable(checkable) + + enabled = contribution.get('enabled', True) + if hasattr(action, 'setEnabled'): + action.setEnabled(enabled) + + class PluginManager(object): - """Manage discovery and setup of plugins using the existing plugin API.""" + """Manage plugin discovery, lifecycle, and callback lookup. + + The manager keeps two internal mappings: + + * ``modules``: imported plugin modules keyed by plugin name + * ``plugins``: instantiated plugin objects keyed by plugin name + + An application normally creates one manager per plugin directory and uses + it to discover modules, instantiate plugin objects, build UI pieces, and + close plugins on shutdown. + """ def __init__( self, plugin_package, @@ -141,6 +968,21 @@ def __init__( default_plugins=(), logger=None, ): + """Create a plugin manager. + + Args: + plugin_package (str): Import package containing plugin subpackages, + for example ``'myapp.plugins'``. + plugins_dir (str): Filesystem path scanned for plugin directories. + config: Config object with ``has_section()``, ``add_section()``, + ``items()``, ``set()``, and ``getboolean()`` methods. + config_section (str): Section containing plugin enable/disable + options. + default_plugins (iterable): Plugin names enabled by default when + first discovered. + logger (logging.Logger, optional): Logger used for plugin import, + setup, callback, and shutdown errors. + """ self.plugin_package = plugin_package self.plugins_dir = plugins_dir self.config = config @@ -149,8 +991,10 @@ def __init__( self.logger = logger or logging.getLogger(__name__) self.modules = {} self.plugins = {} + self.contexts = {} def discover_modules(self): + """Scan the plugin directory, update config defaults, and import enabled modules.""" if not self.config.has_section(self.config_section): self.config.add_section(self.config_section) @@ -164,7 +1008,7 @@ def discover_modules(self): if not os.path.isdir(module_path) or module_name == '__pycache__': continue - # If this is a new plugin, add it to the config. + # Keep the config in sync with what is present on disk. if module_name not in configured_plugins: self.config.set( self.config_section, @@ -190,6 +1034,12 @@ def discover_modules(self): return modules def instantiate_plugins(self, plugin_settings=None): + """Create ``Plugin`` instances for each imported module. + + ``plugin_settings`` should map plugin names to the saved configuration + dictionary for that plugin. Missing entries fall back to an empty + dictionary. Instantiation failures are logged and skipped. + """ if plugin_settings is None: plugin_settings = {} @@ -208,39 +1058,15 @@ def instantiate_plugins(self, plugin_settings=None): self.plugins = plugins return plugins - def create_tabs(self, tab_widget, settings_dict, tablist, settings, tab_data): - for module_name, plugin in self.plugins.items(): - try: - if hasattr(plugin, 'get_tab_classes'): - tab_dict = {} - - for tab_name, TabClass in plugin.get_tab_classes().items(): - settings_key = "{}: {}".format(module_name, tab_name) - settings_dict.setdefault(settings_key, {"tab_name": tab_name}) - settings_dict[settings_key]["front_panel_settings"] = ( - settings[settings_key] if settings_key in settings else {} - ) - settings_dict[settings_key]["saved_data"] = ( - tab_data[settings_key]['data'] - if settings_key in tab_data else {} - ) - - tablist[settings_key] = TabClass( - tab_widget, - settings_dict[settings_key], - ) - tab_dict[tab_name] = tablist[settings_key] - - if hasattr(plugin, 'tabs_created'): - plugin.tabs_created(tab_dict) - - except Exception: - self.logger.exception( - "Could not instantiate tab for plugin '%s'. Skipping" - % module_name - ) - def setup_plugins(self, data, notifications, menu_builder, menubar): + """Collect plugin UI and startup contributions. + + This is typically called after the application has created its legacy + menu and notification hosts and before it finishes wiring its settings + UI and callback registrations. The method returns a tuple of + ``(settings_pages, settings_callbacks)`` for the application to + consume. + """ settings_pages = [] settings_callbacks = [] @@ -280,7 +1106,110 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): return settings_pages, settings_callbacks + def register_context(self, name, context): + """Register an application-owned plugin contribution context.""" + self.contexts[name] = context + + def _get_contributions(self, plugin, module_name, method_name, label): + """Return a plugin contribution iterable, logging malformed results.""" + if not hasattr(plugin, method_name): + return [] + + try: + contributions = getattr(plugin, method_name)() + except Exception: + self.logger.exception( + "Error getting %s contributions from plugin '%s'. Skipping." + % (label, module_name) + ) + return [] + + if contributions is None: + return [] + + if ( + isinstance(contributions, (dict, str, bytes)) + or not hasattr(contributions, '__iter__') + ): + self.logger.error( + "Plugin '%s' %s contributions must be an iterable of " + "dictionaries. Skipping." % (module_name, label) + ) + return [] + + return contributions + + def setup_contexts(self, data): + """Route plugin-declared UI and menu contributions to app contexts.""" + for module_name, plugin in self.plugins.items(): + ui_contributions = self._get_contributions( + plugin, + module_name, + 'get_ui_contributions', + 'UI', + ) + + for contribution in ui_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "UI contribution from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + if 'context' not in contribution: + self.logger.error( + "UI contribution from plugin '%s' missing context. " + "Skipping." % module_name + ) + continue + + context_name = contribution['context'] + if context_name not in self.contexts: + self.logger.error( + "UI contribution from plugin '%s' requested unknown " + "context '%s'. Skipping." % (module_name, context_name) + ) + continue + + try: + self.contexts[context_name].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding UI contribution from plugin '%s' to " + "context '%s'. Skipping." % (module_name, context_name) + ) + + menu_contributions = self._get_contributions( + plugin, + module_name, + 'get_menu_contributions', + 'menu', + ) + + if menu_contributions and 'menus' not in self.contexts: + self.logger.error( + "Plugin '%s' provided menu contributions, but no 'menus' " + "context is registered. Skipping." % module_name + ) + continue + + for contribution in menu_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a " + "dictionary. Skipping." % module_name + ) + continue + try: + self.contexts['menus'].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding menu contribution from plugin '%s'. " + "Skipping." % module_name + ) + def setup_complete(self, data): + """Notify plugins that the application has finished startup.""" for module_name, plugin in self.plugins.items(): try: plugin.plugin_setup_complete(data) @@ -305,7 +1234,7 @@ def setup_complete(self, data): ) def get_callbacks(self, name): - """Return all callbacks for a particular name, in priority order.""" + """Return all callbacks registered for ``name``, sorted by priority.""" callbacks = [] for plugin in self.plugins.values(): @@ -323,6 +1252,7 @@ def get_callbacks(self, name): return callbacks def close_plugins(self): + """Call ``close()`` on every plugin during application shutdown.""" for module_name, plugin in self.plugins.items(): try: plugin.close() diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 026057b1..a6188a08 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -10,6 +10,7 @@ BasePlugin, Callback, MenuBuilder, + MenuContext, PluginManager, callback, ) @@ -56,7 +57,8 @@ def test_base_plugin_defaults_are_no_ops(): assert plugin.get_notification_classes() == [] assert plugin.get_setting_classes() == [] assert plugin.get_callbacks() is None - assert plugin.get_tab_classes() == {} + assert plugin.get_ui_contributions() == [] + assert plugin.get_menu_contributions() == [] assert plugin.get_save_data() == {} @@ -172,6 +174,18 @@ def __init__(self, name, icon=None): self.name = name self.icon = icon self.triggered = FakeSignal() + self.shortcut = None + self.checkable = None + self.enabled = None + + def setShortcut(self, shortcut): + self.shortcut = shortcut + + def setCheckable(self, checkable): + self.checkable = checkable + + def setEnabled(self, enabled): + self.enabled = enabled class FakeMenu(object): @@ -226,3 +240,203 @@ def action(): assert icon.name == 'Icon' assert icon.icon == 'icon:resource' assert icon.triggered.callbacks == [action] + + +class FakeContext(object): + def __init__(self): + self.items = [] + + def add(self, plugin_name, contribution, data): + self.items.append((plugin_name, contribution, data)) + + +class BrokenContext(object): + def add(self, plugin_name, contribution, data): + raise RuntimeError('broken context') + + +def test_setup_contexts_routes_ui_and_menu_contributions(): + class Plugin(object): + def get_ui_contributions(self): + return [{'context': 'mdi', 'key': 'browser'}] + + def get_menu_contributions(self): + return [{'location': 'file', 'name': 'Open'}] + + mdi_context = FakeContext() + menu_context = FakeContext() + data = object() + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + ) + manager.plugins = {'plugin': Plugin(), 'legacy': object()} + manager.register_context('mdi', mdi_context) + manager.register_context('menus', menu_context) + + manager.setup_contexts(data) + + assert mdi_context.items == [ + ('plugin', {'context': 'mdi', 'key': 'browser'}, data), + ] + assert menu_context.items == [ + ('plugin', {'location': 'file', 'name': 'Open'}, data), + ] + + +def test_setup_contexts_logs_and_skips_missing_unknown_and_broken_contexts(caplog): + class MissingContext(object): + def get_ui_contributions(self): + return [{'key': 'browser'}] + + class UnknownContext(object): + def get_ui_contributions(self): + return [{'context': 'unknown', 'key': 'browser'}] + + class BrokenGetter(object): + def get_ui_contributions(self): + raise RuntimeError('broken getter') + + class MalformedGetter(object): + def get_ui_contributions(self): + return {'context': 'mdi'} + + class MalformedItem(object): + def get_ui_contributions(self): + return ['not a dictionary'] + + class BrokenAdd(object): + def get_ui_contributions(self): + return [{'context': 'broken', 'key': 'browser'}] + + class MenuPlugin(object): + def get_menu_contributions(self): + return [{'location': 'file', 'name': 'Open'}] + + logger = logging.getLogger('test.plugins') + manager = PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + manager.plugins = { + 'missing': MissingContext(), + 'unknown': UnknownContext(), + 'broken_getter': BrokenGetter(), + 'malformed_getter': MalformedGetter(), + 'malformed_item': MalformedItem(), + 'broken_add': BrokenAdd(), + 'menu': MenuPlugin(), + } + manager.register_context('broken', BrokenContext()) + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + manager.setup_contexts(object()) + + log_text = caplog.text.lower() + assert 'missing' in log_text + assert 'unknown' in log_text + assert 'broken_getter' in log_text + assert 'malformed_getter' in log_text + assert 'malformed_item' in log_text + assert 'broken_add' in log_text + assert 'menus' in log_text + + +def test_menu_context_renders_locations_paths_groups_order_and_action_options(caplog): + file_menu = FakeMenu('file') + tools_menu = FakeMenu('tools') + + def open_action(): + pass + + def heal_action(): + pass + + def save_action(): + pass + + logger = logging.getLogger('test.plugins') + context = MenuContext(icon_factory=lambda name: 'icon:' + name, logger=logger) + context.register_location('file', file_menu) + context.register_location('tools', tools_menu) + context.add( + 'plugin_b', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'open', + 'order': 20, + 'name': 'Heal', + 'action': heal_action, + }, + {}, + ) + context.add( + 'plugin_a', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'open', + 'order': 10, + 'name': 'Open', + 'icon': 'folder', + 'shortcut': 'Ctrl+O', + 'checkable': True, + 'enabled': False, + 'action': open_action, + }, + {}, + ) + context.add( + 'plugin_a', + { + 'location': 'file', + 'path': ('Project',), + 'group': 'save', + 'order': 5, + 'name': 'Save', + 'action': save_action, + }, + {}, + ) + context.add('plugin_c', {'location': 'tools', 'name': 'Tool'}, {}) + context.add('plugin_malformed', 'not a dictionary', {}) + context.add('plugin_skip', {'location': 'missing', 'name': 'Skip'}, {}) + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + context.render() + + project_menu = file_menu.items[0][1] + open_item = project_menu.items[0][1] + heal_item = project_menu.items[1][1] + save_item = project_menu.items[3][1] + + assert file_menu.items[0][0] == 'menu' + assert project_menu.name == 'Project' + assert [item[0] for item in project_menu.items] == [ + 'action', + 'action', + 'separator', + 'action', + ] + assert [open_item.name, heal_item.name, save_item.name] == [ + 'Open', + 'Heal', + 'Save', + ] + assert open_item.icon == 'icon:folder' + assert open_item.shortcut == 'Ctrl+O' + assert open_item.checkable is True + assert open_item.enabled is False + assert open_item.triggered.callbacks == [open_action] + assert heal_item.triggered.callbacks == [heal_action] + assert save_item.checkable is False + assert save_item.enabled is True + assert tools_menu.items[0][1].name == 'Tool' + assert 'missing' in caplog.text.lower() + assert 'plugin_malformed' in caplog.text From dd8664f032c49782fb2e2957676312558e9b9731 Mon Sep 17 00:00:00 2001 From: spielman Date: Sun, 26 Apr 2026 12:10:24 -0400 Subject: [PATCH 32/42] Final revision. --- labscript_utils/plugins.py | 164 ++++++++++++++++++++++---- tests/test_plugins.py | 231 +++++++++++++++++++++++++++++++++++-- 2 files changed, 361 insertions(+), 34 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 982b32e2..1033a20a 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -232,7 +232,7 @@ def get_notification_classes(self): def get_setting_classes(self): return [SettingsPage] - def get_callbacks(self): + def get_event_handlers(self): return { 'settings_changed': self.on_settings_changed, 'shot_complete': self.on_shot_complete, @@ -528,7 +528,7 @@ class Notification(object): numbers run first when the application asks ``PluginManager.get_callbacks()``:: class SlowPlugin(BasePlugin): - def get_callbacks(self): + def get_event_handlers(self): return { 'shot_complete': self.on_slow, } @@ -538,7 +538,7 @@ def on_slow(self): pass class FastPlugin(BasePlugin): - def get_callbacks(self): + def get_event_handlers(self): return { 'shot_complete': self.on_fast, } @@ -588,6 +588,8 @@ def close(self): import importlib import logging import os +import warnings +from collections.abc import Mapping from types import MethodType @@ -604,6 +606,14 @@ def close(self): ] +def _log_once(logger, seen, level, key, message): + """Log ``message`` once for each hashable ``key``.""" + if key in seen: + return + seen.add(key) + logger.log(level, message) + + class Callback(object): """Wrap a callable with priority metadata and method-style binding. @@ -670,10 +680,13 @@ def __init__(self, initial_settings): def get_menu_class(self): """Return a menu class for this plugin, or ``None``. + Deprecated compatibility hook for BLACS-style nested menus. + The class is constructed as ``MenuClass(data)`` during ``PluginManager.setup_plugins()``. Its instance should provide ``get_menu_items()``, returning the nested menu dictionary consumed by - :class:`MenuBuilder`. + :class:`MenuBuilder`. New shared plugin code should prefer + :meth:`get_menu_contributions` instead. """ return None @@ -696,15 +709,30 @@ def get_setting_classes(self): """ return [] + def get_event_handlers(self): + """Return a mapping of event names to handlers, or ``None``. + + This is the preferred shared event hook surface. Return a mapping such + as ``{'shot_complete': self.on_shot_complete}``. Event names are + application-defined strings; this shared layer only collects, + normalizes, and orders handlers. Values may be plain callables or + :class:`Callback` instances created with the :class:`callback` + decorator. The manager sorts handlers for a given event by their + ``priority`` attribute. + """ + return None + def get_callbacks(self): """Return callbacks keyed by event name, or ``None``. - A typical return value is ``{'shot_complete': self.on_shot_complete}``. - Values may be plain callables or :class:`Callback` instances created - with the :class:`callback` decorator. The manager sorts callbacks for a - given event by their ``priority`` attribute. + Deprecated compatibility alias for :meth:`get_event_handlers`. + + Existing BLACS-style plugins may still override this method. New + shared plugin code should implement :meth:`get_event_handlers` + instead. ``BasePlugin`` routes this legacy name to the modern hook so + the preferred surface is the real implementation path. """ - return None + return self.get_event_handlers() def get_ui_contributions(self): """Return app-context UI contributions. @@ -713,7 +741,7 @@ def get_ui_contributions(self): ``context`` key naming an application-registered context. The application owns what that context means: a known tab area, a deferred MDI workspace, a dialog registry, a fixed frame, or another - app-specific host. + app-specific host. The remaining keys are application-defined. """ return [] @@ -723,7 +751,10 @@ def get_menu_contributions(self): Return an iterable of dictionaries. This is the app-neutral menu path. Existing BLACS-style plugins can continue using ``get_menu_class()`` and :class:`MenuBuilder`; new apps can register a ``menus`` context and - route these dictionaries through :class:`MenuContext`. + route these dictionaries through :class:`MenuContext`. Apart from + stable menu keys such as ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action``, the concrete menu objects remain + application-owned. """ return [] @@ -738,7 +769,9 @@ def set_notification_instances(self, notifications): def plugin_setup_complete(self, data=None): """Run after the application has finished plugin setup. - ``data`` is an application-defined dictionary. For BLACS it contains + ``data`` is an application-defined dictionary. Plugins commonly store + references to services or state they will need later when handling + events, building menus, or opening UI. For BLACS it contains references such as the main UI, experiment config, plugin mapping, and settings object. Older plugins may define this method without a ``data`` argument; ``PluginManager.setup_complete()`` keeps that @@ -747,11 +780,20 @@ def plugin_setup_complete(self, data=None): pass def get_save_data(self): - """Return serializable plugin state for the next application start.""" + """Return serializable plugin state for the next application start. + + The shape of this data is application-owned. Return plain data + structures that the application can persist and pass back as + ``initial_settings`` during the next start. + """ return {} def close(self): - """Clean up resources owned by the plugin during application shutdown.""" + """Clean up resources owned by the plugin during application shutdown. + + Applications call this during shutdown. Stop timers, workers, + listeners, or other plugin-owned resources here. + """ pass @@ -992,6 +1034,7 @@ def __init__( self.modules = {} self.plugins = {} self.contexts = {} + self._logged_plugin_warnings = set() def discover_modules(self): """Scan the plugin directory, update config defaults, and import enabled modules.""" @@ -1076,10 +1119,19 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): settings_pages.extend(plugin.get_setting_classes()) # Setup menu. - if plugin.get_menu_class(): + menu_class = plugin.get_menu_class() + if menu_class: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_menu_class'), + "Plugin '%s' uses deprecated get_menu_class(); " + "prefer get_menu_contributions() instead." % module_name + ) # Must store a reference or else the methods called when # the menu actions are triggered will be garbage collected. - menu = plugin.get_menu_class()(data) + menu = menu_class(data) menu_builder.create_menu(menubar, menu.get_menu_items()) plugin.set_menu_instance(menu) @@ -1093,10 +1145,10 @@ def setup_plugins(self, data, notifications, menu_builder, menubar): plugin.set_notification_instances(plugin_notifications) # Register callbacks. - callbacks = plugin.get_callbacks() + callbacks = self._get_plugin_event_handlers(module_name, plugin) # Save the settings_changed callback in a separate list for # setting up later. - if isinstance(callbacks, dict) and 'settings_changed' in callbacks: + if callbacks and 'settings_changed' in callbacks: settings_callbacks.append(callbacks['settings_changed']) except Exception: @@ -1233,16 +1285,68 @@ def setup_complete(self, data): % module_name ) - def get_callbacks(self, name): - """Return all callbacks registered for ``name``, sorted by priority.""" + def _defines_hook(self, plugin, method_name): + """Return whether ``plugin`` overrides ``method_name`` meaningfully.""" + plugin_method = getattr(type(plugin), method_name, None) + if plugin_method is None: + return False + + base_method = getattr(BasePlugin, method_name, None) + return plugin_method is not base_method + + def _get_plugin_event_handlers(self, module_name, plugin): + """Return normalized event handlers for one plugin.""" + has_modern = self._defines_hook(plugin, 'get_event_handlers') + has_legacy = self._defines_hook(plugin, 'get_callbacks') + + if has_modern: + if has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' defines deprecated get_callbacks() and " + "preferred get_event_handlers(); ignoring get_callbacks()." + % module_name + ) + handlers = plugin.get_event_handlers() + elif has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' uses deprecated get_callbacks(); " + "implement get_event_handlers() instead." % module_name + ) + handlers = plugin.get_callbacks() + else: + return None + + if handlers is None: + return None + if not isinstance(handlers, Mapping): + self.logger.error( + "Plugin '%s' event handlers must be a mapping. Skipping." + % module_name + ) + return None + + return handlers + + def get_event_handlers(self, name): + """Return all event handlers registered for ``name``, sorted by priority.""" callbacks = [] - for plugin in self.plugins.values(): + for module_name, plugin in self.plugins.items(): try: - plugin_callbacks = plugin.get_callbacks() - if plugin_callbacks is not None: - if name in plugin_callbacks: - callbacks.append(plugin_callbacks[name]) + plugin_callbacks = self._get_plugin_event_handlers( + module_name, + plugin, + ) + if plugin_callbacks and name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) except Exception: self.logger.exception('Error getting callbacks from %s.' % str(plugin)) @@ -1251,6 +1355,16 @@ def get_callbacks(self, name): ) return callbacks + def get_callbacks(self, name): + """Deprecated compatibility wrapper for :meth:`get_event_handlers`.""" + warnings.warn( + "PluginManager.get_callbacks() is deprecated; use " + "PluginManager.get_event_handlers() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.get_event_handlers(name) + def close_plugins(self): """Call ``close()`` on every plugin during application shutdown.""" for module_name, plugin in self.plugins.items(): diff --git a/tests/test_plugins.py b/tests/test_plugins.py index a6188a08..47fad04a 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,6 +1,7 @@ import logging import sys import uuid +import warnings from pathlib import Path from types import SimpleNamespace @@ -56,6 +57,7 @@ def test_base_plugin_defaults_are_no_ops(): assert plugin.get_menu_class() is None assert plugin.get_notification_classes() == [] assert plugin.get_setting_classes() == [] + assert plugin.get_event_handlers() is None assert plugin.get_callbacks() is None assert plugin.get_ui_contributions() == [] assert plugin.get_menu_contributions() == [] @@ -77,6 +79,24 @@ def make_plugin_package(tmp_path, modules): return package + '.plugins', plugins_dir +def make_manager(logger=None): + return PluginManager( + 'package.plugins', + 'plugins', + FakeConfig(), + 'app/plugins', + logger=logger, + ) + + +class NoNotifications(object): + def add_notification(self, notification_class): + raise AssertionError('no notifications expected') + + def get_instance(self, notification_class): + raise AssertionError('no notifications expected') + + def test_discovery_defaults_config_and_imports_only_enabled_plugins(tmp_path): plugin_package, plugins_dir = make_plugin_package( tmp_path, @@ -119,7 +139,7 @@ def __init__(self, initial_settings): assert plugins['plugin'].initial_settings == {'answer': 42} -def test_get_callbacks_sorts_by_priority_and_logs_errors(caplog): +def test_get_event_handlers_sorts_legacy_callbacks_by_priority_and_logs_errors(caplog): class Slow(object): def get_callbacks(self): return {'event': self.on_event} @@ -141,13 +161,7 @@ def get_callbacks(self): raise RuntimeError('broken') logger = logging.getLogger('test.plugins') - manager = PluginManager( - 'package.plugins', - 'plugins', - FakeConfig(), - 'app/plugins', - logger=logger, - ) + manager = make_manager(logger=logger) manager.plugins = { 'slow': Slow(), 'broken': Broken(), @@ -155,12 +169,211 @@ def get_callbacks(self): } with caplog.at_level(logging.ERROR, logger='test.plugins'): - callbacks = manager.get_callbacks('event') + callbacks = manager.get_event_handlers('event') assert [callback.priority for callback in callbacks] == [5, 20] assert 'Error getting callbacks from' in caplog.text +def test_get_event_handlers_sorts_modern_handlers_by_priority(): + class Slow(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=20) + def on_event(self): + pass + + class Fast(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + manager = make_manager() + manager.plugins = { + 'slow': Slow(), + 'fast': Fast(), + } + + callbacks = manager.get_event_handlers('event') + + assert [callback.priority for callback in callbacks] == [5, 20] + + +def test_get_callbacks_warns_and_delegates_to_get_event_handlers(): + class Plugin(object): + def get_event_handlers(self): + return {'event': self.on_event} + + @callback(priority=5) + def on_event(self): + pass + + manager = make_manager() + manager.plugins = {'plugin': Plugin()} + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', DeprecationWarning) + callbacks = manager.get_callbacks('event') + + assert [callback.priority for callback in callbacks] == [5] + assert len(caught) == 1 + assert 'PluginManager.get_callbacks() is deprecated' in str(caught[0].message) + + +def test_get_event_handlers_prefers_modern_hook_and_warns_for_legacy_one(caplog): + class ModernAndLegacy(object): + def get_event_handlers(self): + return {'event': self.new_event} + + def get_callbacks(self): + return {'event': self.old_event} + + @callback(priority=5) + def new_event(self): + pass + + @callback(priority=20) + def old_event(self): + pass + + class LegacyOnly(object): + def get_callbacks(self): + return {'event': self.old_event} + + @callback(priority=10) + def old_event(self): + pass + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = { + 'modern_and_legacy': ModernAndLegacy(), + 'legacy_only': LegacyOnly(), + } + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + first_callbacks = manager.get_event_handlers('event') + second_callbacks = manager.get_event_handlers('event') + + assert [callback.priority for callback in first_callbacks] == [5, 10] + assert [callback.priority for callback in second_callbacks] == [5, 10] + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('modern_and_legacy' in message for message in warning_messages) == 1 + assert sum('legacy_only' in message for message in warning_messages) == 1 + assert sum('get_callbacks()' in message for message in warning_messages) == 2 + + +def test_setup_plugins_prefers_modern_settings_callback_and_warns(caplog): + class Plugin(BasePlugin): + def get_event_handlers(self): + return {'settings_changed': self.new_settings_changed} + + def get_callbacks(self): + return {'settings_changed': self.old_settings_changed} + + @callback(priority=5) + def new_settings_changed(self): + pass + + @callback(priority=20) + def old_settings_changed(self): + pass + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = {'plugin': Plugin({})} + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + settings_pages, settings_callbacks = manager.setup_plugins( + data={}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + callbacks = manager.get_event_handlers('settings_changed') + + assert settings_pages == [] + assert [callback.priority for callback in settings_callbacks] == [5] + assert [callback.priority for callback in callbacks] == [5] + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('deprecated get_callbacks()' in message for message in warning_messages) == 1 + + +def test_setup_plugins_warns_when_deprecated_menu_class_is_used(caplog): + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return {'name': 'Legacy'} + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = {'plugin': Plugin({})} + + with caplog.at_level(logging.WARNING, logger='test.plugins'): + manager.setup_plugins( + data={'example': True}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + manager.setup_plugins( + data={'example': True}, + notifications=NoNotifications(), + menu_builder=MenuBuilder(), + menubar=FakeMenu(), + ) + + warning_messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING + ] + assert sum('deprecated get_menu_class()' in message for message in warning_messages) == 1 + + +def test_get_event_handlers_logs_and_skips_malformed_event_handler_mappings(caplog): + class BadModern(object): + def get_event_handlers(self): + return ['event'] + + class BadLegacy(object): + def get_callbacks(self): + return ['event'] + + logger = logging.getLogger('test.plugins') + manager = make_manager(logger=logger) + manager.plugins = { + 'bad_modern': BadModern(), + 'bad_legacy': BadLegacy(), + } + + with caplog.at_level(logging.ERROR, logger='test.plugins'): + callbacks = manager.get_event_handlers('event') + + assert callbacks == [] + assert "bad_modern" in caplog.text + assert "bad_legacy" in caplog.text + assert "event handlers must be a mapping" in caplog.text + + class FakeSignal(object): def __init__(self): self.callbacks = [] From 83f44ff3f49a111a1d65117e503afd81326f470e Mon Sep 17 00:00:00 2001 From: spielman Date: Sun, 26 Apr 2026 12:19:27 -0400 Subject: [PATCH 33/42] Add shared plugin framework to labscript-utils --- labscript_utils/plugins.py | 1377 ++++++++++++++++++++++++++++++++++++ 1 file changed, 1377 insertions(+) create mode 100644 labscript_utils/plugins.py diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py new file mode 100644 index 00000000..1033a20a --- /dev/null +++ b/labscript_utils/plugins.py @@ -0,0 +1,1377 @@ +##################################################################### +# # +# plugins.py # +# # +# Copyright 2013, Monash University # +# # +# This file is part of the labscript suite (see # +# http://labscriptsuite.org) and is licensed under the Simplified # +# BSD License. See the license.txt file in the root of the project # +# for the full license. # +# # +##################################################################### + +"""Plugin discovery, instantiation, and callback plumbing for labscript apps. + +This module is the small framework layer that labscript-suite applications use +to discover plugins, instantiate them, collect app-neutral contributions, and +ask them for callbacks at runtime. The application owns the actual settings +storage, notification manager, menu objects, UI containers, and event firing. +``PluginManager`` only calls plugin hooks and routes contributions to contexts +the application registered explicitly. + +The usual package layout is:: + + myapp/ + plugins/ + __init__.py + example/ + __init__.py # defines class Plugin + +The module name becomes the plugin name. ``PluginManager`` looks for a module +attribute named ``Plugin`` and passes the saved per-plugin settings dictionary +to ``Plugin(initial_settings)``. Existing plugins do not have to inherit +``BasePlugin``; they only need to provide the same methods by duck typing. + +Configuration behavior is intentionally conservative: + +* Missing plugin entries are added to the config when discovered. +* Names listed in ``default_plugins`` are written as enabled by default. +* Only enabled modules are imported. +* Import or instantiation failures are logged and skipped so one broken plugin + does not stop the application from starting. + +``Callback`` and ``callback`` wrap event handlers with priority metadata. +Lower priority numbers run first. ``Callback`` behaves like a descriptor, so +decorated methods still bind to plugin instances normally when accessed through +an instance. + +Two menu paths are supported, intentionally with different contracts: + +* ``MenuBuilder`` is the legacy nested-dictionary renderer used by existing + BLACS-style plugins. It consumes dictionaries with ``name``, ``menu_items``, + ``icon``, ``action``, and ``separator`` keys and renders them immediately + into application menu objects. +* ``MenuContext`` is the shared contribution path for future applications. It + collects actions with explicit ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action`` metadata, then renders all plugins together after + contribution routing is complete. + +``labscript_utils.plugins`` does not know whether a plugin UI belongs in a +tab, MDI subwindow, dock widget, dialog, fixed frame, or plugin-owned modal +container. Plugins declare a named ``context`` and applications register +objects that implement ``add(plugin_name, contribution, data)``. That is the +full shared UI contract. + +Minimal application integration example +--------------------------------------- + +The application owns plugin settings storage, concrete UI objects, and context +implementations. A compact integration with legacy settings/notifications plus +new contribution routing looks like this:: + + from labscript_utils.plugins import MenuBuilder, MenuContext, PluginManager + + + class App(object): + def __init__(self, config, logger): + self.config = config + self.logger = logger + self.plugin_manager = PluginManager( + plugin_package='myapp.plugins', + plugins_dir='/path/to/myapp/plugins', + config=self.config, + config_section='myapp/plugins', + default_plugins=('example',), + logger=self.logger, + ) + + self.plugin_manager.discover_modules() + + plugin_settings = self.load_plugin_settings() + self.plugin_manager.instantiate_plugins(plugin_settings) + + menu_builder = MenuBuilder(icon_factory=self.icon_factory) + settings_pages, settings_callbacks = self.plugin_manager.setup_plugins( + data=self.app_data, + notifications=self.notification_manager, + menu_builder=menu_builder, + menubar=self.menubar, + ) + + menus = MenuContext(icon_factory=self.icon_factory, logger=self.logger) + menus.register_location('file', self.file_menu) + menus.register_location('window', self.window_menu) + + self.plugin_manager.register_context('menus', menus) + self.plugin_manager.register_context('mdi', MDIContext(self.mdi_area)) + self.plugin_manager.register_context( + 'dialogs', + DialogContext(parent=self.main_window), + ) + self.plugin_manager.setup_contexts(self.app_data) + menus.render() + + self.settings_widget = self.build_settings_ui(settings_pages) + for callback in settings_callbacks: + self.register_settings_callback(callback) + + self.plugin_manager.setup_complete(self.app_data) + + def load_plugin_settings(self): + # App-owned storage. Return a mapping from plugin name to dict. + return { + 'example': {'enabled': True}, + } + + def build_settings_ui(self, settings_pages): + # App-owned settings UI. Each item in settings_pages is a plugin + # contributed settings page class. + return settings_pages + + def register_settings_callback(self, callback): + # App-owned settings wiring. + self.settings_changed_callbacks.append(callback) + + def fire_event(self, name): + # App-owned event firing. PluginManager only returns callbacks. + for callback in self.plugin_manager.get_callbacks(name): + callback() + + def close_plugins(self): + self.plugin_manager.close_plugins() + + +Minimal plugin package example +------------------------------ + +This example shows the shared plugin surface. The plugin contributes a legacy +menu, a contribution-based menu action, a settings page, a notification, an MDI +window request, callbacks, ``setup_complete()`` handling, save data, and +shutdown cleanup. Methods can return empty lists or dictionaries when a feature +is not used:: + + from labscript_utils.plugins import BasePlugin, callback + + + class SettingsPage(object): + pass + + + class Notification(object): + pass + + + class DataBrowser(object): + def __init__(self, parent, services): + self.parent = parent + self.services = services + + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Legacy Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Legacy Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.saved_state = initial_settings + self.app_data = None + + def get_menu_class(self): + return Menu + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'path': ('Examples',), + 'group': 'open', + 'order': 20, + 'name': 'Open Data Browser', + 'shortcut': 'Ctrl+D', + 'icon': 'folder-open', + 'action': self.open_data_browser, + }, + ] + + def get_ui_contributions(self): + return [ + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + ] + + def get_notification_classes(self): + return [Notification] + + def get_setting_classes(self): + return [SettingsPage] + + def get_event_handlers(self): + return { + 'settings_changed': self.on_settings_changed, + 'shot_complete': self.on_shot_complete, + } + + def plugin_setup_complete(self, data): + self.app_data = data + + def get_save_data(self): + return {'saved_state': self.saved_state} + + def close(self): + pass + + def open_data_browser(self): + pass + + @callback(priority=5) + def on_settings_changed(self): + pass + + @callback(priority=20) + def on_shot_complete(self): + pass + + +Feature excerpts and expected UI +-------------------------------- + +BasePlugin subclass with saved ``initial_settings`` and ``get_save_data()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The constructor receives the per-plugin saved settings dictionary. Store it if +you need to restore state later. ``get_save_data()`` should return the +serializable data you want written back for the next application start:: + + class Plugin(BasePlugin): + def __init__(self, initial_settings): + super(Plugin, self).__init__(initial_settings) + self.initial_settings = initial_settings + + def get_save_data(self): + return { + 'window_geometry': self.window_geometry, + 'last_path': self.last_path, + } + +Settings page contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_setting_classes()`` returns the page classes the application should add +to its settings UI. The application owns the widget construction and storage:: + + class Plugin(BasePlugin): + def get_setting_classes(self): + return [SettingsPage] + + class SettingsPage(object): + pass + +Expected UI result: the application shows one settings page for each returned +class, using the app's own settings shell and persistence rules. + +Legacy menu skeleton with MenuBuilder +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_class()`` returns a class whose instance exposes +``get_menu_items()``. The nested dictionaries are translated into the app's +menu objects by ``MenuBuilder``. It expects menus and actions to support +``addMenu()``, ``addAction()``, and ``addSeparator()``:: + + class Plugin(BasePlugin): + def get_menu_class(self): + return Menu + + class Menu(object): + def __init__(self, data): + self.data = data + + def get_menu_items(self): + return { + 'name': 'Example', + 'menu_items': [ + {'name': 'Open', 'action': self.open_action}, + {'separator': True}, + { + 'name': 'Settings', + 'icon': 'gear', + 'action': self.settings_action, + }, + ], + } + + def open_action(self): + pass + + def settings_action(self): + pass + +Expected UI result: the app user sees a top-level menu labeled ``Example``, +with an ``Open`` action, a separator, and a ``Settings`` action. If the app +supplies ``icon_factory``, the ``Settings`` action can also show an icon. + +Contribution menu skeleton with MenuContext +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_menu_contributions()`` returns dictionaries routed to the application +``menus`` context. ``MenuContext`` understands these keys: + +* ``location``: app-registered top-level menu id, such as ``file`` or + ``window`` +* ``path``: optional submenu path below that location; defaults to ``()`` +* ``group``: separator group within the same location/path; defaults to + ``None`` +* ``order``: numeric order within the group; defaults to ``DEFAULT_PRIORITY`` +* ``name``: action text +* ``action``: callable connected to the action's ``triggered`` signal +* ``shortcut``, ``icon``, ``checkable``, and ``enabled``: optional action + properties applied when the concrete action object supports them + +File-dialog-style plugin example:: + + class FileDialogsPlugin(BasePlugin): + def __init__(self, initial_settings): + super(FileDialogsPlugin, self).__init__(initial_settings) + self.services = None + + def plugin_setup_complete(self, data): + self.services = data['project_services'] + + def get_menu_contributions(self): + return [ + { + 'location': 'file', + 'group': 'project', + 'order': 10, + 'name': 'New Project', + 'shortcut': 'Ctrl+N', + 'action': self.new_project, + }, + { + 'location': 'file', + 'group': 'project', + 'order': 20, + 'name': 'Open Project...', + 'shortcut': 'Ctrl+O', + 'action': self.open_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 10, + 'name': 'Save', + 'shortcut': 'Ctrl+S', + 'action': self.save_project, + }, + { + 'location': 'file', + 'group': 'save', + 'order': 20, + 'name': 'Save As...', + 'shortcut': 'Ctrl+Shift+S', + 'action': self.save_project_as, + }, + { + 'location': 'file', + 'group': 'application', + 'order': 100, + 'name': 'Quit', + 'shortcut': 'Ctrl+Q', + 'action': self.quit_application, + }, + ] + + def new_project(self): + self.services.new_project_dialog.open() + + def open_project(self): + self.services.load_project_dialog.open() + + def save_project(self): + self.services.save_project_command.execute() + + def save_project_as(self): + self.services.save_as_project_dialog.open() + + def quit_application(self): + self.services.quit_command.execute() + +Expected UI result: the application renders the actions in its registered +``file`` menu, creates separators between the ``project``, ``save``, and +``application`` groups, and applies shortcuts without the shared framework +knowing what a project is. + +App-neutral UI homing with contexts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_ui_contributions()`` returns dictionaries with a required ``context``. +The rest of the dictionary is application-defined. This is enough to support +known containers, deferred containers, and plugin-provided containers without +adding type-specific methods such as ``create_tabs()`` or +``create_mdi_windows()`` to ``PluginManager``:: + + class KnownContainerContext(object): + def __init__(self, tab_widget): + self.tab_widget = tab_widget + + def add(self, plugin_name, contribution, data): + widget = contribution['factory']( + self.tab_widget, + data['settings'][contribution['key']], + ) + self.tab_widget.addTab(widget, contribution['title']) + + + class DeferredMDIContext(object): + def __init__(self, mdi_area): + self.mdi_area = mdi_area + self.openers = {} + + def add(self, plugin_name, contribution, data): + key = contribution['key'] + + def open_window(): + widget = contribution['factory'](parent=self.mdi_area, data=data) + self.mdi_area.addSubWindow(widget) + widget.show() + return widget + + self.openers[key] = open_window + + + class DialogContext(object): + def __init__(self, parent): + self.parent = parent + self.dialogs = {} + + def add(self, plugin_name, contribution, data): + self.dialogs[contribution['key']] = contribution['factory']( + parent=self.parent, + services=data['services'], + ) + + + class Plugin(BasePlugin): + def get_ui_contributions(self): + return [ + { + 'context': 'tabs', + 'key': 'overview', + 'title': 'Overview', + 'factory': OverviewWidget, + }, + { + 'context': 'mdi', + 'key': 'data_browser', + 'title': 'Data Browser', + 'factory': DataBrowser, + }, + { + 'context': 'dialogs', + 'key': 'open_project', + 'title': 'Open Project', + 'factory': OpenProjectDialog, + }, + ] + +Expected UI result: each application-owned context decides how to instantiate, +store, parent, restore, defer, activate, or display its contribution. Unknown +contexts are logged and skipped by ``PluginManager.setup_contexts()``. + +Notification contribution skeleton +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``get_notification_classes()`` returns notification classes. The application +owns the notification manager and decides how the resulting widgets are shown:: + + class Plugin(BasePlugin): + def get_notification_classes(self): + return [Notification] + + class Notification(object): + pass + +Expected UI result: the application creates whatever notification widgets or +panels it normally uses for those classes and inserts them into its own +notification area. + +Callbacks with priorities +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``@callback(priority=...)`` wraps a method in ``Callback`` metadata. Lower +numbers run first when the application asks ``PluginManager.get_callbacks()``:: + + class SlowPlugin(BasePlugin): + def get_event_handlers(self): + return { + 'shot_complete': self.on_slow, + } + + @callback(priority=20) + def on_slow(self): + pass + + class FastPlugin(BasePlugin): + def get_event_handlers(self): + return { + 'shot_complete': self.on_fast, + } + + @callback(priority=5) + def on_fast(self): + pass + +Expected runtime order: when the application fires ``shot_complete`` and calls +``plugin_manager.get_callbacks('shot_complete')``, the priority 5 callback +runs before the priority 20 callback, even if the plugins were discovered in a +different order. + +``setup_complete()`` for app data +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``setup_complete()`` is called after the application has finished startup. +Use it to access app-owned state that was not available during construction, +such as the main window, settings object, registered services, or concrete +context objects:: + + class Plugin(BasePlugin): + def plugin_setup_complete(self, data): + self.app = data['app'] + self.settings = data['settings'] + self.services = data['services'] + +The ``data`` dictionary is app-defined. Old plugins may still define +``plugin_setup_complete(self)`` without the ``data`` argument, and the manager +keeps that compatibility path. + +Shutdown cleanup with ``close()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``close()`` is the plugin shutdown hook. Use it to stop timers, threads, +listeners, or any other plugin-owned resources:: + + class Plugin(BasePlugin): + def close(self): + self.worker.stop() + self.worker.wait() + +Expected behavior: the application calls each plugin's ``close()`` method +during shutdown so the plugin can release resources before exit. +""" + +import importlib +import logging +import os +import warnings +from collections.abc import Mapping +from types import MethodType + + +DEFAULT_PRIORITY = 10 + +__all__ = [ + 'DEFAULT_PRIORITY', + 'Callback', + 'callback', + 'BasePlugin', + 'MenuBuilder', + 'MenuContext', + 'PluginManager', +] + + +def _log_once(logger, seen, level, key, message): + """Log ``message`` once for each hashable ``key``.""" + if key in seen: + return + seen.add(key) + logger.log(level, message) + + +class Callback(object): + """Wrap a callable with priority metadata and method-style binding. + + ``priority`` is used when ``PluginManager.get_callbacks()`` gathers all + callbacks for a named event. Lower numbers run first. + + The descriptor protocol is implemented so that a ``Callback`` stored as a + class attribute binds like an instance method when accessed from a plugin + instance. This keeps decorated methods usable without extra wrapper code. + """ + def __init__(self, func, priority=DEFAULT_PRIORITY): + self.priority = priority + self.func = func + + def __get__(self, instance, class_): + """Bind to ``instance`` the same way a normal function descriptor does.""" + if instance is None: + return self + else: + return MethodType(self, instance) + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + +class callback(object): + """Decorator that turns a function into a :class:`Callback`. + + The decorator is optional. A plain method can still be returned from + ``get_callbacks()`` if it already behaves like a callback. Use the + decorator when you want to attach a non-default priority or make the + callback object explicit in the class body. + """ + # Instantiate the decorator: + def __init__(self, priority=DEFAULT_PRIORITY): + self.priority = priority + + # Call the decorator + def __call__(self, func): + return Callback(func, self.priority) + + +class BasePlugin(object): + """Reference implementation of the plugin interface. + + Subclass this when you want a concrete starting point, but it is not + required. The manager only relies on the methods defined here; a plugin + can also satisfy the contract by duck typing. + + ``initial_settings`` contains the saved configuration for the plugin. + """ + def __init__(self, initial_settings): + """Store the plugin's saved settings. + + Applications pass the dictionary returned by the previous + ``get_save_data()`` call, or an empty dictionary for a first run. + Subclasses commonly keep this dictionary and use it during + ``plugin_setup_complete()`` to restore UI state or background state. + """ + self.initial_settings = initial_settings + self.menu = None + self.notifications = {} + + def get_menu_class(self): + """Return a menu class for this plugin, or ``None``. + + Deprecated compatibility hook for BLACS-style nested menus. + + The class is constructed as ``MenuClass(data)`` during + ``PluginManager.setup_plugins()``. Its instance should provide + ``get_menu_items()``, returning the nested menu dictionary consumed by + :class:`MenuBuilder`. New shared plugin code should prefer + :meth:`get_menu_contributions` instead. + """ + return None + + def get_notification_classes(self): + """Return notification classes contributed by this plugin. + + The application-specific notification manager constructs the + notifications. Instances are later passed back to + ``set_notification_instances()`` in a dictionary keyed by notification + class. + """ + return [] + + def get_setting_classes(self): + """Return settings-page classes contributed by this plugin. + + The application owns the settings UI and storage. The manager only + collects these classes and returns them to the application from + ``setup_plugins()``. + """ + return [] + + def get_event_handlers(self): + """Return a mapping of event names to handlers, or ``None``. + + This is the preferred shared event hook surface. Return a mapping such + as ``{'shot_complete': self.on_shot_complete}``. Event names are + application-defined strings; this shared layer only collects, + normalizes, and orders handlers. Values may be plain callables or + :class:`Callback` instances created with the :class:`callback` + decorator. The manager sorts handlers for a given event by their + ``priority`` attribute. + """ + return None + + def get_callbacks(self): + """Return callbacks keyed by event name, or ``None``. + + Deprecated compatibility alias for :meth:`get_event_handlers`. + + Existing BLACS-style plugins may still override this method. New + shared plugin code should implement :meth:`get_event_handlers` + instead. ``BasePlugin`` routes this legacy name to the modern hook so + the preferred surface is the real implementation path. + """ + return self.get_event_handlers() + + def get_ui_contributions(self): + """Return app-context UI contributions. + + Return an iterable of dictionaries. Each contribution has a + ``context`` key naming an application-registered context. The + application owns what that context means: a known tab area, a deferred + MDI workspace, a dialog registry, a fixed frame, or another + app-specific host. The remaining keys are application-defined. + """ + return [] + + def get_menu_contributions(self): + """Return shared menu contributions. + + Return an iterable of dictionaries. This is the app-neutral menu path. + Existing BLACS-style plugins can continue using ``get_menu_class()`` + and :class:`MenuBuilder`; new apps can register a ``menus`` context and + route these dictionaries through :class:`MenuContext`. Apart from + stable menu keys such as ``location``, ``path``, ``group``, ``order``, + ``name``, and ``action``, the concrete menu objects remain + application-owned. + """ + return [] + + def set_menu_instance(self, menu): + """Receive the menu instance constructed for this plugin.""" + self.menu = menu + + def set_notification_instances(self, notifications): + """Receive notification instances constructed for this plugin.""" + self.notifications = notifications + + def plugin_setup_complete(self, data=None): + """Run after the application has finished plugin setup. + + ``data`` is an application-defined dictionary. Plugins commonly store + references to services or state they will need later when handling + events, building menus, or opening UI. For BLACS it contains + references such as the main UI, experiment config, plugin mapping, and + settings object. Older plugins may define this method without a + ``data`` argument; ``PluginManager.setup_complete()`` keeps that + compatibility behavior. + """ + pass + + def get_save_data(self): + """Return serializable plugin state for the next application start. + + The shape of this data is application-owned. Return plain data + structures that the application can persist and pass back as + ``initial_settings`` during the next start. + """ + return {} + + def close(self): + """Clean up resources owned by the plugin during application shutdown. + + Applications call this during shutdown. Stop timers, workers, + listeners, or other plugin-owned resources here. + """ + pass + + +class MenuBuilder(object): + """Build menus from the nested dictionary format used by plugins. + + Each menu description is a dictionary. Supported keys are: + + * ``name``: text for a menu or action + * ``menu_items``: child menu descriptions, making a submenu + * ``icon``: icon name passed to ``icon_factory`` for actions + * ``action``: callable connected to the action's ``triggered`` signal + * ``separator``: a truthy sentinel that inserts a separator + + ``icon_factory`` is optional and should return a Qt icon object or other + value accepted by ``addAction``. Supplying it keeps this module free of a + direct Qt dependency. + """ + def __init__(self, icon_factory=None): + """Create a menu builder. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` before adding an icon-bearing menu + action. Applications using Qt typically pass ``QIcon``. + """ + self.icon_factory = icon_factory + + def create_menu(self, parent, menu_parameters): + """Recursively build menus and actions from a nested menu dictionary.""" + if 'name' in menu_parameters: + if 'menu_items' in menu_parameters: + child = parent.addMenu(menu_parameters['name']) + for child_menu_params in menu_parameters['menu_items']: + self.create_menu(child, child_menu_params) + else: + # ``icon_factory`` stays outside this module so labscript-utils + # does not need to import a Qt binding directly. + if 'icon' in menu_parameters and self.icon_factory is not None: + child = parent.addAction( + self.icon_factory(menu_parameters['icon']), + menu_parameters['name'], + ) + else: + child = parent.addAction(menu_parameters['name']) + + if 'action' in menu_parameters: + child.triggered.connect(menu_parameters['action']) + + elif 'separator' in menu_parameters: + parent.addSeparator() + + +class MenuContext(object): + """Collect and render contribution-based menu actions. + + Applications register stable top-level menu locations, such as ``file`` or + ``tools``. Plugins contribute action dictionaries that request a location, + optional submenu path, separator group, order, action text, and callback. + Rendering is deferred until all plugins have contributed so ordering and + group separators can be computed across plugins. + """ + def __init__(self, icon_factory=None, logger=None): + """Create a menu context. + + Args: + icon_factory (callable, optional): Function called as + ``icon_factory(icon_name)`` for icon-bearing actions. + logger (logging.Logger, optional): Logger used for skipped or + malformed menu contributions. + """ + self.icon_factory = icon_factory + self.logger = logger or logging.getLogger(__name__) + self.locations = {} + self.contributions = [] + + def register_location(self, name, menu): + """Register an application-owned menu object as a stable location.""" + self.locations[name] = menu + + def add(self, plugin_name, contribution, data): + """Collect one menu contribution for later rendering. + + ``data`` is accepted for the generic context contract. Menu + contributions already contain their callbacks, so this context does not + need to inspect app data directly. + """ + self.contributions.append((plugin_name, contribution)) + + def render(self): + """Render all collected menu contributions into registered locations.""" + grouped = {} + group_orders = {} + + for plugin_name, contribution in self.contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a dictionary. " + "Skipping." % plugin_name + ) + continue + if 'location' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing location. " + "Skipping." % plugin_name + ) + continue + if 'name' not in contribution: + self.logger.error( + "Menu contribution from plugin '%s' missing name. " + "Skipping." % plugin_name + ) + continue + + location = contribution['location'] + if location not in self.locations: + self.logger.error( + "Menu contribution from plugin '%s' requested unknown " + "location '%s'. Skipping." % (plugin_name, location) + ) + continue + + path = contribution.get('path', ()) + if isinstance(path, str): + path = (path,) + else: + path = tuple(path) + + key = (location, path) + group = contribution.get('group', None) + if key not in group_orders: + group_orders[key] = {} + if group not in group_orders[key]: + group_orders[key][group] = len(group_orders[key]) + + grouped.setdefault(key, []).append((plugin_name, contribution)) + + menus = {} + for name, menu in self.locations.items(): + menus[(name, ())] = menu + + for key in sorted(grouped): + location, path = key + parent_path = () + for submenu_name in path: + submenu_path = parent_path + (submenu_name,) + submenu_key = (location, submenu_path) + if submenu_key not in menus: + menus[submenu_key] = menus[(location, parent_path)].addMenu( + submenu_name + ) + parent_path = submenu_path + menu = menus[(location, path)] + + contributions = sorted( + grouped[key], + key=lambda item: ( + group_orders[key][item[1].get('group', None)], + item[1].get('order', DEFAULT_PRIORITY), + item[0], + item[1]['name'], + ), + ) + + previous_group = None + for index, (plugin_name, contribution) in enumerate(contributions): + group = contribution.get('group', None) + if index and group != previous_group: + menu.addSeparator() + previous_group = group + + name = contribution['name'] + icon = contribution.get('icon', None) + if icon is not None and self.icon_factory is not None: + action = menu.addAction(self.icon_factory(icon), name) + else: + action = menu.addAction(name) + + callback = contribution.get('action', None) + if callback is not None: + action.triggered.connect(callback) + + shortcut = contribution.get('shortcut', None) + if shortcut is not None and hasattr(action, 'setShortcut'): + action.setShortcut(shortcut) + + checkable = contribution.get('checkable', False) + if hasattr(action, 'setCheckable'): + action.setCheckable(checkable) + + enabled = contribution.get('enabled', True) + if hasattr(action, 'setEnabled'): + action.setEnabled(enabled) + + +class PluginManager(object): + """Manage plugin discovery, lifecycle, and callback lookup. + + The manager keeps two internal mappings: + + * ``modules``: imported plugin modules keyed by plugin name + * ``plugins``: instantiated plugin objects keyed by plugin name + + An application normally creates one manager per plugin directory and uses + it to discover modules, instantiate plugin objects, build UI pieces, and + close plugins on shutdown. + """ + def __init__( + self, + plugin_package, + plugins_dir, + config, + config_section, + default_plugins=(), + logger=None, + ): + """Create a plugin manager. + + Args: + plugin_package (str): Import package containing plugin subpackages, + for example ``'myapp.plugins'``. + plugins_dir (str): Filesystem path scanned for plugin directories. + config: Config object with ``has_section()``, ``add_section()``, + ``items()``, ``set()``, and ``getboolean()`` methods. + config_section (str): Section containing plugin enable/disable + options. + default_plugins (iterable): Plugin names enabled by default when + first discovered. + logger (logging.Logger, optional): Logger used for plugin import, + setup, callback, and shutdown errors. + """ + self.plugin_package = plugin_package + self.plugins_dir = plugins_dir + self.config = config + self.config_section = config_section + self.default_plugins = set(default_plugins) + self.logger = logger or logging.getLogger(__name__) + self.modules = {} + self.plugins = {} + self.contexts = {} + self._logged_plugin_warnings = set() + + def discover_modules(self): + """Scan the plugin directory, update config defaults, and import enabled modules.""" + if not self.config.has_section(self.config_section): + self.config.add_section(self.config_section) + + configured_plugins = set( + name for name, val in self.config.items(self.config_section) + ) + + modules = {} + for module_name in os.listdir(self.plugins_dir): + module_path = os.path.join(self.plugins_dir, module_name) + if not os.path.isdir(module_path) or module_name == '__pycache__': + continue + + # Keep the config in sync with what is present on disk. + if module_name not in configured_plugins: + self.config.set( + self.config_section, + module_name, + str(module_name in self.default_plugins), + ) + configured_plugins.add(module_name) + + # Only load activated plugins. + if self.config.getboolean(self.config_section, module_name): + try: + module = importlib.import_module( + self.plugin_package + '.' + module_name + ) + except Exception: + self.logger.exception( + "Could not import plugin '%s'. Skipping." % module_name + ) + else: + modules[module_name] = module + + self.modules = modules + return modules + + def instantiate_plugins(self, plugin_settings=None): + """Create ``Plugin`` instances for each imported module. + + ``plugin_settings`` should map plugin names to the saved configuration + dictionary for that plugin. Missing entries fall back to an empty + dictionary. Instantiation failures are logged and skipped. + """ + if plugin_settings is None: + plugin_settings = {} + + plugins = {} + for module_name, module in self.modules.items(): + try: + plugins[module_name] = module.Plugin( + plugin_settings[module_name] + if module_name in plugin_settings else {} + ) + except Exception: + self.logger.exception( + "Could not instantiate plugin '%s'. Skipping" % module_name + ) + + self.plugins = plugins + return plugins + + def setup_plugins(self, data, notifications, menu_builder, menubar): + """Collect plugin UI and startup contributions. + + This is typically called after the application has created its legacy + menu and notification hosts and before it finishes wiring its settings + UI and callback registrations. The method returns a tuple of + ``(settings_pages, settings_callbacks)`` for the application to + consume. + """ + settings_pages = [] + settings_callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + # Setup settings page. + settings_pages.extend(plugin.get_setting_classes()) + + # Setup menu. + menu_class = plugin.get_menu_class() + if menu_class: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_menu_class'), + "Plugin '%s' uses deprecated get_menu_class(); " + "prefer get_menu_contributions() instead." % module_name + ) + # Must store a reference or else the methods called when + # the menu actions are triggered will be garbage collected. + menu = menu_class(data) + menu_builder.create_menu(menubar, menu.get_menu_items()) + plugin.set_menu_instance(menu) + + # Setup notifications. + plugin_notifications = {} + for notification_class in plugin.get_notification_classes(): + notifications.add_notification(notification_class) + plugin_notifications[notification_class] = ( + notifications.get_instance(notification_class) + ) + plugin.set_notification_instances(plugin_notifications) + + # Register callbacks. + callbacks = self._get_plugin_event_handlers(module_name, plugin) + # Save the settings_changed callback in a separate list for + # setting up later. + if callbacks and 'settings_changed' in callbacks: + settings_callbacks.append(callbacks['settings_changed']) + + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." % module_name + ) + + return settings_pages, settings_callbacks + + def register_context(self, name, context): + """Register an application-owned plugin contribution context.""" + self.contexts[name] = context + + def _get_contributions(self, plugin, module_name, method_name, label): + """Return a plugin contribution iterable, logging malformed results.""" + if not hasattr(plugin, method_name): + return [] + + try: + contributions = getattr(plugin, method_name)() + except Exception: + self.logger.exception( + "Error getting %s contributions from plugin '%s'. Skipping." + % (label, module_name) + ) + return [] + + if contributions is None: + return [] + + if ( + isinstance(contributions, (dict, str, bytes)) + or not hasattr(contributions, '__iter__') + ): + self.logger.error( + "Plugin '%s' %s contributions must be an iterable of " + "dictionaries. Skipping." % (module_name, label) + ) + return [] + + return contributions + + def setup_contexts(self, data): + """Route plugin-declared UI and menu contributions to app contexts.""" + for module_name, plugin in self.plugins.items(): + ui_contributions = self._get_contributions( + plugin, + module_name, + 'get_ui_contributions', + 'UI', + ) + + for contribution in ui_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "UI contribution from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + if 'context' not in contribution: + self.logger.error( + "UI contribution from plugin '%s' missing context. " + "Skipping." % module_name + ) + continue + + context_name = contribution['context'] + if context_name not in self.contexts: + self.logger.error( + "UI contribution from plugin '%s' requested unknown " + "context '%s'. Skipping." % (module_name, context_name) + ) + continue + + try: + self.contexts[context_name].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding UI contribution from plugin '%s' to " + "context '%s'. Skipping." % (module_name, context_name) + ) + + menu_contributions = self._get_contributions( + plugin, + module_name, + 'get_menu_contributions', + 'menu', + ) + + if menu_contributions and 'menus' not in self.contexts: + self.logger.error( + "Plugin '%s' provided menu contributions, but no 'menus' " + "context is registered. Skipping." % module_name + ) + continue + + for contribution in menu_contributions: + if not isinstance(contribution, dict): + self.logger.error( + "Menu contribution from plugin '%s' is not a " + "dictionary. Skipping." % module_name + ) + continue + try: + self.contexts['menus'].add(module_name, contribution, data) + except Exception: + self.logger.exception( + "Error adding menu contribution from plugin '%s'. " + "Skipping." % module_name + ) + + def setup_complete(self, data): + """Notify plugins that the application has finished startup.""" + for module_name, plugin in self.plugins.items(): + try: + plugin.plugin_setup_complete(data) + except Exception: + self.logger.exception( + "Error in plugin_setup_complete() for plugin '%s'. " + "Trying again with old call signature..." % module_name + ) + # Backwards compatibility for old plugins. + try: + plugin.plugin_setup_complete() + self.logger.warning( + "Plugin '%s' using old API. Please update " + "Plugin.plugin_setup_complete method to accept a " + "dictionary of blacs_data as the only argument." + % module_name + ) + except Exception: + self.logger.exception( + "Plugin '%s' error. Plugin may not be functional." + % module_name + ) + + def _defines_hook(self, plugin, method_name): + """Return whether ``plugin`` overrides ``method_name`` meaningfully.""" + plugin_method = getattr(type(plugin), method_name, None) + if plugin_method is None: + return False + + base_method = getattr(BasePlugin, method_name, None) + return plugin_method is not base_method + + def _get_plugin_event_handlers(self, module_name, plugin): + """Return normalized event handlers for one plugin.""" + has_modern = self._defines_hook(plugin, 'get_event_handlers') + has_legacy = self._defines_hook(plugin, 'get_callbacks') + + if has_modern: + if has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' defines deprecated get_callbacks() and " + "preferred get_event_handlers(); ignoring get_callbacks()." + % module_name + ) + handlers = plugin.get_event_handlers() + elif has_legacy: + _log_once( + self.logger, + self._logged_plugin_warnings, + logging.WARNING, + (module_name, 'get_callbacks'), + "Plugin '%s' uses deprecated get_callbacks(); " + "implement get_event_handlers() instead." % module_name + ) + handlers = plugin.get_callbacks() + else: + return None + + if handlers is None: + return None + if not isinstance(handlers, Mapping): + self.logger.error( + "Plugin '%s' event handlers must be a mapping. Skipping." + % module_name + ) + return None + + return handlers + + def get_event_handlers(self, name): + """Return all event handlers registered for ``name``, sorted by priority.""" + callbacks = [] + + for module_name, plugin in self.plugins.items(): + try: + plugin_callbacks = self._get_plugin_event_handlers( + module_name, + plugin, + ) + if plugin_callbacks and name in plugin_callbacks: + callbacks.append(plugin_callbacks[name]) + except Exception: + self.logger.exception('Error getting callbacks from %s.' % str(plugin)) + + callbacks.sort( + key=lambda callback: getattr(callback, 'priority', DEFAULT_PRIORITY) + ) + return callbacks + + def get_callbacks(self, name): + """Deprecated compatibility wrapper for :meth:`get_event_handlers`.""" + warnings.warn( + "PluginManager.get_callbacks() is deprecated; use " + "PluginManager.get_event_handlers() instead.", + DeprecationWarning, + stacklevel=2, + ) + return self.get_event_handlers(name) + + def close_plugins(self): + """Call ``close()`` on every plugin during application shutdown.""" + for module_name, plugin in self.plugins.items(): + try: + plugin.close() + except Exception as e: + self.logger.error( + 'Could not close plugin %s. Error was: %s' + % (module_name, str(e)) + ) From aaa642b912dae1a507bd520326876a6df0f5157f Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 27 Apr 2026 12:26:12 -0400 Subject: [PATCH 34/42] Added ability to share services. --- labscript_utils/plugins.py | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 1033a20a..36dc0865 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -788,6 +788,16 @@ def get_save_data(self): """ return {} + def get_services(self): + """Return named services exposed by this plugin. + + Return a mapping of service names to concrete objects. Applications can + aggregate these mappings into a shared service registry before calling + :meth:`plugin_setup_complete`, making plugin-to-plugin dependencies + order-independent. + """ + return {} + def close(self): """Clean up resources owned by the plugin during application shutdown. @@ -1033,6 +1043,7 @@ def __init__( self.logger = logger or logging.getLogger(__name__) self.modules = {} self.plugins = {} + self.services = {} self.contexts = {} self._logged_plugin_warnings = set() @@ -1260,6 +1271,53 @@ def setup_contexts(self, data): "Skipping." % module_name ) + def collect_services(self, base_services=None): + """Merge app-owned and plugin-owned services into one registry.""" + if base_services is None: + services = {} + elif isinstance(base_services, Mapping): + services = dict(base_services) + else: + self.logger.error( + "Base services must be a mapping. Ignoring invalid registry." + ) + services = {} + + for module_name, plugin in self.plugins.items(): + if not hasattr(plugin, "get_services"): + continue + + try: + plugin_services = plugin.get_services() + except Exception: + self.logger.exception( + "Error getting services from plugin '%s'. Skipping." + % module_name + ) + continue + + if plugin_services is None: + continue + if not isinstance(plugin_services, Mapping): + self.logger.error( + "Plugin '%s' services must be a mapping. Skipping." + % module_name + ) + continue + + for service_name, service in plugin_services.items(): + if service_name in services: + self.logger.error( + "Plugin '%s' service '%s' conflicts with an existing " + "service. Keeping the original registration." + % (module_name, service_name) + ) + continue + services[service_name] = service + + self.services = services + return services + def setup_complete(self, data): """Notify plugins that the application has finished startup.""" for module_name, plugin in self.plugins.items(): From 44f7aab15117d3d59b7c390ada24668518fbb998 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 27 Apr 2026 14:07:54 -0400 Subject: [PATCH 35/42] Add plugin service aggregation tests --- tests/test_plugins.py | 106 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/test_plugins.py diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 00000000..c583d32f --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,106 @@ +import logging +import importlib.util +from pathlib import Path +import unittest + +MODULE_PATH = Path(__file__).resolve().parents[1] / "labscript_utils" / "plugins.py" +SPEC = importlib.util.spec_from_file_location("labscript_utils.plugins", MODULE_PATH) +plugins_module = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(plugins_module) + +BasePlugin = plugins_module.BasePlugin +PluginManager = plugins_module.PluginManager + + +class DummyConfig: + def has_section(self, section): + del section + return True + + def add_section(self, section): + del section + + def items(self, section): + del section + return [] + + def set(self, section, name, value): + del section, name, value + + def getboolean(self, section, name): + del section, name + return False + + +class RecordingHandler(logging.Handler): + def __init__(self): + super().__init__() + self.records = [] + + def emit(self, record): + self.records.append(record) + + +class ServicePlugin(BasePlugin): + def __init__(self, services): + super().__init__({}) + self._services = services + + def get_services(self): + return self._services + + +class TestPluginServices(unittest.TestCase): + def make_manager(self): + logger = logging.getLogger("labscript_utils.plugins.tests") + logger.setLevel(logging.DEBUG) + handler = RecordingHandler() + logger.addHandler(handler) + self.addCleanup(logger.removeHandler, handler) + manager = PluginManager( + plugin_package="unused", + plugins_dir="unused", + config=DummyConfig(), + config_section="unused", + logger=logger, + ) + return manager, handler + + def test_base_plugin_exposes_no_services_by_default(self): + plugin = BasePlugin({}) + self.assertEqual(plugin.get_services(), {}) + + def test_collect_services_merges_base_and_plugin_services(self): + manager, _handler = self.make_manager() + manager.plugins = { + "alpha": ServicePlugin({"answer": 42}), + "beta": ServicePlugin({"greeter": str.upper}), + } + + services = manager.collect_services({"execute_command": object()}) + + self.assertIn("execute_command", services) + self.assertEqual(services["answer"], 42) + self.assertIs(services["greeter"], str.upper) + self.assertIs(manager.services, services) + + def test_collect_services_skips_invalid_and_conflicting_entries(self): + manager, handler = self.make_manager() + manager.plugins = { + "alpha": ServicePlugin({"shared": "first", "unique": "ok"}), + "beta": ServicePlugin({"shared": "second"}), + "gamma": ServicePlugin(["not", "a", "mapping"]), + } + + services = manager.collect_services() + + self.assertEqual(services["shared"], "first") + self.assertEqual(services["unique"], "ok") + messages = [record.getMessage() for record in handler.records] + self.assertTrue(any("must be a mapping" in message for message in messages)) + self.assertTrue(any("conflicts with an existing service" in message for message in messages)) + + +if __name__ == "__main__": + unittest.main() From 57c889fc35c2752408038f195ef538388f9da881 Mon Sep 17 00:00:00 2001 From: spielman Date: Mon, 11 May 2026 09:44:23 -0400 Subject: [PATCH 36/42] Add ordered plugin setup activities --- labscript_utils/plugins.py | 141 ++++++++++++++++++++++++++++++-- tests/test_plugins.py | 160 +++++++++++++++++++++++++++++++++++++ 2 files changed, 296 insertions(+), 5 deletions(-) diff --git a/labscript_utils/plugins.py b/labscript_utils/plugins.py index 36dc0865..46306648 100644 --- a/labscript_utils/plugins.py +++ b/labscript_utils/plugins.py @@ -558,7 +558,8 @@ def on_fast(self): ``setup_complete()`` is called after the application has finished startup. Use it to access app-owned state that was not available during construction, such as the main window, settings object, registered services, or concrete -context objects:: +context objects. Existing plugins can keep overriding +``plugin_setup_complete()``:: class Plugin(BasePlugin): def plugin_setup_complete(self, data): @@ -570,6 +571,28 @@ def plugin_setup_complete(self, data): ``plugin_setup_complete(self)`` without the ``data`` argument, and the manager keeps that compatibility path. +Plugins that need ordered startup work can return setup activity records from +``get_setup_activities()``. Each record has ``name``, ``priority``, and +``action`` keys; lower priority values run first. The default +``get_setup_activities()`` implementation returns one activity for +``plugin_setup_complete()``, so existing plugins use the same ordered pipeline +as modern plugins:: + + class Plugin(BasePlugin): + def get_setup_activities(self): + return [ + { + 'name': 'bind_services', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': self.bind_services, + }, + { + 'name': 'start_worker', + 'priority': DEFAULT_SETUP_PRIORITY + 10, + 'action': self.start_worker, + }, + ] + Shutdown cleanup with ``close()`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -594,9 +617,11 @@ def close(self): DEFAULT_PRIORITY = 10 +DEFAULT_SETUP_PRIORITY = 0 __all__ = [ 'DEFAULT_PRIORITY', + 'DEFAULT_SETUP_PRIORITY', 'Callback', 'callback', 'BasePlugin', @@ -779,6 +804,23 @@ def plugin_setup_complete(self, data=None): """ pass + def get_setup_activities(self): + """Return ordered setup activities for this plugin. + + Existing plugins can override :meth:`plugin_setup_complete` and inherit + this method. Modern plugins can override this method to return multiple + activity records. Each record must provide ``name``, ``priority``, and + ``action`` keys; the manager executes all plugin activities through one + ordered setup pipeline. + """ + return [ + { + 'name': 'plugin_setup_complete', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': self.plugin_setup_complete, + }, + ] + def get_save_data(self): """Return serializable plugin state for the next application start. @@ -1319,18 +1361,107 @@ def collect_services(self, base_services=None): return services def setup_complete(self, data): - """Notify plugins that the application has finished startup.""" + """Run plugin setup activities after application startup.""" + setup_activities = [] + for module_name, plugin in self.plugins.items(): + if hasattr(plugin, 'get_setup_activities'): + try: + plugin_activities = plugin.get_setup_activities() + except Exception: + self.logger.exception( + "Error getting setup activities from plugin '%s'. " + "Skipping." % module_name + ) + continue + elif hasattr(plugin, 'plugin_setup_complete'): + plugin_activities = [ + { + 'name': 'plugin_setup_complete', + 'priority': DEFAULT_SETUP_PRIORITY, + 'action': plugin.plugin_setup_complete, + }, + ] + else: + continue + + if plugin_activities is None: + continue + if ( + isinstance(plugin_activities, (dict, str, bytes)) + or not hasattr(plugin_activities, '__iter__') + ): + self.logger.error( + "Plugin '%s' setup activities must be an iterable of " + "dictionaries. Skipping." % module_name + ) + continue + + for activity in plugin_activities: + if not isinstance(activity, Mapping): + self.logger.error( + "Setup activity from plugin '%s' is not a dictionary. " + "Skipping." % module_name + ) + continue + + try: + name = activity['name'] + priority = activity['priority'] + action = activity['action'] + except KeyError as exc: + self.logger.error( + "Setup activity from plugin '%s' missing key '%s'. " + "Skipping." % (module_name, exc.args[0]) + ) + continue + + if not callable(action): + self.logger.error( + "Setup activity '%s' from plugin '%s' action is not " + "callable. Skipping." % (name, module_name) + ) + continue + + setup_activities.append( + { + 'module_name': module_name, + 'plugin': plugin, + 'name': name, + 'priority': priority, + 'action': action, + } + ) + + setup_activities.sort( + key=lambda activity: ( + activity['priority'], + activity['module_name'], + activity['name'], + ) + ) + + for activity in setup_activities: + module_name = activity['module_name'] + activity_name = activity['name'] + action = activity['action'] try: - plugin.plugin_setup_complete(data) + action(data) except Exception: + if activity_name != 'plugin_setup_complete': + self.logger.exception( + "Error in setup activity '%s' for plugin '%s'. " + "Plugin may not be functional." + % (activity_name, module_name) + ) + continue + self.logger.exception( "Error in plugin_setup_complete() for plugin '%s'. " "Trying again with old call signature..." % module_name ) - # Backwards compatibility for old plugins. try: - plugin.plugin_setup_complete() + action() self.logger.warning( "Plugin '%s' using old API. Please update " "Plugin.plugin_setup_complete method to accept a " diff --git a/tests/test_plugins.py b/tests/test_plugins.py index c583d32f..3ebc1ca9 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -11,6 +11,7 @@ BasePlugin = plugins_module.BasePlugin PluginManager = plugins_module.PluginManager +DEFAULT_SETUP_PRIORITY = plugins_module.DEFAULT_SETUP_PRIORITY class DummyConfig: @@ -102,5 +103,164 @@ def test_collect_services_skips_invalid_and_conflicting_entries(self): self.assertTrue(any("conflicts with an existing service" in message for message in messages)) +class TestPluginSetupActivities(unittest.TestCase): + def make_manager(self): + logger = logging.getLogger("labscript_utils.plugins.setup_tests") + logger.setLevel(logging.DEBUG) + handler = RecordingHandler() + logger.addHandler(handler) + self.addCleanup(logger.removeHandler, handler) + manager = PluginManager( + plugin_package="unused", + plugins_dir="unused", + config=DummyConfig(), + config_section="unused", + logger=logger, + ) + return manager, handler + + def test_legacy_plugin_setup_complete_runs_as_default_activity(self): + calls = [] + data = {"services": object()} + + class LegacySetupPlugin(BasePlugin): + def plugin_setup_complete(self, setup_data): + calls.append(("legacy", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"legacy": LegacySetupPlugin({})} + + manager.setup_complete(data) + + self.assertEqual(calls, [("legacy", data)]) + + def test_duck_typed_plugin_setup_complete_still_runs(self): + calls = [] + data = {"services": object()} + + class DuckTypedSetupPlugin: + def plugin_setup_complete(self, setup_data): + calls.append(("duck", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"duck": DuckTypedSetupPlugin()} + + manager.setup_complete(data) + + self.assertEqual(calls, [("duck", data)]) + + def test_multiple_setup_activities_run_in_priority_order(self): + calls = [] + data = {"app": object()} + + class ActivityPlugin(BasePlugin): + def get_setup_activities(self): + return [ + { + "name": "late", + "priority": DEFAULT_SETUP_PRIORITY + 10, + "action": self.late, + }, + { + "name": "early", + "priority": DEFAULT_SETUP_PRIORITY - 10, + "action": self.early, + }, + ] + + def early(self, setup_data): + calls.append(("early", setup_data)) + + def late(self, setup_data): + calls.append(("late", setup_data)) + + manager, _handler = self.make_manager() + manager.plugins = {"activity": ActivityPlugin({})} + + manager.setup_complete(data) + + self.assertEqual(calls, [("early", data), ("late", data)]) + + def test_setup_activity_ordering_is_deterministic_across_plugins(self): + calls = [] + + class NamedActivityPlugin(BasePlugin): + def __init__(self, label): + super().__init__({}) + self.label = label + + def get_setup_activities(self): + return [ + { + "name": "same_priority", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.record, + }, + ] + + def record(self, data): + del data + calls.append(self.label) + + manager, _handler = self.make_manager() + manager.plugins = { + "zeta": NamedActivityPlugin("zeta"), + "alpha": NamedActivityPlugin("alpha"), + } + + manager.setup_complete({}) + + self.assertEqual(calls, ["alpha", "zeta"]) + + def test_setup_activity_names_break_priority_and_plugin_ties(self): + calls = [] + + class TiedActivityPlugin(BasePlugin): + def get_setup_activities(self): + return [ + { + "name": "second", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.second, + }, + { + "name": "first", + "priority": DEFAULT_SETUP_PRIORITY, + "action": self.first, + }, + ] + + def first(self, data): + del data + calls.append("first") + + def second(self, data): + del data + calls.append("second") + + manager, _handler = self.make_manager() + manager.plugins = {"plugin": TiedActivityPlugin({})} + + manager.setup_complete({}) + + self.assertEqual(calls, ["first", "second"]) + + def test_no_argument_plugin_setup_complete_fallback_still_runs(self): + calls = [] + + class NoArgumentSetupPlugin(BasePlugin): + def plugin_setup_complete(self): + calls.append("legacy-no-arg") + + manager, handler = self.make_manager() + manager.plugins = {"legacy": NoArgumentSetupPlugin({})} + + manager.setup_complete({"unused": object()}) + + messages = [record.getMessage() for record in handler.records] + self.assertEqual(calls, ["legacy-no-arg"]) + self.assertTrue(any("using old API" in message for message in messages)) + + if __name__ == "__main__": unittest.main() From 17e36cc8b5a45f61157f24decba3de4a2b686230 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:01:51 -0400 Subject: [PATCH 37/42] Set macOS Qt application names from desktop app config --- labscript_utils/splash.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 6b24b095..8d8d7205 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -11,7 +11,9 @@ # # ##################################################################### +import json import sys +from pathlib import Path from labscript_utils import dedent try: @@ -45,7 +47,23 @@ def configure_qapplication(qapplication): if sys.platform == 'darwin': icon_path = qapplication.property('_labscript_icon_path') if icon_path: - icon = QtGui.QIcon(icon_path) + icon_path = Path(icon_path) + try: + config = json.loads( + (icon_path.parent / 'desktop-app.json').read_text(encoding='utf8') + ) + except (OSError, json.JSONDecodeError): + application_name = None + else: + module_config = config.get('modules', {}).get(icon_path.stem, {}) + application_name = ( + module_config.get('short_display_name') + or module_config.get('display_name') + ) + if application_name: + qapplication.setApplicationName(application_name) + qapplication.setApplicationDisplayName(application_name) + icon = QtGui.QIcon(str(icon_path)) if not icon.isNull(): qapplication.setWindowIcon(icon) if qapplication.property('_labscript_qapplication_configured'): From b8989e341716bc5a9ec8dc19657f8bc7691cf442 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:12:22 -0400 Subject: [PATCH 38/42] Read macOS Qt app name from application property --- labscript_utils/splash.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 8d8d7205..82370340 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -11,9 +11,7 @@ # # ##################################################################### -import json import sys -from pathlib import Path from labscript_utils import dedent try: @@ -45,25 +43,13 @@ def configure_qapplication(qapplication): """Apply labscript-wide QApplication configuration.""" qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) if sys.platform == 'darwin': + application_name = qapplication.property('_labscript_application_name') + if application_name: + qapplication.setApplicationName(application_name) + qapplication.setApplicationDisplayName(application_name) icon_path = qapplication.property('_labscript_icon_path') if icon_path: - icon_path = Path(icon_path) - try: - config = json.loads( - (icon_path.parent / 'desktop-app.json').read_text(encoding='utf8') - ) - except (OSError, json.JSONDecodeError): - application_name = None - else: - module_config = config.get('modules', {}).get(icon_path.stem, {}) - application_name = ( - module_config.get('short_display_name') - or module_config.get('display_name') - ) - if application_name: - qapplication.setApplicationName(application_name) - qapplication.setApplicationDisplayName(application_name) - icon = QtGui.QIcon(str(icon_path)) + icon = QtGui.QIcon(icon_path) if not icon.isNull(): qapplication.setWindowIcon(icon) if qapplication.property('_labscript_qapplication_configured'): @@ -93,11 +79,16 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath): + def __init__(self, imagepath, application_name=None): + if application_name is not None: + QtCore.QCoreApplication.setApplicationName(application_name) + QtGui.QGuiApplication.setApplicationDisplayName(application_name) self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: self.qapplication = QtWidgets.QApplication(sys.argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) + if application_name is not None: + self.qapplication.setProperty('_labscript_application_name', application_name) configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() From a95f8835dc005a7d6b1fef202c4d65d8094339b1 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:39:41 -0400 Subject: [PATCH 39/42] Set macOS QApplication argv name from splash --- labscript_utils/splash.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 82370340..d284e8db 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -43,10 +43,6 @@ def configure_qapplication(qapplication): """Apply labscript-wide QApplication configuration.""" qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) if sys.platform == 'darwin': - application_name = qapplication.property('_labscript_application_name') - if application_name: - qapplication.setApplicationName(application_name) - qapplication.setApplicationDisplayName(application_name) icon_path = qapplication.property('_labscript_icon_path') if icon_path: icon = QtGui.QIcon(icon_path) @@ -68,7 +64,6 @@ def configure_qapplication(qapplication): qapplication.setProperty('_labscript_qapplication_configured', True) return qapplication - class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -80,15 +75,14 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath, application_name=None): - if application_name is not None: - QtCore.QCoreApplication.setApplicationName(application_name) - QtGui.QGuiApplication.setApplicationDisplayName(application_name) self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + argv = sys.argv + if application_name is not None: + # Create a new argv so QApplication can alter it without mutating sys.argv. + argv = [application_name] + argv[1:] + self.qapplication = QtWidgets.QApplication(argv) self.qapplication.setProperty('_labscript_icon_path', imagepath) - if application_name is not None: - self.qapplication.setProperty('_labscript_application_name', application_name) configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() From c5cbf710e28211cd80901e7c954427511afdc189 Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:43:45 -0400 Subject: [PATCH 40/42] Configure splash icon through QApplication helper --- labscript_utils/splash.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 1a93223c..9d478dc3 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -64,7 +64,7 @@ def configure_qapplication(qapplication): qapplication.setProperty('_labscript_qapplication_configured', True) return qapplication -def get_qapplication(argv=None, application_name=None): +def get_qapplication(argv=None, application_name=None, icon_path=None): qapplication = QtWidgets.QApplication.instance() if qapplication is None: @@ -75,6 +75,8 @@ def get_qapplication(argv=None, application_name=None): qapplication = QtWidgets.QApplication(argv) + if icon_path is not None: + qapplication.setProperty('_labscript_icon_path', icon_path) return configure_qapplication(qapplication) @@ -89,9 +91,9 @@ class Splash(QtWidgets.QFrame): FG = '#000000' def __init__(self, imagepath, application_name=None): - self.qapplication = get_qapplication(application_name=application_name) - self.qapplication.setProperty('_labscript_icon_path', imagepath) - configure_qapplication(self.qapplication) + self.qapplication = get_qapplication( + application_name=application_name, icon_path=imagepath + ) super().__init__() self.icon = QtGui.QPixmap() self.icon.load(imagepath) From 6dae69b897933708a127c3052364eea796644c8a Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 11:46:05 -0400 Subject: [PATCH 41/42] Rename splash image path argument --- labscript_utils/splash.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 9d478dc3..46922905 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -90,15 +90,15 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath, application_name=None): + def __init__(self, icon_path, application_name=None): self.qapplication = get_qapplication( - application_name=application_name, icon_path=imagepath + application_name=application_name, icon_path=icon_path ) super().__init__() self.icon = QtGui.QPixmap() - self.icon.load(imagepath) + self.icon.load(icon_path) if self.icon.isNull(): - raise ValueError("Invalid image file: {}.\n".format(imagepath)) + raise ValueError("Invalid image file: {}.\n".format(icon_path)) self.icon = self.icon.scaled( self.imwidth, self.imheight, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation ) From 7862f9256ee04d81db6f2124ec3ef967c9dcc6db Mon Sep 17 00:00:00 2001 From: spielman Date: Tue, 12 May 2026 12:02:21 -0400 Subject: [PATCH 42/42] Configure macOS Qt application presentation --- labscript_utils/splash.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/labscript_utils/splash.py b/labscript_utils/splash.py index 75cedf59..88f73cda 100644 --- a/labscript_utils/splash.py +++ b/labscript_utils/splash.py @@ -39,6 +39,31 @@ QtWidgets.QApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True) +def configure_qapplication(qapplication): + """Apply labscript-wide QApplication configuration.""" + qapplication.setAttribute(Qt.AA_DontShowIconsInMenus, False) + if sys.platform == 'darwin': + icon_path = qapplication.property('_labscript_icon_path') + if icon_path: + icon = QtGui.QIcon(icon_path) + if not icon.isNull(): + qapplication.setWindowIcon(icon) + if qapplication.property('_labscript_qapplication_configured'): + return qapplication + # Native macOS widget styling makes many Qt controls look inconsistent + # with the rest of the suite. Use Qt's own style, but preserve the + # current palette so dark/light appearance still follows the active + # theme. + palette = QtGui.QPalette(qapplication.palette()) + style = QtWidgets.QStyleFactory.create('Fusion') + if style is not None: + qapplication.setStyle(style) + qapplication.setPalette(palette) + elif qapplication.property('_labscript_qapplication_configured'): + return qapplication + qapplication.setProperty('_labscript_qapplication_configured', True) + return qapplication + class Splash(QtWidgets.QFrame): w = 250 h = 230 @@ -49,15 +74,21 @@ class Splash(QtWidgets.QFrame): BG = '#ffffff' FG = '#000000' - def __init__(self, imagepath): + def __init__(self, icon_path, application_name=None): self.qapplication = QtWidgets.QApplication.instance() if self.qapplication is None: - self.qapplication = QtWidgets.QApplication(sys.argv) + argv = sys.argv + if application_name is not None: + # Create a new argv so QApplication can alter it without mutating sys.argv. + argv = [application_name] + argv[1:] + self.qapplication = QtWidgets.QApplication(argv) + self.qapplication.setProperty('_labscript_icon_path', icon_path) + configure_qapplication(self.qapplication) super().__init__() self.icon = QtGui.QPixmap() - self.icon.load(imagepath) + self.icon.load(icon_path) if self.icon.isNull(): - raise ValueError("Invalid image file: {}.\n".format(imagepath)) + raise ValueError("Invalid image file: {}.\n".format(icon_path)) self.icon = self.icon.scaled( self.imwidth, self.imheight, Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation )