Runmanager queue simple

NEWS.md

@@ -1,3 +1,10 @@
+## Unreleased
+
+- TOML-backed labconfig and app-config writing now depends on `tomli-w`. The package
+  dependency is declared in `labscript-utils`, but existing editable or conda
+  environments may need that dependency installed explicitly when upgrading.
+
+
 ## [2.15.0] - 2019-12-04
 
 This release includes one bugfix, one enhancement, one update for compatibility with a
@@ -60,4 +67,4 @@ compatibility with a newer version of a library.
   ([PR #91](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/91))
 
 - Compatibility with `importlib_metadata` >= 0.21. Contributed by Chris Billington
-  ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88))
+  ([PR #92](https://bitbucket.org/labscript_suite/labscript_utils/pull-requests/88))

README.md

@@ -19,3 +19,9 @@ Shared modules used by the [*labscript suite*](https://github.com/labscript-suit
 ## Installation
 
 labscript-utils is distributed as a Python package on [PyPI](https://pypi.org/user/labscript-suite) and [Anaconda Cloud](https://anaconda.org/labscript-suite), and should be installed with other components of the _labscript suite_. Please see the [installation guide](https://docs.labscriptsuite.org/en/latest/installation) for details.
+
+TOML-backed configuration support depends on `tomli-w` at runtime. This dependency is
+declared in `labscript-utils` package metadata, so fresh installs should receive it
+automatically. Existing editable or conda environments created before this dependency
+was added may need `conda install tomli-w` or an updated `labscript-utils`
+installation.

docs/source/labconfig.rst

@@ -1,19 +1,19 @@
-The *labconfig.ini* file
+The *labconfig.toml* file
 ========================
 
-The `labconfig.ini` file is a global configuration file for your **labscript-suite** installation.
+The `labconfig.toml` file is a global configuration file for your **labscript-suite** installation.
 It contains configurable settings that govern how the individual components of the suite operate.
 The name of this file must be the host computer's system name.
-So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.ini`.
+So if my system's name was `heisenberg`, the labconfig file name would be `heisenberg.toml`.
 This file should be located in the `labscript-suite` directory in the user space, in the `labconfig` subdirectory.
 
 When :doc:`installing the **labscript-suite** for the first time <labscript-suite:installation/index>`,
 running the `labscript-profile-create` command will automatically generate the `labscript-suite` user space directory in the correct place
-and generate a `labconfig.ini` file for use on your system.
-By editing the `ini` file named after your system, you can update the configuration settings of your **labscript-suite** installation.
+and generate a `labconfig.toml` file for use on your system.
+By editing the TOML file named after your system, you can update the configuration settings of your **labscript-suite** installation.
 
-The Default *labconfig.ini*
----------------------------
+The Default *labconfig.toml*
+----------------------------
 
 Below is a copy of the default lab configuration if you were to install the **labscript-suite** today.
 
@@ -23,5 +23,5 @@ Below is a copy of the default lab configuration if you were to install the **la
 	Instead, if keys are missing from your local profile, default behavior will be assumed.
 	To implement the added functionality, you will need to manually add/change the keys in your local labconfig.
 
-.. include:: ../../labscript_profile/default_profile/labconfig/example.ini
+.. include:: ../../labscript_profile/default_profile/labconfig/example.toml
 	:code:

labscript_profile/__init__.py

@@ -1,12 +1,14 @@
 import site
 import sys
 import os
-from configparser import ConfigParser, NoSectionError, NoOptionError
+import configparser
+from ast import literal_eval
 from pathlib import Path
 from subprocess import check_output
 import socket
 from getpass import getuser
 
+from .toml_config import TomlConfigParser, dump_toml_file, load_toml_file
 
 # The contents of this file are imported every time the Python interpreter starts up,
 # owing to our custom .pth file that runs the below two functions. This ensures that
@@ -36,6 +38,13 @@ def hostname():
 
 
 def default_labconfig_path():
+    if LABSCRIPT_SUITE_PROFILE is None:
+        return None
+    return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.toml'
+
+# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+def legacy_labconfig_path():
+    """Temporary legacy INI labconfig location. Slated for removal soon."""
     if LABSCRIPT_SUITE_PROFILE is None:
         return None
     return LABSCRIPT_SUITE_PROFILE / 'labconfig' / f'{hostname()}.ini'
@@ -47,17 +56,113 @@ def add_userlib_and_pythonlib():
     re-implements finding and reading the config file so as to not import
     labscript_utils, since we dont' want to import something like labscript_utils every
     time the interpreter starts up"""
-    labconfig = default_labconfig_path()
-    if labconfig is not None and labconfig.exists():
-        # str() below is for py36 compat, where ConfigParser can't deal with Path objs
-        config = ConfigParser(
-            defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)}
-        )
-        config.read(labconfig)
-        for option in ['userlib', 'pythonlib']:
-            try:
-                paths = config.get('DEFAULT', option).split(',')
-            except (NoSectionError, NoOptionError):
-                paths = []
-            for path in paths:
+    labconfig = ensure_labconfig()
+    if labconfig is None:
+        return
+    if not labconfig.exists():
+        return
+    config = TomlConfigParser(defaults={'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)})
+    config.read_toml(labconfig)
+    for option in ['userlib', 'pythonlib']:
+        try:
+            paths = config.get('default', option)
+        except (configparser.NoSectionError, configparser.NoOptionError):
+            paths = ''
+        if paths:
+            for path in paths.split(','):
                 site.addsitedir(path)
+
+
+def ensure_labconfig():
+    """Return the TOML labconfig path, converting a legacy INI file if needed.
+
+    The legacy conversion path is temporary and slated for removal soon.
+    """
+    labconfig = default_labconfig_path()
+    if labconfig is None:
+        return None
+    if labconfig.exists():
+        return labconfig
+    # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+    legacy = legacy_labconfig_path()
+    if legacy is None or not legacy.exists():
+        return labconfig
+    _convert_legacy_labconfig(legacy, labconfig)
+    return labconfig
+
+
+def _example_labconfig_data():
+    if LABSCRIPT_SUITE_PROFILE is not None:
+        profile_example = LABSCRIPT_SUITE_PROFILE / 'labconfig' / 'example.toml'
+        if profile_example.exists():
+            return load_toml_file(profile_example)
+    bundled_example = Path(__file__).resolve().parent / 'default_profile' / 'labconfig' / 'example.toml'
+    if bundled_example.exists():
+        return load_toml_file(bundled_example)
+    return {}
+
+# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+def _coerce_legacy_value(value, template_value):
+    if template_value is None:
+        return value
+    if isinstance(template_value, bool):
+        if isinstance(value, bool):
+            return value
+        if isinstance(value, str):
+            lowered = value.strip().lower()
+            if lowered in configparser.ConfigParser.BOOLEAN_STATES:
+                return configparser.ConfigParser.BOOLEAN_STATES[lowered]
+        return value
+    if isinstance(template_value, int):
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return value
+    if isinstance(template_value, float):
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return value
+    if isinstance(template_value, list):
+        if not isinstance(value, str):
+            return value
+        try:
+            parsed = literal_eval(value)
+        except (SyntaxError, ValueError):
+            return value
+        if not isinstance(parsed, (list, tuple)):
+            return value
+        if not template_value:
+            return list(parsed)
+        template_item = template_value[0]
+        return [_coerce_legacy_value(item, template_item) for item in parsed]
+    return value
+
+# LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+def _convert_legacy_labconfig(legacy_path, toml_path):
+    """Temporary legacy migration path from INI to TOML. Slated for removal soon."""
+    config = configparser.ConfigParser(interpolation=None)
+    config.read(legacy_path)
+    template_data = _example_labconfig_data()
+    default_template = template_data.get('default', {})
+    data = {
+        'default': {
+            key: _coerce_legacy_value(value, default_template.get(key))
+            for key, value in config.defaults().items()
+        }
+    }
+    for section in config.sections():
+        canonical_section = section.lower()
+        section_items = dict(config._sections[section])
+        section_items.pop('__name__', None)
+        autoload = section_items.get('autoload_config_file')
+        # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+        if isinstance(autoload, str) and autoload.lower().endswith('.ini'):
+            section_items['autoload_config_file'] = autoload[:-4] + '.toml'
+        section_template = template_data.get(canonical_section, {})
+        data[canonical_section] = {
+            key: _coerce_legacy_value(value, section_template.get(key))
+            for key, value in section_items.items()
+        }
+    toml_path.parent.mkdir(parents=True, exist_ok=True)
+    dump_toml_file(toml_path, data)

labscript_profile/create.py

@@ -1,16 +1,31 @@
 import sys
 import os
 import shutil
-import configparser
 from pathlib import Path
 from subprocess import check_output
-from labscript_profile import LABSCRIPT_SUITE_PROFILE, default_labconfig_path
+import h5py
+from labscript_profile import (
+    LABSCRIPT_SUITE_PROFILE,
+    default_labconfig_path,
+    legacy_labconfig_path,
+)
 import argparse
+from labscript_profile.toml_config import dump_toml_file, load_toml_file
 
 _here = os.path.dirname(os.path.abspath(__file__))
 DEFAULT_PROFILE_CONTENTS = os.path.join(_here, 'default_profile')
 
 
+def _replace_backslashes(value):
+    if isinstance(value, dict):
+        return {key: _replace_backslashes(child) for key, child in value.items()}
+    if isinstance(value, list):
+        return [_replace_backslashes(child) for child in value]
+    if isinstance(value, str):
+        return value.replace('\\', os.path.sep)
+    return value
+
+
 def make_shared_secret(directory):
     """Create a new zprocess shared secret file in the given directory and return its
     filepath"""
@@ -31,37 +46,34 @@ def make_labconfig_file(apparatus_name = None):
         Overrides the default apparatus name with the provided one if not None
     """
 
-    source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.ini')
+    source_path = os.path.join(LABSCRIPT_SUITE_PROFILE, 'labconfig', 'example.toml')
     target_path = default_labconfig_path()
-    if os.path.exists(target_path):
+    # LEGACY INI COMPATIBILITY. DEPRECATED CODE, WILL BE REMOVED.
+    legacy_path = legacy_labconfig_path()
+    if os.path.exists(target_path) or (legacy_path is not None and os.path.exists(legacy_path)):
         raise FileExistsError(target_path)
-    with open(source_path) as infile, open(target_path, 'w') as outfile:
-        data = infile.read()
-        data = data.replace('\\', os.path.sep)
-        outfile.write(data)
-
-    # Now change some things about it:
-    config = configparser.ConfigParser(interpolation=None)
-    config.read(target_path)
+    config = load_toml_file(source_path)
+    if os.path.sep != '\\':
+        config = _replace_backslashes(config)
     if sys.platform == 'linux':
-        config.set('programs', 'text_editor', 'gedit')
+        config['programs']['text_editor'] = 'gedit'
     elif sys.platform == 'darwin':
-        config.set('programs', 'text_editor', 'open')
-        config.set('programs', 'text_editor_arguments', '-a TextEdit {file}')
+        config['programs']['text_editor'] = 'open'
+        config['programs']['text_editor_arguments'] = '-a TextEdit {file}'
     if sys.platform != 'win32':
-        config.set('programs', 'hdf5_viewer', 'hdfview')
-        config.set('DEFAULT', 'shared_drive', '$HOME/labscript_shared')
+        config['programs']['hdf5_viewer'] = 'hdfview'
+        config['default']['shared_drive'] = '$HOME/labscript_shared'
     shared_secret = make_shared_secret(target_path.parent)
     shared_secret_entry = Path(
         '%(labscript_suite)s', shared_secret.relative_to(LABSCRIPT_SUITE_PROFILE)
     )
-    config.set('security', 'shared_secret', str(shared_secret_entry))
+    config['security']['shared_secret'] = str(shared_secret_entry)
     if apparatus_name is not None:
         print(f'\tSetting apparatus name to \'{apparatus_name}\'')
-        config.set('DEFAULT', 'apparatus_name', apparatus_name)
+        config['default']['apparatus_name'] = apparatus_name
 
-    with open(target_path, 'w') as f:
-        config.write(f)
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+    dump_toml_file(target_path, config)
 
 def compile_connection_table():
     """Compile the connection table defined in the labconfig file
@@ -75,17 +87,19 @@ def compile_connection_table():
         # if runmanager doesn't import, skip compilation
         return
 
-    config = configparser.ConfigParser(defaults = {'labscript_suite': str(LABSCRIPT_SUITE_PROFILE)})
-    config.read(default_labconfig_path())
+    from labscript_utils.labconfig import LabConfig
+
+    config = LabConfig()
 
     # The path to the user's connection_table.py script
     script_path = os.path.expandvars(config['paths']['connection_table_py'])
     # path to the connection_table.h5 destination
     output_h5_path = os.path.expandvars(config['paths']['connection_table_h5'])
     # create output directory, if needed
     Path(output_h5_path).parent.mkdir(parents=True, exist_ok=True)
-    # compile the h5 file
-    runmanager.new_globals_file(output_h5_path)
+    # Create a fresh HDF5 target for the compile output.
+    with h5py.File(output_h5_path, 'w'):
+        pass
 
     def dummy_callback(success):
         pass
@@ -164,4 +178,4 @@ def create_profile(apparatus_name = None, compile_table = False):
 
     if compile_table:
         print('Compiling the example connection table')
-        compile_connection_table()
+        compile_connection_table()