daemon: added capability to config services to shadow directory structures, from a given path, or from a local source, files may be templates or a straight copy and can be sourced from node named unique paths for node specific files, also refactored and renamed file creation related functions for nodes

daemon/core/configservice/base.py

@@ -3,8 +3,9 @@ import enum
 import inspect
 import logging
 import time
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from mako import exceptions
 from mako.lookup import TemplateLookup
@@ -28,6 +29,14 @@ class ConfigServiceBootError(Exception):
     pass
 
 
+@dataclass
+class ShadowDir:
+    path: str
+    src: Optional[str] = None
+    templates: bool = False
+    has_node_paths: bool = False
+
+
 class ConfigService(abc.ABC):
     """
     Base class for creating configurable services.
@@ -39,6 +48,9 @@ class ConfigService(abc.ABC):
     # time to wait in seconds for determining if service started successfully
     validation_timer: int = 5
 
+    # directories to shadow and copy files from
+    shadow_directories: List[ShadowDir] = []
+
     def __init__(self, node: CoreNode) -> None:
         """
         Create ConfigService instance.
@@ -135,6 +147,7 @@ class ConfigService(abc.ABC):
         :raises ConfigServiceBootError: when there is an error starting service
         """
         logger.info("node(%s) service(%s) starting...", self.node.name, self.name)
+        self.create_shadow_dirs()
         self.create_dirs()
         self.create_files()
         wait = self.validation_mode == ConfigServiceMode.BLOCKING
@@ -169,6 +182,64 @@ class ConfigService(abc.ABC):
         self.stop()
         self.start()
 
+    def create_shadow_dirs(self) -> None:
+        """
+        Creates a shadow of a host system directory recursively
+        to be mapped and live within a node.
+
+        :return: nothing
+        :raises CoreError: when there is a failure creating a directory or file
+        """
+        for shadow_dir in self.shadow_directories:
+            # setup shadow and src paths, using node unique paths when configured
+            shadow_path = Path(shadow_dir.path)
+            if shadow_dir.src is None:
+                src_path = shadow_path
+            else:
+                src_path = Path(shadow_dir.src)
+            if shadow_dir.has_node_paths:
+                src_path = src_path / self.node.name
+            # validate shadow and src paths
+            if not shadow_path.is_absolute():
+                raise CoreError(f"shadow dir({shadow_path}) is not absolute")
+            if not src_path.is_absolute():
+                raise CoreError(f"shadow source dir({src_path}) is not absolute")
+            if not src_path.is_dir():
+                raise CoreError(f"shadow source dir({src_path}) does not exist")
+            # create root of the shadow path within node
+            logger.info(
+                "node(%s) creating shadow directory(%s) src(%s) node paths(%s) "
+                "templates(%s)",
+                self.node.name,
+                shadow_path,
+                src_path,
+                shadow_dir.has_node_paths,
+                shadow_dir.templates,
+            )
+            self.node.create_dir(shadow_path)
+            # find all directories and files to create
+            dir_paths = []
+            file_paths = []
+            for path in src_path.rglob("*"):
+                shadow_src_path = shadow_path / path.relative_to(src_path)
+                if path.is_dir():
+                    dir_paths.append(shadow_src_path)
+                else:
+                    file_paths.append((path, shadow_src_path))
+            # create all directories within node
+            for path in dir_paths:
+                self.node.create_dir(path)
+            # create all files within node, from templates when configured
+            data = self.data()
+            templates = TemplateLookup(directories=src_path)
+            for path, dst_path in file_paths:
+                if shadow_dir.templates:
+                    template = templates.get_template(path.name)
+                    rendered = self._render(template, data)
+                    self.node.create_file(dst_path, rendered)
+                else:
+                    self.node.copy_file(path, dst_path)
+
     def create_dirs(self) -> None:
         """
         Creates directories for service.
@@ -176,10 +247,11 @@ class ConfigService(abc.ABC):
         :return: nothing
         :raises CoreError: when there is a failure creating a directory
         """
-        for directory in self.directories:
+        logger.debug("creating config service directories")
+        for directory in sorted(self.directories):
             dir_path = Path(directory)
             try:
-                self.node.privatedir(dir_path)
+                self.node.create_dir(dir_path)
             except (CoreCommandError, CoreError):
                 raise CoreError(
                     f"node({self.node.name}) service({self.name}) "
@@ -221,17 +293,21 @@ class ConfigService(abc.ABC):
         :return: mapping of files to templates
         """
         templates = {}
-        for name in self.files:
+        for file in self.files:
-            basename = Path(name).name
+            file_path = Path(file)
-            if name in self.custom_templates:
+            if file_path.is_absolute():
-                template = self.custom_templates[name]
+                template_path = str(file_path.relative_to("/"))
-                template = self.clean_text(template)
-            elif self.templates.has_template(basename):
-                template = self.templates.get_template(basename).source
             else:
-                template = self.get_text_template(name)
+                template_path = str(file_path)
+            if file in self.custom_templates:
+                template = self.custom_templates[file]
                 template = self.clean_text(template)
-            templates[name] = template
+            elif self.templates.has_template(template_path):
+                template = self.templates.get_template(template_path).source
+            else:
+                template = self.get_text_template(file)
+                template = self.clean_text(template)
+            templates[file] = template
         return templates
 
     def create_files(self) -> None:
@@ -241,24 +317,20 @@ class ConfigService(abc.ABC):
         :return: nothing
         """
         data = self.data()
-        for name in self.files:
+        for file in sorted(self.files):
-            file_path = Path(name)
+            logger.debug(
-            if name in self.custom_templates:
+                "node(%s) service(%s) template(%s)", self.node.name, self.name, file
-                text = self.custom_templates[name]
+            )
+            file_path = Path(file)
+            if file in self.custom_templates:
+                text = self.custom_templates[file]
                 rendered = self.render_text(text, data)
             elif self.templates.has_template(file_path.name):
                 rendered = self.render_template(file_path.name, data)
             else:
-                text = self.get_text_template(name)
+                text = self.get_text_template(file)
                 rendered = self.render_text(text, data)
-            logger.debug(
+            self.node.create_file(file_path, rendered)
-                "node(%s) service(%s) template(%s): \n%s",
-                self.node.name,
-                self.name,
-                name,
-                rendered,
-            )
-            self.node.nodefile(file_path, rendered)
 
     def run_startup(self, wait: bool) -> None:
         """

daemon/core/configservices/simpleservice.py

@@ -1,49 +0,0 @@
-from typing import Dict, List
-
-from core.config import Configuration
-from core.configservice.base import ConfigService, ConfigServiceMode
-from core.emulator.enumerations import ConfigDataTypes
-
-
-class SimpleService(ConfigService):
-    name: str = "Simple"
-    group: str = "SimpleGroup"
-    directories: List[str] = ["/etc/quagga", "/usr/local/lib"]
-    files: List[str] = ["test1.sh", "test2.sh"]
-    executables: List[str] = []
-    dependencies: List[str] = []
-    startup: List[str] = []
-    validate: List[str] = []
-    shutdown: List[str] = []
-    validation_mode: ConfigServiceMode = ConfigServiceMode.BLOCKING
-    default_configs: List[Configuration] = [
-        Configuration(id="value1", type=ConfigDataTypes.STRING, label="Text"),
-        Configuration(id="value2", type=ConfigDataTypes.BOOL, label="Boolean"),
-        Configuration(
-            id="value3",
-            type=ConfigDataTypes.STRING,
-            label="Multiple Choice",
-            options=["value1", "value2", "value3"],
-        ),
-    ]
-    modes: Dict[str, Dict[str, str]] = {
-        "mode1": {"value1": "value1", "value2": "0", "value3": "value2"},
-        "mode2": {"value1": "value2", "value2": "1", "value3": "value3"},
-        "mode3": {"value1": "value3", "value2": "0", "value3": "value1"},
-    }
-
-    def get_text_template(self, name: str) -> str:
-        if name == "test1.sh":
-            return """
-            # sample script 1
-            # node id(${node.id}) name(${node.name})
-            # config: ${config}
-            echo hello
-            """
-        elif name == "test2.sh":
-            return """
-            # sample script 2
-            # node id(${node.id}) name(${node.name})
-            # config: ${config}
-            echo hello2
-            """

daemon/core/emulator/session.py

@@ -690,11 +690,11 @@ class Session:
         :param data: file data
         :return: nothing
         """
-        node = self.get_node(node_id, CoreNodeBase)
+        node = self.get_node(node_id, CoreNode)
         if src_path is not None:
             node.addfile(src_path, file_path)
         elif data is not None:
-            node.nodefile(file_path, data)
+            node.create_file(file_path, data)
 
     def clear(self) -> None:
         """

daemon/core/nodes/base.py

@@ -237,7 +237,17 @@ class CoreNodeBase(NodeBase):
         raise NotImplementedError
 
     @abc.abstractmethod
-    def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
+    def create_dir(self, dir_path: Path) -> None:
+        """
+        Create a node private directory.
+
+        :param dir_path: path to create
+        :return: nothing
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
         """
         Create a node file with a given mode.
 
@@ -248,6 +258,19 @@ class CoreNodeBase(NodeBase):
         """
         raise NotImplementedError
 
+    @abc.abstractmethod
+    def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
+        """
+        Copy source file to node host destination, updating the file mode when
+        provided.
+
+        :param src_path: source file to copy
+        :param dst_path: node host destination
+        :param mode: file mode
+        :return: nothing
+        """
+        raise NotImplementedError
+
     @abc.abstractmethod
     def addfile(self, src_path: Path, file_path: Path) -> None:
         """
@@ -567,7 +590,7 @@ class CoreNode(CoreNodeBase):
 
             # create private directories
             for dir_path in PRIVATE_DIRS:
-                self.privatedir(dir_path)
+                self.create_dir(dir_path)
 
     def shutdown(self) -> None:
         """
@@ -648,19 +671,23 @@ class CoreNode(CoreNodeBase):
         else:
             return f"ssh -X -f {self.server.host} xterm -e {terminal}"
 
-    def privatedir(self, dir_path: Path) -> None:
+    def create_dir(self, dir_path: Path) -> None:
         """
-        Create a private directory.
+        Create a node private directory.
 
         :param dir_path: path to create
         :return: nothing
         """
-        logger.info("creating private directory: %s", dir_path)
+        if not dir_path.is_absolute():
-        if not str(dir_path).startswith("/"):
             raise CoreError(f"private directory path not fully qualified: {dir_path}")
-        host_path = self.host_path(dir_path, is_dir=True)
+        logger.debug("node(%s) creating private directory: %s", self.name, dir_path)
-        self.host_cmd(f"mkdir -p {host_path}")
+        parent_path = self._find_parent_path(dir_path)
-        self.mount(host_path, dir_path)
+        if parent_path:
+            self.host_cmd(f"mkdir -p {parent_path}")
+        else:
+            host_path = self.host_path(dir_path, is_dir=True)
+            self.host_cmd(f"mkdir -p {host_path}")
+            self.mount(host_path, dir_path)
 
     def mount(self, src_path: Path, target_path: Path) -> None:
         """
@@ -880,16 +907,40 @@ class CoreNode(CoreNodeBase):
             self.host_cmd(f"mkdir -p {directory}")
             self.server.remote_put(src_path, file_path)
 
-    def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
+    def _find_parent_path(self, path: Path) -> Optional[Path]:
         """
-        Create a node file with a given mode.
+        Check if there is an existing mounted parent directory created for this node.
 
-        :param file_path: name of file to create
+        :param path: existing parent path to use
+        :return: exist parent path if exists, None otherwise
+        """
+        logger.debug("looking for existing parent: %s", path)
+        existing_path = None
+        for parent in path.parents:
+            node_path = self.host_path(parent, is_dir=True)
+            if node_path == self.directory:
+                break
+            if self.path_exists(str(node_path)):
+                relative_path = path.relative_to(parent)
+                existing_path = node_path / relative_path
+                break
+        return existing_path
+
+    def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
+        """
+        Create file within a node at the given path, using contents and mode.
+
+        :param file_path: desired path for file
         :param contents: contents of file
-        :param mode: mode for file
+        :param mode: mode to create file with
         :return: nothing
         """
-        host_path = self.host_path(file_path)
+        logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
+        host_path = self._find_parent_path(file_path)
+        if host_path:
+            self.host_cmd(f"mkdir -p {host_path.parent}")
+        else:
+            host_path = self.host_path(file_path)
         directory = host_path.parent
         if self.server is None:
             if not directory.exists():
@@ -901,26 +952,35 @@ class CoreNode(CoreNodeBase):
             self.host_cmd(f"mkdir -m {0o755:o} -p {directory}")
             self.server.remote_put_temp(host_path, contents)
             self.host_cmd(f"chmod {mode:o} {host_path}")
-        logger.debug("node(%s) added file: %s; mode: 0%o", self.name, host_path, mode)
 
-    def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
+    def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
         """
-        Copy a file to a node, following symlinks and preserving metadata.
+        Copy source file to node host destination, updating the file mode when
-        Change file mode if specified.
+        provided.
 
-        :param file_path: file name to copy file to
+        :param src_path: source file to copy
-        :param src_path: file to copy
+        :param dst_path: node host destination
-        :param mode: mode to copy to
+        :param mode: file mode
         :return: nothing
         """
-        host_path = self.host_path(file_path)
+        logger.debug(
+            "node(%s) copying file src(%s) to dst(%s) mode(%o)",
+            self.name,
+            src_path,
+            dst_path,
+            mode or 0,
+        )
+        host_path = self._find_parent_path(dst_path)
+        if host_path:
+            self.host_cmd(f"mkdir -p {host_path.parent}")
+        else:
+            host_path = self.host_path(dst_path)
         if self.server is None:
             shutil.copy2(src_path, host_path)
         else:
             self.server.remote_put(src_path, host_path)
         if mode is not None:
             self.host_cmd(f"chmod {mode:o} {host_path}")
-        logger.info("node(%s) copied file: %s; mode: %s", self.name, host_path, mode)
 
 
 class CoreNetworkBase(NodeBase):

daemon/core/nodes/docker.py

@@ -164,7 +164,7 @@ class DockerNode(CoreNode):
         """
         return f"docker exec -it {self.name} bash"
 
-    def privatedir(self, dir_path: str) -> None:
+    def create_dir(self, dir_path: Path) -> None:
         """
         Create a private directory.
 
@@ -187,7 +187,7 @@ class DockerNode(CoreNode):
         logger.debug("mounting source(%s) target(%s)", src_path, target_path)
         raise Exception("not supported")
 
-    def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
+    def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
         """
         Create a node file with a given mode.
 
@@ -196,7 +196,7 @@ class DockerNode(CoreNode):
         :param mode: mode for file
         :return: nothing
         """
-        logger.debug("nodefile filename(%s) mode(%s)", file_path, mode)
+        logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
         temp = NamedTemporaryFile(delete=False)
         temp.write(contents.encode("utf-8"))
         temp.close()
@@ -211,26 +211,26 @@ class DockerNode(CoreNode):
         if self.server is not None:
             self.host_cmd(f"rm -f {temp_path}")
         temp_path.unlink()
-        logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
 
-    def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
+    def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
         """
         Copy a file to a node, following symlinks and preserving metadata.
         Change file mode if specified.
 
-        :param file_path: file name to copy file to
+        :param dst_path: file name to copy file to
         :param src_path: file to copy
         :param mode: mode to copy to
         :return: nothing
         """
         logger.info(
-            "node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
+            "node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0
         )
-        self.cmd(f"mkdir -p {file_path.parent}")
+        self.cmd(f"mkdir -p {dst_path.parent}")
         if self.server:
             temp = NamedTemporaryFile(delete=False)
             temp_path = Path(temp.name)
             src_path = temp_path
             self.server.remote_put(src_path, temp_path)
-        self.client.copy_file(src_path, file_path)
+        self.client.copy_file(src_path, dst_path)
-        self.cmd(f"chmod {mode:o} {file_path}")
+        if mode is not None:
+            self.cmd(f"chmod {mode:o} {dst_path}")

daemon/core/nodes/lxd.py

@@ -140,7 +140,7 @@ class LxcNode(CoreNode):
         """
         return f"lxc exec {self.name} -- {sh}"
 
-    def privatedir(self, dir_path: Path) -> None:
+    def create_dir(self, dir_path: Path) -> None:
         """
         Create a private directory.
 
@@ -163,7 +163,7 @@ class LxcNode(CoreNode):
         logger.debug("mounting source(%s) target(%s)", src_path, target_path)
         raise Exception("not supported")
 
-    def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
+    def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
         """
         Create a node file with a given mode.
 
@@ -172,7 +172,7 @@ class LxcNode(CoreNode):
         :param mode: mode for file
         :return: nothing
         """
-        logger.debug("nodefile filename(%s) mode(%s)", file_path, mode)
+        logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
         temp = NamedTemporaryFile(delete=False)
         temp.write(contents.encode("utf-8"))
         temp.close()
@@ -189,27 +189,28 @@ class LxcNode(CoreNode):
         temp_path.unlink()
         logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
 
-    def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
+    def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
         """
         Copy a file to a node, following symlinks and preserving metadata.
         Change file mode if specified.
 
-        :param file_path: file name to copy file to
+        :param dst_path: file name to copy file to
         :param src_path: file to copy
         :param mode: mode to copy to
         :return: nothing
         """
         logger.info(
-            "node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
+            "node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0
         )
-        self.cmd(f"mkdir -p {file_path.parent}")
+        self.cmd(f"mkdir -p {dst_path.parent}")
         if self.server:
             temp = NamedTemporaryFile(delete=False)
             temp_path = Path(temp.name)
             src_path = temp_path
             self.server.remote_put(src_path, temp_path)
-        self.client.copy_file(src_path, file_path)
+        self.client.copy_file(src_path, dst_path)
-        self.cmd(f"chmod {mode:o} {file_path}")
+        if mode is not None:
+            self.cmd(f"chmod {mode:o} {dst_path}")
 
     def add_iface(self, iface: CoreInterface, iface_id: int) -> None:
         super().add_iface(iface, iface_id)

daemon/core/services/coreservices.py

@@ -492,7 +492,7 @@ class CoreServices:
         for directory in service.dirs:
             dir_path = Path(directory)
             try:
-                node.privatedir(dir_path)
+                node.create_dir(dir_path)
             except (CoreCommandError, CoreError) as e:
                 logger.warning(
                     "error mounting private dir '%s' for service '%s': %s",
@@ -553,7 +553,7 @@ class CoreServices:
             src = src.split("\n")[0]
             src = utils.expand_corepath(src, node.session, node)
             # TODO: glob here
-            node.nodefilecopy(file_path, src, mode=0o644)
+            node.copy_file(src, file_path, mode=0o644)
             return True
         return False
 
@@ -750,7 +750,7 @@ class CoreServices:
                     continue
             else:
                 cfg = service.generate_config(node, file_name)
-            node.nodefile(file_path, cfg)
+            node.create_file(file_path, cfg)
 
     def service_reconfigure(self, node: CoreNode, service: "CoreService") -> None:
         """
@@ -771,7 +771,7 @@ class CoreServices:
             cfg = service.config_data.get(file_name)
             if cfg is None:
                 cfg = service.generate_config(node, file_name)
-            node.nodefile(file_path, cfg)
+            node.create_file(file_path, cfg)
 
 
 class CoreService:

daemon/tests/conftest.py

@@ -60,7 +60,7 @@ def patcher(request):
         patch_manager.patch_obj(
             LinuxNetClient, "get_mac", return_value="00:00:00:00:00:00"
         )
-        patch_manager.patch_obj(CoreNode, "nodefile")
+        patch_manager.patch_obj(CoreNode, "create_file")
         patch_manager.patch_obj(Session, "write_state")
         patch_manager.patch_obj(Session, "write_nodes")
     yield patch_manager

daemon/tests/test_config_services.py

@@ -70,7 +70,7 @@ class TestConfigServices:
 
         # then
         directory = Path(MyService.directories[0])
-        node.privatedir.assert_called_with(directory)
+        node.create_dir.assert_called_with(directory)
 
     def test_create_files_custom(self):
         # given
@@ -84,7 +84,7 @@ class TestConfigServices:
 
         # then
         file_path = Path(MyService.files[0])
-        node.nodefile.assert_called_with(file_path, text)
+        node.create_file.assert_called_with(file_path, text)
 
     def test_create_files_text(self):
         # given
@@ -96,7 +96,7 @@ class TestConfigServices:
 
         # then
         file_path = Path(MyService.files[0])
-        node.nodefile.assert_called_with(file_path, TEMPLATE_TEXT)
+        node.create_file.assert_called_with(file_path, TEMPLATE_TEXT)
 
     def test_run_startup(self):
         # given

daemon/tests/test_gui.py

@@ -441,10 +441,8 @@ class TestGui:
         coretlv.handle_message(message)
 
         if not request.config.getoption("mock"):
-            directory = str(file_path.parent)
+            expected_path = node.directory / "var.log/test" / file_path.name
-            created_directory = directory[1:].replace("/", ".")
+            assert expected_path.exists()
-            create_path = node.directory / created_directory / file_path.name
-            assert create_path.exists()
 
     def test_exec_node_tty(self, coretlv: CoreHandler):
         coretlv.dispatch_replies = mock.MagicMock()