From bd896d1336cc67dbb8a5d4e6a17d8847e6c5e18d Mon Sep 17 00:00:00 2001 From: Blake Harnden <32446120+bharnden@users.noreply.github.com> Date: Fri, 17 Sep 2021 14:34:37 -0700 Subject: [PATCH] daemon: added capability to config services to shadow directory structures, from a given path, or from a local source, files may be templates or a straight copy and can be sourced from node named unique paths for node specific files, also refactored and renamed file creation related functions for nodes --- daemon/core/configservice/base.py | 122 ++++++++++++++++---- daemon/core/configservices/simpleservice.py | 49 -------- daemon/core/emulator/session.py | 4 +- daemon/core/nodes/base.py | 106 +++++++++++++---- daemon/core/nodes/docker.py | 20 ++-- daemon/core/nodes/lxd.py | 19 +-- daemon/core/services/coreservices.py | 8 +- daemon/tests/conftest.py | 2 +- daemon/tests/test_config_services.py | 6 +- daemon/tests/test_gui.py | 6 +- 10 files changed, 212 insertions(+), 130 deletions(-) delete mode 100644 daemon/core/configservices/simpleservice.py diff --git a/daemon/core/configservice/base.py b/daemon/core/configservice/base.py index 64a5dd03..386ab26d 100644 --- a/daemon/core/configservice/base.py +++ b/daemon/core/configservice/base.py @@ -3,8 +3,9 @@ import enum import inspect import logging import time +from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from mako import exceptions from mako.lookup import TemplateLookup @@ -28,6 +29,14 @@ class ConfigServiceBootError(Exception): pass +@dataclass +class ShadowDir: + path: str + src: Optional[str] = None + templates: bool = False + has_node_paths: bool = False + + class ConfigService(abc.ABC): """ Base class for creating configurable services. @@ -39,6 +48,9 @@ class ConfigService(abc.ABC): # time to wait in seconds for determining if service started successfully validation_timer: int = 5 + # directories to shadow and copy files from + shadow_directories: List[ShadowDir] = [] + def __init__(self, node: CoreNode) -> None: """ Create ConfigService instance. @@ -135,6 +147,7 @@ class ConfigService(abc.ABC): :raises ConfigServiceBootError: when there is an error starting service """ logger.info("node(%s) service(%s) starting...", self.node.name, self.name) + self.create_shadow_dirs() self.create_dirs() self.create_files() wait = self.validation_mode == ConfigServiceMode.BLOCKING @@ -169,6 +182,64 @@ class ConfigService(abc.ABC): self.stop() self.start() + def create_shadow_dirs(self) -> None: + """ + Creates a shadow of a host system directory recursively + to be mapped and live within a node. + + :return: nothing + :raises CoreError: when there is a failure creating a directory or file + """ + for shadow_dir in self.shadow_directories: + # setup shadow and src paths, using node unique paths when configured + shadow_path = Path(shadow_dir.path) + if shadow_dir.src is None: + src_path = shadow_path + else: + src_path = Path(shadow_dir.src) + if shadow_dir.has_node_paths: + src_path = src_path / self.node.name + # validate shadow and src paths + if not shadow_path.is_absolute(): + raise CoreError(f"shadow dir({shadow_path}) is not absolute") + if not src_path.is_absolute(): + raise CoreError(f"shadow source dir({src_path}) is not absolute") + if not src_path.is_dir(): + raise CoreError(f"shadow source dir({src_path}) does not exist") + # create root of the shadow path within node + logger.info( + "node(%s) creating shadow directory(%s) src(%s) node paths(%s) " + "templates(%s)", + self.node.name, + shadow_path, + src_path, + shadow_dir.has_node_paths, + shadow_dir.templates, + ) + self.node.create_dir(shadow_path) + # find all directories and files to create + dir_paths = [] + file_paths = [] + for path in src_path.rglob("*"): + shadow_src_path = shadow_path / path.relative_to(src_path) + if path.is_dir(): + dir_paths.append(shadow_src_path) + else: + file_paths.append((path, shadow_src_path)) + # create all directories within node + for path in dir_paths: + self.node.create_dir(path) + # create all files within node, from templates when configured + data = self.data() + templates = TemplateLookup(directories=src_path) + for path, dst_path in file_paths: + if shadow_dir.templates: + template = templates.get_template(path.name) + rendered = self._render(template, data) + self.node.create_file(dst_path, rendered) + else: + self.node.copy_file(path, dst_path) + def create_dirs(self) -> None: """ Creates directories for service. @@ -176,10 +247,11 @@ class ConfigService(abc.ABC): :return: nothing :raises CoreError: when there is a failure creating a directory """ - for directory in self.directories: + logger.debug("creating config service directories") + for directory in sorted(self.directories): dir_path = Path(directory) try: - self.node.privatedir(dir_path) + self.node.create_dir(dir_path) except (CoreCommandError, CoreError): raise CoreError( f"node({self.node.name}) service({self.name}) " @@ -221,17 +293,21 @@ class ConfigService(abc.ABC): :return: mapping of files to templates """ templates = {} - for name in self.files: - basename = Path(name).name - if name in self.custom_templates: - template = self.custom_templates[name] - template = self.clean_text(template) - elif self.templates.has_template(basename): - template = self.templates.get_template(basename).source + for file in self.files: + file_path = Path(file) + if file_path.is_absolute(): + template_path = str(file_path.relative_to("/")) else: - template = self.get_text_template(name) + template_path = str(file_path) + if file in self.custom_templates: + template = self.custom_templates[file] template = self.clean_text(template) - templates[name] = template + elif self.templates.has_template(template_path): + template = self.templates.get_template(template_path).source + else: + template = self.get_text_template(file) + template = self.clean_text(template) + templates[file] = template return templates def create_files(self) -> None: @@ -241,24 +317,20 @@ class ConfigService(abc.ABC): :return: nothing """ data = self.data() - for name in self.files: - file_path = Path(name) - if name in self.custom_templates: - text = self.custom_templates[name] + for file in sorted(self.files): + logger.debug( + "node(%s) service(%s) template(%s)", self.node.name, self.name, file + ) + file_path = Path(file) + if file in self.custom_templates: + text = self.custom_templates[file] rendered = self.render_text(text, data) elif self.templates.has_template(file_path.name): rendered = self.render_template(file_path.name, data) else: - text = self.get_text_template(name) + text = self.get_text_template(file) rendered = self.render_text(text, data) - logger.debug( - "node(%s) service(%s) template(%s): \n%s", - self.node.name, - self.name, - name, - rendered, - ) - self.node.nodefile(file_path, rendered) + self.node.create_file(file_path, rendered) def run_startup(self, wait: bool) -> None: """ diff --git a/daemon/core/configservices/simpleservice.py b/daemon/core/configservices/simpleservice.py deleted file mode 100644 index 4370977d..00000000 --- a/daemon/core/configservices/simpleservice.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Dict, List - -from core.config import Configuration -from core.configservice.base import ConfigService, ConfigServiceMode -from core.emulator.enumerations import ConfigDataTypes - - -class SimpleService(ConfigService): - name: str = "Simple" - group: str = "SimpleGroup" - directories: List[str] = ["/etc/quagga", "/usr/local/lib"] - files: List[str] = ["test1.sh", "test2.sh"] - executables: List[str] = [] - dependencies: List[str] = [] - startup: List[str] = [] - validate: List[str] = [] - shutdown: List[str] = [] - validation_mode: ConfigServiceMode = ConfigServiceMode.BLOCKING - default_configs: List[Configuration] = [ - Configuration(id="value1", type=ConfigDataTypes.STRING, label="Text"), - Configuration(id="value2", type=ConfigDataTypes.BOOL, label="Boolean"), - Configuration( - id="value3", - type=ConfigDataTypes.STRING, - label="Multiple Choice", - options=["value1", "value2", "value3"], - ), - ] - modes: Dict[str, Dict[str, str]] = { - "mode1": {"value1": "value1", "value2": "0", "value3": "value2"}, - "mode2": {"value1": "value2", "value2": "1", "value3": "value3"}, - "mode3": {"value1": "value3", "value2": "0", "value3": "value1"}, - } - - def get_text_template(self, name: str) -> str: - if name == "test1.sh": - return """ - # sample script 1 - # node id(${node.id}) name(${node.name}) - # config: ${config} - echo hello - """ - elif name == "test2.sh": - return """ - # sample script 2 - # node id(${node.id}) name(${node.name}) - # config: ${config} - echo hello2 - """ diff --git a/daemon/core/emulator/session.py b/daemon/core/emulator/session.py index 72161717..6dad8e2d 100644 --- a/daemon/core/emulator/session.py +++ b/daemon/core/emulator/session.py @@ -690,11 +690,11 @@ class Session: :param data: file data :return: nothing """ - node = self.get_node(node_id, CoreNodeBase) + node = self.get_node(node_id, CoreNode) if src_path is not None: node.addfile(src_path, file_path) elif data is not None: - node.nodefile(file_path, data) + node.create_file(file_path, data) def clear(self) -> None: """ diff --git a/daemon/core/nodes/base.py b/daemon/core/nodes/base.py index 960ec056..2ec7fb7f 100644 --- a/daemon/core/nodes/base.py +++ b/daemon/core/nodes/base.py @@ -237,7 +237,17 @@ class CoreNodeBase(NodeBase): raise NotImplementedError @abc.abstractmethod - def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None: + def create_dir(self, dir_path: Path) -> None: + """ + Create a node private directory. + + :param dir_path: path to create + :return: nothing + """ + raise NotImplementedError + + @abc.abstractmethod + def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None: """ Create a node file with a given mode. @@ -248,6 +258,19 @@ class CoreNodeBase(NodeBase): """ raise NotImplementedError + @abc.abstractmethod + def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None: + """ + Copy source file to node host destination, updating the file mode when + provided. + + :param src_path: source file to copy + :param dst_path: node host destination + :param mode: file mode + :return: nothing + """ + raise NotImplementedError + @abc.abstractmethod def addfile(self, src_path: Path, file_path: Path) -> None: """ @@ -567,7 +590,7 @@ class CoreNode(CoreNodeBase): # create private directories for dir_path in PRIVATE_DIRS: - self.privatedir(dir_path) + self.create_dir(dir_path) def shutdown(self) -> None: """ @@ -648,19 +671,23 @@ class CoreNode(CoreNodeBase): else: return f"ssh -X -f {self.server.host} xterm -e {terminal}" - def privatedir(self, dir_path: Path) -> None: + def create_dir(self, dir_path: Path) -> None: """ - Create a private directory. + Create a node private directory. :param dir_path: path to create :return: nothing """ - logger.info("creating private directory: %s", dir_path) - if not str(dir_path).startswith("/"): + if not dir_path.is_absolute(): raise CoreError(f"private directory path not fully qualified: {dir_path}") - host_path = self.host_path(dir_path, is_dir=True) - self.host_cmd(f"mkdir -p {host_path}") - self.mount(host_path, dir_path) + logger.debug("node(%s) creating private directory: %s", self.name, dir_path) + parent_path = self._find_parent_path(dir_path) + if parent_path: + self.host_cmd(f"mkdir -p {parent_path}") + else: + host_path = self.host_path(dir_path, is_dir=True) + self.host_cmd(f"mkdir -p {host_path}") + self.mount(host_path, dir_path) def mount(self, src_path: Path, target_path: Path) -> None: """ @@ -880,16 +907,40 @@ class CoreNode(CoreNodeBase): self.host_cmd(f"mkdir -p {directory}") self.server.remote_put(src_path, file_path) - def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None: + def _find_parent_path(self, path: Path) -> Optional[Path]: """ - Create a node file with a given mode. + Check if there is an existing mounted parent directory created for this node. - :param file_path: name of file to create + :param path: existing parent path to use + :return: exist parent path if exists, None otherwise + """ + logger.debug("looking for existing parent: %s", path) + existing_path = None + for parent in path.parents: + node_path = self.host_path(parent, is_dir=True) + if node_path == self.directory: + break + if self.path_exists(str(node_path)): + relative_path = path.relative_to(parent) + existing_path = node_path / relative_path + break + return existing_path + + def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None: + """ + Create file within a node at the given path, using contents and mode. + + :param file_path: desired path for file :param contents: contents of file - :param mode: mode for file + :param mode: mode to create file with :return: nothing """ - host_path = self.host_path(file_path) + logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode) + host_path = self._find_parent_path(file_path) + if host_path: + self.host_cmd(f"mkdir -p {host_path.parent}") + else: + host_path = self.host_path(file_path) directory = host_path.parent if self.server is None: if not directory.exists(): @@ -901,26 +952,35 @@ class CoreNode(CoreNodeBase): self.host_cmd(f"mkdir -m {0o755:o} -p {directory}") self.server.remote_put_temp(host_path, contents) self.host_cmd(f"chmod {mode:o} {host_path}") - logger.debug("node(%s) added file: %s; mode: 0%o", self.name, host_path, mode) - def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None: + def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None: """ - Copy a file to a node, following symlinks and preserving metadata. - Change file mode if specified. + Copy source file to node host destination, updating the file mode when + provided. - :param file_path: file name to copy file to - :param src_path: file to copy - :param mode: mode to copy to + :param src_path: source file to copy + :param dst_path: node host destination + :param mode: file mode :return: nothing """ - host_path = self.host_path(file_path) + logger.debug( + "node(%s) copying file src(%s) to dst(%s) mode(%o)", + self.name, + src_path, + dst_path, + mode or 0, + ) + host_path = self._find_parent_path(dst_path) + if host_path: + self.host_cmd(f"mkdir -p {host_path.parent}") + else: + host_path = self.host_path(dst_path) if self.server is None: shutil.copy2(src_path, host_path) else: self.server.remote_put(src_path, host_path) if mode is not None: self.host_cmd(f"chmod {mode:o} {host_path}") - logger.info("node(%s) copied file: %s; mode: %s", self.name, host_path, mode) class CoreNetworkBase(NodeBase): diff --git a/daemon/core/nodes/docker.py b/daemon/core/nodes/docker.py index 8118807f..6dca41e1 100644 --- a/daemon/core/nodes/docker.py +++ b/daemon/core/nodes/docker.py @@ -164,7 +164,7 @@ class DockerNode(CoreNode): """ return f"docker exec -it {self.name} bash" - def privatedir(self, dir_path: str) -> None: + def create_dir(self, dir_path: Path) -> None: """ Create a private directory. @@ -187,7 +187,7 @@ class DockerNode(CoreNode): logger.debug("mounting source(%s) target(%s)", src_path, target_path) raise Exception("not supported") - def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None: + def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None: """ Create a node file with a given mode. @@ -196,7 +196,7 @@ class DockerNode(CoreNode): :param mode: mode for file :return: nothing """ - logger.debug("nodefile filename(%s) mode(%s)", file_path, mode) + logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode) temp = NamedTemporaryFile(delete=False) temp.write(contents.encode("utf-8")) temp.close() @@ -211,26 +211,26 @@ class DockerNode(CoreNode): if self.server is not None: self.host_cmd(f"rm -f {temp_path}") temp_path.unlink() - logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode) - def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None: + def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None: """ Copy a file to a node, following symlinks and preserving metadata. Change file mode if specified. - :param file_path: file name to copy file to + :param dst_path: file name to copy file to :param src_path: file to copy :param mode: mode to copy to :return: nothing """ logger.info( - "node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode + "node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0 ) - self.cmd(f"mkdir -p {file_path.parent}") + self.cmd(f"mkdir -p {dst_path.parent}") if self.server: temp = NamedTemporaryFile(delete=False) temp_path = Path(temp.name) src_path = temp_path self.server.remote_put(src_path, temp_path) - self.client.copy_file(src_path, file_path) - self.cmd(f"chmod {mode:o} {file_path}") + self.client.copy_file(src_path, dst_path) + if mode is not None: + self.cmd(f"chmod {mode:o} {dst_path}") diff --git a/daemon/core/nodes/lxd.py b/daemon/core/nodes/lxd.py index 0f1a2799..54fc8341 100644 --- a/daemon/core/nodes/lxd.py +++ b/daemon/core/nodes/lxd.py @@ -140,7 +140,7 @@ class LxcNode(CoreNode): """ return f"lxc exec {self.name} -- {sh}" - def privatedir(self, dir_path: Path) -> None: + def create_dir(self, dir_path: Path) -> None: """ Create a private directory. @@ -163,7 +163,7 @@ class LxcNode(CoreNode): logger.debug("mounting source(%s) target(%s)", src_path, target_path) raise Exception("not supported") - def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None: + def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None: """ Create a node file with a given mode. @@ -172,7 +172,7 @@ class LxcNode(CoreNode): :param mode: mode for file :return: nothing """ - logger.debug("nodefile filename(%s) mode(%s)", file_path, mode) + logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode) temp = NamedTemporaryFile(delete=False) temp.write(contents.encode("utf-8")) temp.close() @@ -189,27 +189,28 @@ class LxcNode(CoreNode): temp_path.unlink() logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode) - def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None: + def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None: """ Copy a file to a node, following symlinks and preserving metadata. Change file mode if specified. - :param file_path: file name to copy file to + :param dst_path: file name to copy file to :param src_path: file to copy :param mode: mode to copy to :return: nothing """ logger.info( - "node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode + "node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0 ) - self.cmd(f"mkdir -p {file_path.parent}") + self.cmd(f"mkdir -p {dst_path.parent}") if self.server: temp = NamedTemporaryFile(delete=False) temp_path = Path(temp.name) src_path = temp_path self.server.remote_put(src_path, temp_path) - self.client.copy_file(src_path, file_path) - self.cmd(f"chmod {mode:o} {file_path}") + self.client.copy_file(src_path, dst_path) + if mode is not None: + self.cmd(f"chmod {mode:o} {dst_path}") def add_iface(self, iface: CoreInterface, iface_id: int) -> None: super().add_iface(iface, iface_id) diff --git a/daemon/core/services/coreservices.py b/daemon/core/services/coreservices.py index 25cfbf12..b12f21c4 100644 --- a/daemon/core/services/coreservices.py +++ b/daemon/core/services/coreservices.py @@ -492,7 +492,7 @@ class CoreServices: for directory in service.dirs: dir_path = Path(directory) try: - node.privatedir(dir_path) + node.create_dir(dir_path) except (CoreCommandError, CoreError) as e: logger.warning( "error mounting private dir '%s' for service '%s': %s", @@ -553,7 +553,7 @@ class CoreServices: src = src.split("\n")[0] src = utils.expand_corepath(src, node.session, node) # TODO: glob here - node.nodefilecopy(file_path, src, mode=0o644) + node.copy_file(src, file_path, mode=0o644) return True return False @@ -750,7 +750,7 @@ class CoreServices: continue else: cfg = service.generate_config(node, file_name) - node.nodefile(file_path, cfg) + node.create_file(file_path, cfg) def service_reconfigure(self, node: CoreNode, service: "CoreService") -> None: """ @@ -771,7 +771,7 @@ class CoreServices: cfg = service.config_data.get(file_name) if cfg is None: cfg = service.generate_config(node, file_name) - node.nodefile(file_path, cfg) + node.create_file(file_path, cfg) class CoreService: diff --git a/daemon/tests/conftest.py b/daemon/tests/conftest.py index 5ced3fc8..98552540 100644 --- a/daemon/tests/conftest.py +++ b/daemon/tests/conftest.py @@ -60,7 +60,7 @@ def patcher(request): patch_manager.patch_obj( LinuxNetClient, "get_mac", return_value="00:00:00:00:00:00" ) - patch_manager.patch_obj(CoreNode, "nodefile") + patch_manager.patch_obj(CoreNode, "create_file") patch_manager.patch_obj(Session, "write_state") patch_manager.patch_obj(Session, "write_nodes") yield patch_manager diff --git a/daemon/tests/test_config_services.py b/daemon/tests/test_config_services.py index 432f2089..598450c1 100644 --- a/daemon/tests/test_config_services.py +++ b/daemon/tests/test_config_services.py @@ -70,7 +70,7 @@ class TestConfigServices: # then directory = Path(MyService.directories[0]) - node.privatedir.assert_called_with(directory) + node.create_dir.assert_called_with(directory) def test_create_files_custom(self): # given @@ -84,7 +84,7 @@ class TestConfigServices: # then file_path = Path(MyService.files[0]) - node.nodefile.assert_called_with(file_path, text) + node.create_file.assert_called_with(file_path, text) def test_create_files_text(self): # given @@ -96,7 +96,7 @@ class TestConfigServices: # then file_path = Path(MyService.files[0]) - node.nodefile.assert_called_with(file_path, TEMPLATE_TEXT) + node.create_file.assert_called_with(file_path, TEMPLATE_TEXT) def test_run_startup(self): # given diff --git a/daemon/tests/test_gui.py b/daemon/tests/test_gui.py index 5f4ab487..b14f1fb1 100644 --- a/daemon/tests/test_gui.py +++ b/daemon/tests/test_gui.py @@ -441,10 +441,8 @@ class TestGui: coretlv.handle_message(message) if not request.config.getoption("mock"): - directory = str(file_path.parent) - created_directory = directory[1:].replace("/", ".") - create_path = node.directory / created_directory / file_path.name - assert create_path.exists() + expected_path = node.directory / "var.log/test" / file_path.name + assert expected_path.exists() def test_exec_node_tty(self, coretlv: CoreHandler): coretlv.dispatch_replies = mock.MagicMock()