daemon: added capability to config services to shadow directory structures, from a given path, or from a local source, files may be templates or a straight copy and can be sourced from node named unique paths for node specific files, also refactored and renamed file creation related functions for nodes

This commit is contained in:
Blake Harnden 2021-09-17 14:34:37 -07:00
parent b96dc621cd
commit bd896d1336
10 changed files with 212 additions and 130 deletions

View file

@ -3,8 +3,9 @@ import enum
import inspect
import logging
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from mako import exceptions
from mako.lookup import TemplateLookup
@ -28,6 +29,14 @@ class ConfigServiceBootError(Exception):
pass
@dataclass
class ShadowDir:
path: str
src: Optional[str] = None
templates: bool = False
has_node_paths: bool = False
class ConfigService(abc.ABC):
"""
Base class for creating configurable services.
@ -39,6 +48,9 @@ class ConfigService(abc.ABC):
# time to wait in seconds for determining if service started successfully
validation_timer: int = 5
# directories to shadow and copy files from
shadow_directories: List[ShadowDir] = []
def __init__(self, node: CoreNode) -> None:
"""
Create ConfigService instance.
@ -135,6 +147,7 @@ class ConfigService(abc.ABC):
:raises ConfigServiceBootError: when there is an error starting service
"""
logger.info("node(%s) service(%s) starting...", self.node.name, self.name)
self.create_shadow_dirs()
self.create_dirs()
self.create_files()
wait = self.validation_mode == ConfigServiceMode.BLOCKING
@ -169,6 +182,64 @@ class ConfigService(abc.ABC):
self.stop()
self.start()
def create_shadow_dirs(self) -> None:
"""
Creates a shadow of a host system directory recursively
to be mapped and live within a node.
:return: nothing
:raises CoreError: when there is a failure creating a directory or file
"""
for shadow_dir in self.shadow_directories:
# setup shadow and src paths, using node unique paths when configured
shadow_path = Path(shadow_dir.path)
if shadow_dir.src is None:
src_path = shadow_path
else:
src_path = Path(shadow_dir.src)
if shadow_dir.has_node_paths:
src_path = src_path / self.node.name
# validate shadow and src paths
if not shadow_path.is_absolute():
raise CoreError(f"shadow dir({shadow_path}) is not absolute")
if not src_path.is_absolute():
raise CoreError(f"shadow source dir({src_path}) is not absolute")
if not src_path.is_dir():
raise CoreError(f"shadow source dir({src_path}) does not exist")
# create root of the shadow path within node
logger.info(
"node(%s) creating shadow directory(%s) src(%s) node paths(%s) "
"templates(%s)",
self.node.name,
shadow_path,
src_path,
shadow_dir.has_node_paths,
shadow_dir.templates,
)
self.node.create_dir(shadow_path)
# find all directories and files to create
dir_paths = []
file_paths = []
for path in src_path.rglob("*"):
shadow_src_path = shadow_path / path.relative_to(src_path)
if path.is_dir():
dir_paths.append(shadow_src_path)
else:
file_paths.append((path, shadow_src_path))
# create all directories within node
for path in dir_paths:
self.node.create_dir(path)
# create all files within node, from templates when configured
data = self.data()
templates = TemplateLookup(directories=src_path)
for path, dst_path in file_paths:
if shadow_dir.templates:
template = templates.get_template(path.name)
rendered = self._render(template, data)
self.node.create_file(dst_path, rendered)
else:
self.node.copy_file(path, dst_path)
def create_dirs(self) -> None:
"""
Creates directories for service.
@ -176,10 +247,11 @@ class ConfigService(abc.ABC):
:return: nothing
:raises CoreError: when there is a failure creating a directory
"""
for directory in self.directories:
logger.debug("creating config service directories")
for directory in sorted(self.directories):
dir_path = Path(directory)
try:
self.node.privatedir(dir_path)
self.node.create_dir(dir_path)
except (CoreCommandError, CoreError):
raise CoreError(
f"node({self.node.name}) service({self.name}) "
@ -221,17 +293,21 @@ class ConfigService(abc.ABC):
:return: mapping of files to templates
"""
templates = {}
for name in self.files:
basename = Path(name).name
if name in self.custom_templates:
template = self.custom_templates[name]
template = self.clean_text(template)
elif self.templates.has_template(basename):
template = self.templates.get_template(basename).source
for file in self.files:
file_path = Path(file)
if file_path.is_absolute():
template_path = str(file_path.relative_to("/"))
else:
template = self.get_text_template(name)
template_path = str(file_path)
if file in self.custom_templates:
template = self.custom_templates[file]
template = self.clean_text(template)
templates[name] = template
elif self.templates.has_template(template_path):
template = self.templates.get_template(template_path).source
else:
template = self.get_text_template(file)
template = self.clean_text(template)
templates[file] = template
return templates
def create_files(self) -> None:
@ -241,24 +317,20 @@ class ConfigService(abc.ABC):
:return: nothing
"""
data = self.data()
for name in self.files:
file_path = Path(name)
if name in self.custom_templates:
text = self.custom_templates[name]
for file in sorted(self.files):
logger.debug(
"node(%s) service(%s) template(%s)", self.node.name, self.name, file
)
file_path = Path(file)
if file in self.custom_templates:
text = self.custom_templates[file]
rendered = self.render_text(text, data)
elif self.templates.has_template(file_path.name):
rendered = self.render_template(file_path.name, data)
else:
text = self.get_text_template(name)
text = self.get_text_template(file)
rendered = self.render_text(text, data)
logger.debug(
"node(%s) service(%s) template(%s): \n%s",
self.node.name,
self.name,
name,
rendered,
)
self.node.nodefile(file_path, rendered)
self.node.create_file(file_path, rendered)
def run_startup(self, wait: bool) -> None:
"""

View file

@ -1,49 +0,0 @@
from typing import Dict, List
from core.config import Configuration
from core.configservice.base import ConfigService, ConfigServiceMode
from core.emulator.enumerations import ConfigDataTypes
class SimpleService(ConfigService):
name: str = "Simple"
group: str = "SimpleGroup"
directories: List[str] = ["/etc/quagga", "/usr/local/lib"]
files: List[str] = ["test1.sh", "test2.sh"]
executables: List[str] = []
dependencies: List[str] = []
startup: List[str] = []
validate: List[str] = []
shutdown: List[str] = []
validation_mode: ConfigServiceMode = ConfigServiceMode.BLOCKING
default_configs: List[Configuration] = [
Configuration(id="value1", type=ConfigDataTypes.STRING, label="Text"),
Configuration(id="value2", type=ConfigDataTypes.BOOL, label="Boolean"),
Configuration(
id="value3",
type=ConfigDataTypes.STRING,
label="Multiple Choice",
options=["value1", "value2", "value3"],
),
]
modes: Dict[str, Dict[str, str]] = {
"mode1": {"value1": "value1", "value2": "0", "value3": "value2"},
"mode2": {"value1": "value2", "value2": "1", "value3": "value3"},
"mode3": {"value1": "value3", "value2": "0", "value3": "value1"},
}
def get_text_template(self, name: str) -> str:
if name == "test1.sh":
return """
# sample script 1
# node id(${node.id}) name(${node.name})
# config: ${config}
echo hello
"""
elif name == "test2.sh":
return """
# sample script 2
# node id(${node.id}) name(${node.name})
# config: ${config}
echo hello2
"""

View file

@ -690,11 +690,11 @@ class Session:
:param data: file data
:return: nothing
"""
node = self.get_node(node_id, CoreNodeBase)
node = self.get_node(node_id, CoreNode)
if src_path is not None:
node.addfile(src_path, file_path)
elif data is not None:
node.nodefile(file_path, data)
node.create_file(file_path, data)
def clear(self) -> None:
"""

View file

@ -237,7 +237,17 @@ class CoreNodeBase(NodeBase):
raise NotImplementedError
@abc.abstractmethod
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
def create_dir(self, dir_path: Path) -> None:
"""
Create a node private directory.
:param dir_path: path to create
:return: nothing
"""
raise NotImplementedError
@abc.abstractmethod
def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
@ -248,6 +258,19 @@ class CoreNodeBase(NodeBase):
"""
raise NotImplementedError
@abc.abstractmethod
def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
"""
Copy source file to node host destination, updating the file mode when
provided.
:param src_path: source file to copy
:param dst_path: node host destination
:param mode: file mode
:return: nothing
"""
raise NotImplementedError
@abc.abstractmethod
def addfile(self, src_path: Path, file_path: Path) -> None:
"""
@ -567,7 +590,7 @@ class CoreNode(CoreNodeBase):
# create private directories
for dir_path in PRIVATE_DIRS:
self.privatedir(dir_path)
self.create_dir(dir_path)
def shutdown(self) -> None:
"""
@ -648,16 +671,20 @@ class CoreNode(CoreNodeBase):
else:
return f"ssh -X -f {self.server.host} xterm -e {terminal}"
def privatedir(self, dir_path: Path) -> None:
def create_dir(self, dir_path: Path) -> None:
"""
Create a private directory.
Create a node private directory.
:param dir_path: path to create
:return: nothing
"""
logger.info("creating private directory: %s", dir_path)
if not str(dir_path).startswith("/"):
if not dir_path.is_absolute():
raise CoreError(f"private directory path not fully qualified: {dir_path}")
logger.debug("node(%s) creating private directory: %s", self.name, dir_path)
parent_path = self._find_parent_path(dir_path)
if parent_path:
self.host_cmd(f"mkdir -p {parent_path}")
else:
host_path = self.host_path(dir_path, is_dir=True)
self.host_cmd(f"mkdir -p {host_path}")
self.mount(host_path, dir_path)
@ -880,15 +907,39 @@ class CoreNode(CoreNodeBase):
self.host_cmd(f"mkdir -p {directory}")
self.server.remote_put(src_path, file_path)
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
def _find_parent_path(self, path: Path) -> Optional[Path]:
"""
Create a node file with a given mode.
Check if there is an existing mounted parent directory created for this node.
:param file_path: name of file to create
:param path: existing parent path to use
:return: exist parent path if exists, None otherwise
"""
logger.debug("looking for existing parent: %s", path)
existing_path = None
for parent in path.parents:
node_path = self.host_path(parent, is_dir=True)
if node_path == self.directory:
break
if self.path_exists(str(node_path)):
relative_path = path.relative_to(parent)
existing_path = node_path / relative_path
break
return existing_path
def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create file within a node at the given path, using contents and mode.
:param file_path: desired path for file
:param contents: contents of file
:param mode: mode for file
:param mode: mode to create file with
:return: nothing
"""
logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
host_path = self._find_parent_path(file_path)
if host_path:
self.host_cmd(f"mkdir -p {host_path.parent}")
else:
host_path = self.host_path(file_path)
directory = host_path.parent
if self.server is None:
@ -901,26 +952,35 @@ class CoreNode(CoreNodeBase):
self.host_cmd(f"mkdir -m {0o755:o} -p {directory}")
self.server.remote_put_temp(host_path, contents)
self.host_cmd(f"chmod {mode:o} {host_path}")
logger.debug("node(%s) added file: %s; mode: 0%o", self.name, host_path, mode)
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
Copy source file to node host destination, updating the file mode when
provided.
:param file_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:param src_path: source file to copy
:param dst_path: node host destination
:param mode: file mode
:return: nothing
"""
host_path = self.host_path(file_path)
logger.debug(
"node(%s) copying file src(%s) to dst(%s) mode(%o)",
self.name,
src_path,
dst_path,
mode or 0,
)
host_path = self._find_parent_path(dst_path)
if host_path:
self.host_cmd(f"mkdir -p {host_path.parent}")
else:
host_path = self.host_path(dst_path)
if self.server is None:
shutil.copy2(src_path, host_path)
else:
self.server.remote_put(src_path, host_path)
if mode is not None:
self.host_cmd(f"chmod {mode:o} {host_path}")
logger.info("node(%s) copied file: %s; mode: %s", self.name, host_path, mode)
class CoreNetworkBase(NodeBase):

View file

@ -164,7 +164,7 @@ class DockerNode(CoreNode):
"""
return f"docker exec -it {self.name} bash"
def privatedir(self, dir_path: str) -> None:
def create_dir(self, dir_path: Path) -> None:
"""
Create a private directory.
@ -187,7 +187,7 @@ class DockerNode(CoreNode):
logger.debug("mounting source(%s) target(%s)", src_path, target_path)
raise Exception("not supported")
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
@ -196,7 +196,7 @@ class DockerNode(CoreNode):
:param mode: mode for file
:return: nothing
"""
logger.debug("nodefile filename(%s) mode(%s)", file_path, mode)
logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
temp = NamedTemporaryFile(delete=False)
temp.write(contents.encode("utf-8"))
temp.close()
@ -211,26 +211,26 @@ class DockerNode(CoreNode):
if self.server is not None:
self.host_cmd(f"rm -f {temp_path}")
temp_path.unlink()
logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
:param file_path: file name to copy file to
:param dst_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:return: nothing
"""
logger.info(
"node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
"node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0
)
self.cmd(f"mkdir -p {file_path.parent}")
self.cmd(f"mkdir -p {dst_path.parent}")
if self.server:
temp = NamedTemporaryFile(delete=False)
temp_path = Path(temp.name)
src_path = temp_path
self.server.remote_put(src_path, temp_path)
self.client.copy_file(src_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")
self.client.copy_file(src_path, dst_path)
if mode is not None:
self.cmd(f"chmod {mode:o} {dst_path}")

View file

@ -140,7 +140,7 @@ class LxcNode(CoreNode):
"""
return f"lxc exec {self.name} -- {sh}"
def privatedir(self, dir_path: Path) -> None:
def create_dir(self, dir_path: Path) -> None:
"""
Create a private directory.
@ -163,7 +163,7 @@ class LxcNode(CoreNode):
logger.debug("mounting source(%s) target(%s)", src_path, target_path)
raise Exception("not supported")
def nodefile(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
def create_file(self, file_path: Path, contents: str, mode: int = 0o644) -> None:
"""
Create a node file with a given mode.
@ -172,7 +172,7 @@ class LxcNode(CoreNode):
:param mode: mode for file
:return: nothing
"""
logger.debug("nodefile filename(%s) mode(%s)", file_path, mode)
logger.debug("node(%s) create file(%s) mode(%o)", self.name, file_path, mode)
temp = NamedTemporaryFile(delete=False)
temp.write(contents.encode("utf-8"))
temp.close()
@ -189,27 +189,28 @@ class LxcNode(CoreNode):
temp_path.unlink()
logger.debug("node(%s) added file: %s; mode: 0%o", self.name, file_path, mode)
def nodefilecopy(self, file_path: Path, src_path: Path, mode: int = None) -> None:
def copy_file(self, src_path: Path, dst_path: Path, mode: int = None) -> None:
"""
Copy a file to a node, following symlinks and preserving metadata.
Change file mode if specified.
:param file_path: file name to copy file to
:param dst_path: file name to copy file to
:param src_path: file to copy
:param mode: mode to copy to
:return: nothing
"""
logger.info(
"node file copy file(%s) source(%s) mode(%s)", file_path, src_path, mode
"node file copy file(%s) source(%s) mode(%o)", dst_path, src_path, mode or 0
)
self.cmd(f"mkdir -p {file_path.parent}")
self.cmd(f"mkdir -p {dst_path.parent}")
if self.server:
temp = NamedTemporaryFile(delete=False)
temp_path = Path(temp.name)
src_path = temp_path
self.server.remote_put(src_path, temp_path)
self.client.copy_file(src_path, file_path)
self.cmd(f"chmod {mode:o} {file_path}")
self.client.copy_file(src_path, dst_path)
if mode is not None:
self.cmd(f"chmod {mode:o} {dst_path}")
def add_iface(self, iface: CoreInterface, iface_id: int) -> None:
super().add_iface(iface, iface_id)

View file

@ -492,7 +492,7 @@ class CoreServices:
for directory in service.dirs:
dir_path = Path(directory)
try:
node.privatedir(dir_path)
node.create_dir(dir_path)
except (CoreCommandError, CoreError) as e:
logger.warning(
"error mounting private dir '%s' for service '%s': %s",
@ -553,7 +553,7 @@ class CoreServices:
src = src.split("\n")[0]
src = utils.expand_corepath(src, node.session, node)
# TODO: glob here
node.nodefilecopy(file_path, src, mode=0o644)
node.copy_file(src, file_path, mode=0o644)
return True
return False
@ -750,7 +750,7 @@ class CoreServices:
continue
else:
cfg = service.generate_config(node, file_name)
node.nodefile(file_path, cfg)
node.create_file(file_path, cfg)
def service_reconfigure(self, node: CoreNode, service: "CoreService") -> None:
"""
@ -771,7 +771,7 @@ class CoreServices:
cfg = service.config_data.get(file_name)
if cfg is None:
cfg = service.generate_config(node, file_name)
node.nodefile(file_path, cfg)
node.create_file(file_path, cfg)
class CoreService:

View file

@ -60,7 +60,7 @@ def patcher(request):
patch_manager.patch_obj(
LinuxNetClient, "get_mac", return_value="00:00:00:00:00:00"
)
patch_manager.patch_obj(CoreNode, "nodefile")
patch_manager.patch_obj(CoreNode, "create_file")
patch_manager.patch_obj(Session, "write_state")
patch_manager.patch_obj(Session, "write_nodes")
yield patch_manager

View file

@ -70,7 +70,7 @@ class TestConfigServices:
# then
directory = Path(MyService.directories[0])
node.privatedir.assert_called_with(directory)
node.create_dir.assert_called_with(directory)
def test_create_files_custom(self):
# given
@ -84,7 +84,7 @@ class TestConfigServices:
# then
file_path = Path(MyService.files[0])
node.nodefile.assert_called_with(file_path, text)
node.create_file.assert_called_with(file_path, text)
def test_create_files_text(self):
# given
@ -96,7 +96,7 @@ class TestConfigServices:
# then
file_path = Path(MyService.files[0])
node.nodefile.assert_called_with(file_path, TEMPLATE_TEXT)
node.create_file.assert_called_with(file_path, TEMPLATE_TEXT)
def test_run_startup(self):
# given

View file

@ -441,10 +441,8 @@ class TestGui:
coretlv.handle_message(message)
if not request.config.getoption("mock"):
directory = str(file_path.parent)
created_directory = directory[1:].replace("/", ".")
create_path = node.directory / created_directory / file_path.name
assert create_path.exists()
expected_path = node.directory / "var.log/test" / file_path.name
assert expected_path.exists()
def test_exec_node_tty(self, coretlv: CoreHandler):
coretlv.dispatch_replies = mock.MagicMock()