fixed issues in zebra config service, updated config services to start and validate different modes appropriately, added service dependency startup for config services

daemon/core/configservice/base.py

@@ -23,6 +23,10 @@ class ConfigServiceMode(enum.Enum):
     TIMER = 2
 
 
+class ConfigServiceBootError(Exception):
+    pass
+
+
 class ConfigService(abc.ABC):
     # validation period in seconds, how frequent validation is attempted
     validation_period = 0.5
@@ -107,10 +111,16 @@ class ConfigService(abc.ABC):
         raise NotImplementedError
 
     def start(self) -> None:
+        logging.info("node(%s) service(%s) starting...", self.node.name, self.name)
         self.create_dirs()
         self.create_files()
-        self.run_startup()
-        self.run_validation()
+        wait = self.validation_mode == ConfigServiceMode.BLOCKING
+        self.run_startup(wait)
+        if not wait:
+            if self.validation_mode == ConfigServiceMode.TIMER:
+                time.sleep(self.validation_timer)
+            else:
+                self.run_validation()
 
     def stop(self) -> None:
         for cmd in self.shutdown:
@@ -166,13 +176,11 @@ class ConfigService(abc.ABC):
             basename = pathlib.Path(name).name
             if name in self.custom_templates:
                 text = self.custom_templates[name]
-                text = self.clean_text(text)
                 rendered = self.render_text(text, data)
             elif self.templates.has_template(basename):
                 rendered = self.render_template(basename, data)
             else:
                 text = self.get_text_template(name)
-                text = self.clean_text(text)
                 rendered = self.render_text(text, data)
             logging.info(
                 "node(%s) service(%s) template(%s): \n%s",
@@ -183,25 +191,23 @@ class ConfigService(abc.ABC):
             )
             self.node.nodefile(name, rendered)
 
-    def run_startup(self) -> None:
+    def run_startup(self, wait: bool) -> None:
         for cmd in self.startup:
             try:
-                self.node.cmd(cmd)
-            except CoreCommandError:
-                raise CoreError(
-                    f"node({self.node.name}) service({self.name}) "
-                    f"failed startup: {cmd}"
+                self.node.cmd(cmd, wait=wait)
+            except CoreCommandError as e:
+                raise ConfigServiceBootError(
+                    f"node({self.node.name}) service({self.name}) failed startup: {e}"
                 )
 
     def run_validation(self) -> None:
-        wait = self.validation_mode == ConfigServiceMode.BLOCKING
         start = time.monotonic()
-        index = 0
         cmds = self.validate[:]
+        index = 0
         while cmds:
             cmd = cmds[index]
             try:
-                self.node.cmd(cmd, wait=wait)
+                self.node.cmd(cmd)
                 del cmds[index]
                 index += 1
             except CoreCommandError:
@@ -211,10 +217,9 @@ class ConfigService(abc.ABC):
                 )
                 time.sleep(self.validation_period)
 
-            if time.monotonic() - start > 0:
-                raise CoreError(
-                    f"node({self.node.name}) service({self.name}) "
-                    f"failed to validate"
+            if cmds and time.monotonic() - start > 0:
+                raise ConfigServiceBootError(
+                    f"node({self.node.name}) service({self.name}) failed to validate"
                 )
 
     def _render(self, template: Template, data: Dict[str, Any] = None) -> str:
@@ -225,6 +230,7 @@ class ConfigService(abc.ABC):
         )
 
     def render_text(self, text: str, data: Dict[str, Any] = None) -> str:
+        text = self.clean_text(text)
         try:
             template = Template(text)
             return self._render(template, data)
@@ -235,6 +241,13 @@ class ConfigService(abc.ABC):
             )
 
     def render_template(self, basename: str, data: Dict[str, Any] = None) -> str:
+        logging.info(
+            "node(%s) service(%s) rendering template(%s): %s",
+            self.node.name,
+            self.name,
+            basename,
+            data,
+        )
         try:
             template = self.templates.get_template(basename)
             return self._render(template, data)

daemon/core/configservice/dependencies.py

@@ -0,0 +1,97 @@
+import logging
+from typing import TYPE_CHECKING, Dict, List
+
+if TYPE_CHECKING:
+    from core.configservice.base import ConfigService
+
+
+class ConfigServiceDependencies:
+    """
+    Can generate boot paths for services, based on their dependencies. Will validate
+    that all services will be booted and that all dependencies exist within the services provided.
+    """
+
+    def __init__(self, services: Dict[str, "ConfigService"]) -> None:
+        # helpers to check validity
+        self.dependents = {}
+        self.booted = set()
+        self.node_services = {}
+        for service in services.values():
+            self.node_services[service.name] = service
+            for dependency in service.dependencies:
+                dependents = self.dependents.setdefault(dependency, set())
+                dependents.add(service.name)
+
+        # used to find paths
+        self.path = []
+        self.visited = set()
+        self.visiting = set()
+
+    def boot_paths(self) -> List[List["ConfigService"]]:
+        paths = []
+        for name in self.node_services:
+            service = self.node_services[name]
+            if service.name in self.booted:
+                logging.debug(
+                    "skipping service that will already be booted: %s", service.name
+                )
+                continue
+
+            path = self._start(service)
+            if path:
+                paths.append(path)
+
+        if self.booted != set(self.node_services):
+            raise ValueError(
+                "failure to boot all services: %s != %s"
+                % (self.booted, self.node_services.keys())
+            )
+
+        return paths
+
+    def _reset(self) -> None:
+        self.path = []
+        self.visited.clear()
+        self.visiting.clear()
+
+    def _start(self, service: "ConfigService") -> List["ConfigService"]:
+        logging.debug("starting service dependency check: %s", service.name)
+        self._reset()
+        return self._visit(service)
+
+    def _visit(self, current_service: "ConfigService") -> List["ConfigService"]:
+        logging.debug("visiting service(%s): %s", current_service.name, self.path)
+        self.visited.add(current_service.name)
+        self.visiting.add(current_service.name)
+
+        # dive down
+        for service_name in current_service.dependencies:
+            if service_name not in self.node_services:
+                raise ValueError(
+                    "required dependency was not included in node services: %s"
+                    % service_name
+                )
+
+            if service_name in self.visiting:
+                raise ValueError(
+                    "cyclic dependency at service(%s): %s"
+                    % (current_service.name, service_name)
+                )
+
+            if service_name not in self.visited:
+                service = self.node_services[service_name]
+                self._visit(service)
+
+        # add service when bottom is found
+        logging.debug("adding service to boot path: %s", current_service.name)
+        self.booted.add(current_service.name)
+        self.path.append(current_service)
+        self.visiting.remove(current_service.name)
+
+        # rise back up
+        for service_name in self.dependents.get(current_service.name, []):
+            if service_name not in self.visited:
+                service = self.node_services[service_name]
+                self._visit(service)
+
+        return self.path

daemon/core/configservices/quaggaservices/services.py

@@ -62,10 +62,10 @@ class Zebra(ConfigService):
     def data(self) -> Dict[str, Any]:
         quagga_bin_search = self.node.session.options.get_config(
             "quagga_bin_search", default="/usr/local/bin /usr/bin /usr/lib/quagga"
-        )
+        ).strip('"')
         quagga_sbin_search = self.node.session.options.get_config(
             "quagga_sbin_search", default="/usr/local/sbin /usr/sbin /usr/lib/quagga"
-        )
+        ).strip('"')
         quagga_state_dir = constants.QUAGGA_STATE_DIR
         quagga_conf = self.files[0]
 
@@ -102,6 +102,7 @@ class Zebra(ConfigService):
             interfaces=interfaces,
             want_ip4=want_ip4,
             want_ip6=want_ip6,
+            services=services,
         )
 
 

daemon/core/configservices/quaggaservices/templates/Quagga.conf

@@ -1,4 +1,4 @@
-% for ifc, ip4s, ip6s in interfaces:
+% for ifc, ip4s, ip6s, is_control in interfaces:
 interface ${ifc.name}
     % if want_ip4:
         % for addr in ip4s:
@@ -19,5 +19,5 @@ interface ${ifc.name}
 % endfor
 
 % for service in services:
-    ${service.quagga_config()}
+${service.quagga_config()}
 % endfor

daemon/core/emulator/session.py

@@ -1611,8 +1611,7 @@ class Session:
         logging.info("booting node(%s): %s", node.name, [x.name for x in node.services])
         self.add_remove_control_interface(node=node, remove=False)
         self.services.boot_services(node)
-        for service in node.config_services.values():
-            service.start()
+        node.start_config_services()
 
     def boot_nodes(self) -> List[Exception]:
         """

daemon/core/nodes/base.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Type
 import netaddr
 
 from core import utils
+from core.configservice.dependencies import ConfigServiceDependencies
 from core.constants import MOUNT_BIN, VNODED_BIN
 from core.emulator.data import LinkData, NodeData
 from core.emulator.enumerations import LinkTypes, NodeTypes
@@ -296,6 +297,12 @@ class CoreNodeBase(NodeBase):
             raise CoreError(f"node({self.name}) does not have service({name})")
         service.set_config(data)
 
+    def start_config_services(self) -> None:
+        boot_paths = ConfigServiceDependencies(self.config_services).boot_paths()
+        for boot_path in boot_paths:
+            for service in boot_path:
+                service.start()
+
     def makenodedir(self) -> None:
         """
         Create the node directory.