afonso/core-extra
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'rel/5.2' into core-rest-flask

Browse source
This commit is contained in:
Blake J. Harnden 2018-08-03 09:49:27 -07:00
commit f053f11eb4
62 changed files with 2252 additions and 3787 deletions
Download patch file Download diff file Expand all files Collapse all files

2
daemon/Makefile.am
View file

@ -15,7 +15,7 @@ if WANT_DOCS
endif
SCRIPT_FILES := $(notdir $(wildcard scripts/*))
MAN_FILES := $(notdir $(wildcard ../doc/man/*.1))
MAN_FILES := $(notdir $(wildcard ../man/*.1))
# Python package build
noinst_SCRIPTS = build

173
daemon/core/service.py
View file

@ -31,6 +31,92 @@ class ServiceMode(enum.Enum):
TIMER = 2
class ServiceDependencies(object):
"""
Can generate boot paths for services, based on their dependencies. Will validate
that all services will be booted and that all dependencies exist within the services provided.
"""
def __init__(self, services):
# helpers to check validity
self.dependents = {}
self.booted = set()
self.node_services = {}
for service in services:
self.node_services[service.name] = service
for dependency in service.dependencies:
dependents = self.dependents.setdefault(dependency, set())
dependents.add(service.name)
# used to find paths
self.path = []
self.visited = set()
self.visiting = set()
def boot_paths(self):
"""
Generates the boot paths for the services provided to the class.
:return: list of services to boot, in order
:rtype: list[core.service.CoreService]
"""
paths = []
for service in self.node_services.itervalues():
if service.name in self.booted:
logger.debug("skipping service that will already be booted: %s", service.name)
continue
path = self._start(service)
if path:
paths.append(path)
if self.booted != set(self.node_services.iterkeys()):
raise ValueError("failure to boot all services: %s != %s" % (self.booted, self.node_services.keys()))
return paths
def _reset(self):
self.path = []
self.visited.clear()
self.visiting.clear()
def _start(self, service):
logger.debug("starting service dependency check: %s", service.name)
self._reset()
return self._visit(service)
def _visit(self, current_service):
logger.debug("visiting service(%s): %s", current_service.name, self.path)
self.visited.add(current_service.name)
self.visiting.add(current_service.name)
# dive down
for service_name in current_service.dependencies:
if service_name not in self.node_services:
raise ValueError("required dependency was not included in node services: %s" % service_name)
if service_name in self.visiting:
raise ValueError("cyclic dependency at service(%s): %s" % (current_service.name, service_name))
if service_name not in self.visited:
service = self.node_services[service_name]
self._visit(service)
# add service when bottom is found
logger.debug("adding service to boot path: %s", current_service.name)
self.booted.add(current_service.name)
self.path.append(current_service)
self.visiting.remove(current_service.name)
# rise back up
for service_name in self.dependents.get(current_service.name, []):
if service_name not in self.visited:
service = self.node_services[service_name]
self._visit(service)
return self.path
class ServiceShim(object):
keys = ["dirs", "files", "startidx", "cmdup", "cmddown", "cmdval", "meta", "starttime"]
@ -84,7 +170,7 @@ class ServiceShim(object):
:return: nothing
"""
if key not in cls.keys:
raise ValueError('key `%s` not in `%s`' % (key, cls.keys))
raise ValueError("key `%s` not in `%s`" % (key, cls.keys))
# this handles data conversion to int, string, and tuples
if value:
if key == "startidx":
@ -217,87 +303,6 @@ class CoreServices(object):
self.default_services.clear()
self.custom_services.clear()
def create_boot_paths(self, services):
"""
Create boot paths for starting up services based on dependencies. All services provided and their dependencies
must exist within this set of services, to be valid.
:param list[CoreService] services: service to create boot paths for
:return: list of boot paths for services
:rtype: list[list[CoreService]]
"""
# generate service map and find starting points
node_services = {service.name: service for service in services}
all_services = set()
has_dependency = set()
dependency_map = {}
for service in services:
all_services.add(service.name)
if service.dependencies:
has_dependency.add(service.name)
for dependency in service.dependencies:
dependents = dependency_map.setdefault(dependency, set())
dependents.add(service.name)
starting_points = all_services - has_dependency
# cycles means no starting points
if not starting_points:
raise ValueError("no valid service starting points")
stack = [iter(starting_points)]
# information used to traverse dependency graph
visited = set()
path = []
path_set = set()
# store startup orderings
startups = []
startup = []
logger.debug("starting points: %s", starting_points)
while stack:
for service_name in stack[-1]:
service = node_services[service_name]
logger.debug("evaluating: %s", service.name)
# check this is not a cycle
if service.name in path_set:
raise ValueError("service has a cyclic dependency: %s" % service.name)
# check that we have not already visited this node
elif service.name not in visited:
logger.debug("visiting: %s", service.name)
visited.add(service.name)
path.append(service.name)
path_set.add(service.name)
# retrieve and dependent services and add to stack
dependents = iter(dependency_map.get(service.name, []))
stack.append(dependents)
startup.append(service)
break
# for loop completed without a break
else:
logger.debug("finished a visit: path(%s)", path)
if path:
path_set.remove(path.pop())
if not path and startup:
# finalize startup path
startups.append(startup)
# reset new startup path
startup = []
stack.pop()
if visited != all_services:
raise ValueError("failure to visit all services for boot path")
return startups
def get_default_services(self, node_type):
"""
Get the list of default services that should be enabled for a
@ -422,7 +427,7 @@ class CoreServices(object):
pool = ThreadPool()
results = []
boot_paths = self.create_boot_paths(node.services)
boot_paths = ServiceDependencies(node.services).boot_paths()
for boot_path in boot_paths:
result = pool.apply_async(self._start_boot_paths, (node, boot_path))
results.append(result)
@ -440,7 +445,7 @@ class CoreServices(object):
:param list[CoreService] boot_path: service to start in dependent order
:return: nothing
"""
logger.debug("booting node service dependencies: %s", boot_path)
logger.info("booting node services: %s", boot_path)
for service in boot_path:
self.boot_service(node, service)

2
daemon/core/session.py
View file

@ -734,7 +734,7 @@ class Session(object):
pool.join()
for result in results:
result.get()
logger.info("BOOT RUN TIME: %s", time.time() - start)
logger.debug("boot run time: %s", time.time() - start)
self.update_control_interface_hosts()

2
daemon/core/xml/emanexml.py
View file

@ -155,7 +155,7 @@ def build_node_platform_xml(emane_manager, control_net, node, nem_id, platform_x
eventdev = None
platform_element = platform_xmls.get(key)
if not platform_element:
if platform_element is None:
platform_element = etree.Element("platform")
if otadev:

98
daemon/examples/myservices/sample.py
View file

@ -2,57 +2,81 @@
Sample user-defined service.
"""
from core.misc.ipaddress import Ipv4Prefix
from core.service import CoreService
from core.service import ServiceMode
## Custom CORE Service
class MyService(CoreService):
"""
This is a sample user-defined service.
"""
# a unique name is required, without spaces
### Service Attributes
# Name used as a unique ID for this service and is required, no spaces.
name = "MyService"
# you can create your own group here
# Allows you to group services within the GUI under a common name.
group = "Utility"
# list executables that this service requires
# Executables this service depends on to function, if executable is not on the path, service will not be loaded.
executables = ()
# list of other services this service depends on
# Services that this service depends on for startup, tuple of service names.
dependencies = ()
# per-node directories
# Directories that this service will create within a node.
dirs = ()
# generated files (without a full path this file goes in the node's dir,
# e.g. /tmp/pycore.12345/n1.conf/)
configs = ("myservice.sh",)
# list of startup commands, also may be generated during startup
startup = ("sh myservice.sh",)
# list of shutdown commands
# Files that this service will generate, without a full path this file goes in the node's directory.
# e.g. /tmp/pycore.12345/n1.conf/myfile
configs = ("sh myservice1.sh", "sh myservice2.sh")
# Commands used to start this service, any non-zero exit code will cause a failure.
startup = ("sh %s" % configs[0], "sh %s" % configs[1])
# Commands used to validate that a service was started, any non-zero exit code will cause a failure.
validate = ()
# Validation mode, used to determine startup success.
# * NON_BLOCKING - runs startup commands, and validates success with validation commands
# * BLOCKING - runs startup commands, and validates success with the startup commands themselves
# * TIMER - runs startup commands, and validates success by waiting for "validation_timer" alone
validation_mode = ServiceMode.NON_BLOCKING
# Time for a service to wait before running validation commands or determining success in TIMER mode.
validation_timer = 0
# Shutdown commands to stop this service.
shutdown = ()
### On Load
@classmethod
def on_load(cls):
# Provides a way to run some arbitrary logic when the service is loaded, possibly to help facilitate
# dynamic settings for the environment.
pass
### Get Configs
@classmethod
def get_configs(cls, node):
# Provides a way to dynamically generate the config files from the node a service will run.
# Defaults to the class definition and can be left out entirely if not needed.
return cls.configs
### Generate Config
@classmethod
def generate_config(cls, node, filename):
"""
Return a string that will be written to filename, or sent to the
GUI for user customization.
"""
# Returns a string representation for a file, given the node the service is starting on the config filename
# that this information will be used for. This must be defined, if "configs" are defined.
cfg = "#!/bin/sh\n"
cfg += "# auto-generated by MyService (sample.py)\n"
for ifc in node.netifs():
cfg += 'echo "Node %s has interface %s"\n' % (node.name, ifc.name)
# here we do something interesting
cfg += "\n".join(map(cls.subnetentry, ifc.addrlist))
break
if filename == cls.configs[0]:
cfg += "# auto-generated by MyService (sample.py)\n"
for ifc in node.netifs():
cfg += 'echo "Node %s has interface %s"\n' % (node.name, ifc.name)
elif filename == cls.configs[1]:
cfg += "echo hello"
return cfg
@staticmethod
def subnetentry(x):
"""
Generate a subnet declaration block given an IPv4 prefix string
for inclusion in the config file.
"""
if x.find(":") >= 0:
# this is an IPv6 address
return ""
else:
net = Ipv4Prefix(x)
return 'echo " network %s"' % net
### Get Startup
@classmethod
def get_startup(cls, node):
# Provides a way to dynamically generate the startup commands from the node a service will run.
# Defaults to the class definition and can be left out entirely if not needed.
return cls.startup
### Get Validate
@classmethod
def get_validate(cls, node):
# Provides a way to dynamically generate the validate commands from the node a service will run.
# Defaults to the class definition and can be left out entirely if not needed.
return cls.validate

2
daemon/setup.py
View file

@ -33,7 +33,7 @@ data_files = [
"data/core.conf",
"data/logging.conf",
]),
(_MAN_DIR, glob_files("../doc/man/**.1")),
(_MAN_DIR, glob_files("../man/**.1")),
]
data_files.extend(recursive_files(_EXAMPLES_DIR, "examples"))

65
daemon/tests/test_services.py
View file

@ -3,6 +3,7 @@ import os
import pytest
from core.service import CoreService
from core.service import ServiceDependencies
from core.service import ServiceManager
_PATH = os.path.abspath(os.path.dirname(__file__))
@ -19,20 +20,20 @@ class ServiceA(CoreService):
class ServiceB(CoreService):
name = "B"
dependencies = ("C",)
dependencies = ()
class ServiceC(CoreService):
name = "C"
dependencies = ()
dependencies = ("B", "D")
class ServiceD(CoreService):
name = "D"
dependencies = ("A",)
dependencies = ()
class ServiceE(CoreService):
class ServiceBadDependency(CoreService):
name = "E"
dependencies = ("Z",)
@ -42,6 +43,10 @@ class ServiceF(CoreService):
dependencies = ()
class ServiceCycleDependency(CoreService):
name = "G"
class TestServices:
def test_service_all_files(self, session):
# given
@ -245,7 +250,23 @@ class TestServices:
assert default_service == my_service
assert custom_service and custom_service != my_service
def test_services_dependencies(self, session):
def test_services_dependencies(self):
# given
services = [
ServiceA,
ServiceB,
ServiceC,
ServiceD,
ServiceF
]
# when
boot_paths = ServiceDependencies(services).boot_paths()
# then
assert len(boot_paths) == 2
def test_services_dependencies_not_present(self):
# given
services = [
ServiceA,
@ -253,39 +274,25 @@ class TestServices:
ServiceC,
ServiceD,
ServiceF,
ServiceBadDependency
]
# when
startups = session.services.create_boot_paths(services)
# when, then
with pytest.raises(ValueError):
ServiceDependencies(services).boot_paths()
# then
assert len(startups) == 2
def test_services_dependencies_not_present(self, session):
def test_services_dependencies_cycle(self):
# given
service_d = ServiceD()
service_d.dependencies = ("C",)
services = [
ServiceA,
ServiceB,
ServiceC,
ServiceE
]
# when
with pytest.raises(ValueError):
session.services.create_boot_paths(services)
def test_services_dependencies_cycle(self, session):
# given
service_c = ServiceC()
service_c.dependencies = ("D",)
services = [
ServiceA,
ServiceB,
service_c,
ServiceD,
service_d,
ServiceF
]
# when
# when, then
with pytest.raises(ValueError):
session.services.create_boot_paths(services)
ServiceDependencies(services).boot_paths()